diff --git a/include/contractor/contractor.hpp b/include/contractor/contractor.hpp index d4b3d1fecdd..1c6b5873665 100644 --- a/include/contractor/contractor.hpp +++ b/include/contractor/contractor.hpp @@ -66,8 +66,6 @@ class Contractor std::vector &is_core_node, std::vector &inout_node_levels) const; void WriteCoreNodeMarker(std::vector &&is_core_node) const; - void WriteNodeLevels(std::vector &&node_levels) const; - void ReadNodeLevels(std::vector &contraction_order) const; void WriteContractedGraph(unsigned number_of_edge_based_nodes, util::DeallocatingVector contracted_edge_list); void FindComponents(unsigned max_edge_id, diff --git a/include/contractor/files.hpp b/include/contractor/files.hpp index 55eba978ca7..8dd570e8725 100644 --- a/include/contractor/files.hpp +++ b/include/contractor/files.hpp @@ -6,6 +6,7 @@ #include "util/serialization.hpp" #include "storage/io.hpp" +#include "storage/serialization.hpp" namespace osrm { @@ -43,6 +44,24 @@ writeGraph(const boost::filesystem::path &path, unsigned checksum, const QueryGr writer.WriteOne(checksum); util::serialization::write(writer, graph); } + +// reads .levels file +inline void readLevels(const boost::filesystem::path &path, std::vector &node_levels) +{ + const auto fingerprint = storage::io::FileReader::VerifyFingerprint; + storage::io::FileReader reader{path, fingerprint}; + + storage::serialization::read(reader, node_levels); +} + +// writes .levels file +inline void writeLevels(const boost::filesystem::path &path, const std::vector &node_levels) +{ + const auto fingerprint = storage::io::FileWriter::GenerateFingerprint; + storage::io::FileWriter writer{path, fingerprint}; + + storage::serialization::write(writer, node_levels); +} } } } diff --git a/include/engine/engine.hpp b/include/engine/engine.hpp index 978d750bc73..7f32d3e720f 100644 --- a/include/engine/engine.hpp +++ b/include/engine/engine.hpp @@ -22,6 +22,7 @@ #include "engine/status.hpp" #include "util/exception.hpp" #include "util/exception_utils.hpp" +#include "util/fingerprint.hpp" #include "util/json_container.hpp" #include @@ -184,7 +185,8 @@ bool Engine::CheckCompability(const Engin auto mem = storage::makeSharedMemory(barrier.data().region); auto layout = reinterpret_cast(mem->Ptr()); - return layout->GetBlockSize(storage::DataLayout::CH_CORE_MARKER) > 4; + return layout->GetBlockSize(storage::DataLayout::CH_CORE_MARKER) > + sizeof(std::uint64_t) + sizeof(util::FingerPrint); } else { @@ -194,8 +196,8 @@ bool Engine::CheckCompability(const Engin in.seekg(0, std::ios::end); std::size_t size = in.tellg(); - // An empty core files is only the 4 byte size header. - return size > sizeof(std::uint32_t); + // An empty core files is only the 8 byte size header plus the 8 byte Fingerprint. + return size > sizeof(std::uint64_t) + sizeof(util::FingerPrint); } } } diff --git a/include/extractor/files.hpp b/include/extractor/files.hpp index 5b1be335032..ae941e55d4f 100644 --- a/include/extractor/files.hpp +++ b/include/extractor/files.hpp @@ -71,7 +71,7 @@ inline void writeNBGMapping(const boost::filesystem::path &path, // reads .osrm.datasource_names inline void readDatasources(const boost::filesystem::path &path, Datasources &sources) { - const auto fingerprint = storage::io::FileReader::HasNoFingerprint; + const auto fingerprint = storage::io::FileReader::VerifyFingerprint; storage::io::FileReader reader{path, fingerprint}; serialization::read(reader, sources); @@ -80,7 +80,7 @@ inline void readDatasources(const boost::filesystem::path &path, Datasources &so // writes .osrm.datasource_names inline void writeDatasources(const boost::filesystem::path &path, Datasources &sources) { - const auto fingerprint = storage::io::FileWriter::HasNoFingerprint; + const auto fingerprint = storage::io::FileWriter::GenerateFingerprint; storage::io::FileWriter writer{path, fingerprint}; serialization::write(writer, sources); @@ -93,7 +93,7 @@ inline void readSegmentData(const boost::filesystem::path &path, SegmentDataT &s static_assert(std::is_same::value || std::is_same::value, ""); - const auto fingerprint = storage::io::FileReader::HasNoFingerprint; + const auto fingerprint = storage::io::FileReader::VerifyFingerprint; storage::io::FileReader reader{path, fingerprint}; serialization::read(reader, segment_data); @@ -106,7 +106,7 @@ inline void writeSegmentData(const boost::filesystem::path &path, const SegmentD static_assert(std::is_same::value || std::is_same::value, ""); - const auto fingerprint = storage::io::FileWriter::HasNoFingerprint; + const auto fingerprint = storage::io::FileWriter::GenerateFingerprint; storage::io::FileWriter writer{path, fingerprint}; serialization::write(writer, segment_data); @@ -119,7 +119,7 @@ inline void readTurnData(const boost::filesystem::path &path, TurnDataT &turn_da static_assert(std::is_same::value || std::is_same::value, ""); - const auto fingerprint = storage::io::FileReader::HasNoFingerprint; + const auto fingerprint = storage::io::FileReader::VerifyFingerprint; storage::io::FileReader reader{path, fingerprint}; serialization::read(reader, turn_data); @@ -132,7 +132,7 @@ inline void writeTurnData(const boost::filesystem::path &path, const TurnDataT & static_assert(std::is_same::value || std::is_same::value, ""); - const auto fingerprint = storage::io::FileWriter::HasNoFingerprint; + const auto fingerprint = storage::io::FileWriter::GenerateFingerprint; storage::io::FileWriter writer{path, fingerprint}; serialization::write(writer, turn_data); @@ -149,7 +149,7 @@ inline void readTurnLaneDescriptions(const boost::filesystem::path &path, ""); static_assert(std::is_same::value, ""); - const auto fingerprint = storage::io::FileReader::HasNoFingerprint; + const auto fingerprint = storage::io::FileReader::VerifyFingerprint; storage::io::FileReader reader{path, fingerprint}; storage::serialization::read(reader, turn_offsets); @@ -167,7 +167,7 @@ inline void writeTurnLaneDescriptions(const boost::filesystem::path &path, ""); static_assert(std::is_same::value, ""); - const auto fingerprint = storage::io::FileWriter::HasNoFingerprint; + const auto fingerprint = storage::io::FileWriter::GenerateFingerprint; storage::io::FileWriter writer{path, fingerprint}; storage::serialization::write(writer, turn_offsets); diff --git a/include/extractor/raster_source.hpp b/include/extractor/raster_source.hpp index 602c6f02daa..1c0473a35f7 100644 --- a/include/extractor/raster_source.hpp +++ b/include/extractor/raster_source.hpp @@ -47,7 +47,7 @@ class RasterGrid storage::io::FileReader file_reader(filepath, storage::io::FileReader::HasNoFingerprint); std::string buffer; - buffer.resize(file_reader.Size()); + buffer.resize(file_reader.GetSize()); BOOST_ASSERT(buffer.size() > 1); diff --git a/include/storage/io.hpp b/include/storage/io.hpp index 6a7741ea7ad..12fd18e3b9b 100644 --- a/include/storage/io.hpp +++ b/include/storage/io.hpp @@ -23,10 +23,6 @@ namespace io class FileReader { - private: - const boost::filesystem::path filepath; - boost::filesystem::ifstream input_stream; - public: class LineWrapper : public std::string { @@ -50,7 +46,7 @@ class FileReader } FileReader(const boost::filesystem::path &filepath_, const FingerprintFlag flag) - : filepath(filepath_) + : filepath(filepath_), fingerprint(flag) { input_stream.open(filepath, std::ios::binary); if (!input_stream) @@ -75,7 +71,15 @@ class FileReader // restore the current position input_stream.seekg(positon, std::ios::beg); - return file_size; + + if (fingerprint == FingerprintFlag::VerifyFingerprint) + { + return std::size_t(file_size) - sizeof(util::FingerPrint); + } + else + { + return file_size; + } } /* Read count objects of type T into pointer dest */ @@ -125,7 +129,6 @@ class FileReader /*******************************************/ - std::uint32_t ReadElementCount32() { return ReadOne(); } std::uint64_t ReadElementCount64() { return ReadOne(); } template std::size_t ReadVectorSize() @@ -164,14 +167,10 @@ class FileReader return true; } - std::size_t Size() - { - auto current_pos = input_stream.tellg(); - input_stream.seekg(0, input_stream.end); - auto length = input_stream.tellg(); - input_stream.seekg(current_pos, input_stream.beg); - return length; - } + private: + const boost::filesystem::path filepath; + boost::filesystem::ifstream input_stream; + FingerprintFlag fingerprint; }; class FileWriter @@ -230,7 +229,6 @@ class FileWriter template void WriteOne(const T tmp) { WriteFrom(tmp); } - void WriteElementCount32(const std::uint32_t count) { WriteOne(count); } void WriteElementCount64(const std::uint64_t count) { WriteOne(count); } void WriteFingerprint() diff --git a/include/util/graph_loader.hpp b/include/util/graph_loader.hpp index e413aadf895..01e246b73c5 100644 --- a/include/util/graph_loader.hpp +++ b/include/util/graph_loader.hpp @@ -38,7 +38,7 @@ namespace util inline unsigned loadRestrictionsFromFile(storage::io::FileReader &file_reader, std::vector &restriction_list) { - unsigned number_of_usable_restrictions = file_reader.ReadElementCount32(); + auto number_of_usable_restrictions = file_reader.ReadElementCount64(); restriction_list.resize(number_of_usable_restrictions); if (number_of_usable_restrictions > 0) { @@ -61,7 +61,7 @@ NodeID loadNodesFromFile(storage::io::FileReader &file_reader, std::vector &coordinates, util::PackedVector &osm_node_ids) { - NodeID number_of_nodes = file_reader.ReadElementCount32(); + auto number_of_nodes = file_reader.ReadElementCount64(); Log() << "Importing number_of_nodes new = " << number_of_nodes << " nodes "; coordinates.resize(number_of_nodes); @@ -98,8 +98,7 @@ NodeID loadNodesFromFile(storage::io::FileReader &file_reader, inline NodeID loadEdgesFromFile(storage::io::FileReader &file_reader, std::vector &edge_list) { - EdgeID number_of_edges = file_reader.ReadElementCount32(); - BOOST_ASSERT(sizeof(EdgeID) == sizeof(number_of_edges)); + auto number_of_edges = file_reader.ReadElementCount64(); edge_list.resize(number_of_edges); Log() << " and " << number_of_edges << " edges "; diff --git a/include/util/range_table.hpp b/include/util/range_table.hpp index 318d6c86f9d..ef5305c0567 100644 --- a/include/util/range_table.hpp +++ b/include/util/range_table.hpp @@ -143,9 +143,9 @@ template class RangeTable void Write(storage::io::FileWriter &filewriter) { - unsigned number_of_blocks = diff_blocks.size(); + auto number_of_blocks = diff_blocks.size(); - filewriter.WriteElementCount32(number_of_blocks); + filewriter.WriteElementCount64(number_of_blocks); filewriter.WriteOne(sum_lengths); @@ -155,7 +155,7 @@ template class RangeTable void Read(storage::io::FileReader &filereader) { - unsigned number_of_blocks = filereader.ReadElementCount32(); + auto number_of_blocks = filereader.ReadElementCount64(); // read total length filereader.ReadInto(&sum_lengths, 1); diff --git a/include/util/static_rtree.hpp b/include/util/static_rtree.hpp index 5d69a7ed3c8..8dd8bec65d1 100644 --- a/include/util/static_rtree.hpp +++ b/include/util/static_rtree.hpp @@ -334,7 +334,7 @@ class StaticRTree // open tree file storage::io::FileWriter tree_node_file(tree_node_filename, - storage::io::FileWriter::HasNoFingerprint); + storage::io::FileWriter::GenerateFingerprint); std::uint64_t size_of_tree = m_search_tree.size(); BOOST_ASSERT_MSG(0 < size_of_tree, "tree empty"); @@ -351,7 +351,7 @@ class StaticRTree : m_coordinate_list(coordinate_list) { storage::io::FileReader tree_node_file(node_file, - storage::io::FileReader::HasNoFingerprint); + storage::io::FileReader::VerifyFingerprint); const auto tree_size = tree_node_file.ReadElementCount64(); diff --git a/src/benchmarks/static_rtree.cpp b/src/benchmarks/static_rtree.cpp index a19f1283461..1e421bf9ef8 100644 --- a/src/benchmarks/static_rtree.cpp +++ b/src/benchmarks/static_rtree.cpp @@ -32,10 +32,10 @@ using BenchStaticRTree = util::StaticRTree loadCoordinates(const boost::filesystem::path &nodes_file) { storage::io::FileReader nodes_path_file_reader(nodes_file, - storage::io::FileReader::HasNoFingerprint); + storage::io::FileReader::VerifyFingerprint); extractor::QueryNode current_node; - unsigned coordinate_count = nodes_path_file_reader.ReadElementCount32(); + auto coordinate_count = nodes_path_file_reader.ReadElementCount64(); std::vector coords(coordinate_count); for (unsigned i = 0; i < coordinate_count; ++i) { diff --git a/src/contractor/contractor.cpp b/src/contractor/contractor.cpp index 3d9f41a860f..9d4fb22151a 100644 --- a/src/contractor/contractor.cpp +++ b/src/contractor/contractor.cpp @@ -67,7 +67,7 @@ int Contractor::Run() std::vector node_levels; if (config.use_cached_priority) { - ReadNodeLevels(node_levels); + files::readLevels(config.level_output_path, node_levels); } util::DeallocatingVector contracted_edge_list; @@ -89,7 +89,9 @@ int Contractor::Run() WriteCoreNodeMarker(std::move(is_core_node)); if (!config.use_cached_priority) { - WriteNodeLevels(std::move(node_levels)); + std::vector out_node_levels(std::move(node_levels)); + + files::writeLevels(config.level_output_path, node_levels); } TIMER_STOP(preparing); @@ -101,26 +103,6 @@ int Contractor::Run() return 0; } -void Contractor::ReadNodeLevels(std::vector &node_levels) const -{ - storage::io::FileReader order_file(config.level_output_path, - storage::io::FileReader::HasNoFingerprint); - - const auto level_size = order_file.ReadElementCount32(); - node_levels.resize(level_size); - order_file.ReadInto(node_levels); -} - -void Contractor::WriteNodeLevels(std::vector &&in_node_levels) const -{ - std::vector node_levels(std::move(in_node_levels)); - - storage::io::FileWriter writer(config.level_output_path, - storage::io::FileWriter::HasNoFingerprint); - - storage::serialization::write(writer, node_levels); -} - void Contractor::WriteCoreNodeMarker(std::vector &&in_is_core_node) const { std::vector is_core_node(std::move(in_is_core_node)); @@ -131,10 +113,10 @@ void Contractor::WriteCoreNodeMarker(std::vector &&in_is_core_node) const } storage::io::FileWriter core_marker_output_file(config.core_output_path, - storage::io::FileWriter::HasNoFingerprint); + storage::io::FileWriter::GenerateFingerprint); const std::size_t count = unpacked_bool_flags.size(); - core_marker_output_file.WriteElementCount32(count); + core_marker_output_file.WriteElementCount64(count); core_marker_output_file.WriteFrom(unpacked_bool_flags.data(), count); } diff --git a/src/extractor/edge_based_graph_factory.cpp b/src/extractor/edge_based_graph_factory.cpp index b123bfadf5b..1907ac92443 100644 --- a/src/extractor/edge_based_graph_factory.cpp +++ b/src/extractor/edge_based_graph_factory.cpp @@ -560,13 +560,13 @@ void EdgeBasedGraphFactory::GenerateEdgeExpandedEdges( BOOST_ASSERT(turn_weight_penalties.size() == turn_duration_penalties.size()); { storage::io::FileWriter writer(turn_weight_penalties_filename, - storage::io::FileWriter::HasNoFingerprint); + storage::io::FileWriter::GenerateFingerprint); storage::serialization::write(writer, turn_weight_penalties); } { storage::io::FileWriter writer(turn_duration_penalties_filename, - storage::io::FileWriter::HasNoFingerprint); + storage::io::FileWriter::GenerateFingerprint); storage::serialization::write(writer, turn_duration_penalties); } @@ -576,7 +576,7 @@ void EdgeBasedGraphFactory::GenerateEdgeExpandedEdges( util::Log() << "Writing Turn Lane Data to File..."; { storage::io::FileWriter writer(turn_lane_data_filename, - storage::io::FileWriter::HasNoFingerprint); + storage::io::FileWriter::GenerateFingerprint); std::vector lane_data(lane_data_map.size()); // extract lane data sorted by ID diff --git a/src/extractor/extraction_containers.cpp b/src/extractor/extraction_containers.cpp index f94867471f2..52ac1629e4e 100644 --- a/src/extractor/extraction_containers.cpp +++ b/src/extractor/extraction_containers.cpp @@ -175,6 +175,10 @@ void ExtractionContainers::WriteCharData(const std::string &file_name) TIMER_START(write_index); boost::filesystem::ofstream file(file_name, std::ios::binary); + // TODO wrap this in a FileWriter + const auto fingerprint = util::FingerPrint::GetValid(); + file.write(reinterpret_cast(&fingerprint), sizeof(util::FingerPrint)); + const util::NameTable::IndexedData indexed_data; indexed_data.write(file, name_offsets.begin(), name_offsets.end(), name_char_data.begin()); @@ -255,7 +259,7 @@ void ExtractionContainers::PrepareNodes() "supports 2^32 unique nodes, but there were " + std::to_string(internal_id) + SOURCE_REF); } - max_internal_node_id = boost::numeric_cast(internal_id); + max_internal_node_id = boost::numeric_cast(internal_id); TIMER_STOP(id_map); log << "ok, after " << TIMER_SEC(id_map) << "s"; } @@ -571,7 +575,7 @@ void ExtractionContainers::WriteEdges(storage::io::FileWriter &file_out) const throw util::exception("There are too many edges, OSRM only supports 2^32" + SOURCE_REF); } - file_out.WriteElementCount32(normal_edges.size()); + file_out.WriteElementCount64(normal_edges.size()); file_out.WriteFrom(normal_edges.data(), normal_edges.size()); TIMER_STOP(write_edges); @@ -586,7 +590,7 @@ void ExtractionContainers::WriteNodes(storage::io::FileWriter &file_out) const // write dummy value, will be overwritten later util::UnbufferedLog log; log << "setting number of nodes ... " << std::flush; - file_out.WriteElementCount32(max_internal_node_id); + file_out.WriteElementCount64(max_internal_node_id); log << "ok"; } @@ -629,11 +633,11 @@ void ExtractionContainers::WriteNodes(storage::io::FileWriter &file_out) const void ExtractionContainers::WriteRestrictions(const std::string &path) const { // serialize restrictions - unsigned written_restriction_count = 0; + std::uint64_t written_restriction_count = 0; storage::io::FileWriter restrictions_out_file(path, storage::io::FileWriter::GenerateFingerprint); - restrictions_out_file.WriteElementCount32(written_restriction_count); + restrictions_out_file.WriteElementCount64(written_restriction_count); for (const auto &restriction_container : restrictions_list) { @@ -646,7 +650,7 @@ void ExtractionContainers::WriteRestrictions(const std::string &path) const } } restrictions_out_file.SkipToBeginning(); - restrictions_out_file.WriteElementCount32(written_restriction_count); + restrictions_out_file.WriteElementCount64(written_restriction_count); util::Log() << "usable restrictions: " << written_restriction_count; } diff --git a/src/extractor/extractor.cpp b/src/extractor/extractor.cpp index e1074413df7..7a0972395f2 100644 --- a/src/extractor/extractor.cpp +++ b/src/extractor/extractor.cpp @@ -168,7 +168,7 @@ int Extractor::run(ScriptingEnvironment &scripting_environment) util::Log() << "timestamp: " << timestamp; storage::io::FileWriter timestamp_file(config.timestamp_file_name, - storage::io::FileWriter::HasNoFingerprint); + storage::io::FileWriter::GenerateFingerprint); timestamp_file.WriteFrom(timestamp.c_str(), timestamp.length()); @@ -320,7 +320,7 @@ int Extractor::run(ScriptingEnvironment &scripting_environment) void Extractor::WriteProfileProperties(const std::string &output_path, const ProfileProperties &properties) const { - storage::io::FileWriter file(output_path, storage::io::FileWriter::HasNoFingerprint); + storage::io::FileWriter file(output_path, storage::io::FileWriter::GenerateFingerprint); file.WriteOne(properties); } @@ -658,9 +658,7 @@ void Extractor::WriteTurnLaneData(const std::string &turn_lane_file) const util::Log() << "Writing turn lane masks..."; TIMER_START(turn_lane_timer); - storage::io::FileWriter writer(turn_lane_file, storage::io::FileWriter::HasNoFingerprint); - storage::serialization::write(writer, turn_lane_offsets); - storage::serialization::write(writer, turn_lane_masks); + files::writeTurnLaneDescriptions(turn_lane_file, turn_lane_offsets, turn_lane_masks); TIMER_STOP(turn_lane_timer); util::Log() << "done (" << TIMER_SEC(turn_lane_timer) << ")"; diff --git a/src/storage/storage.cpp b/src/storage/storage.cpp index 1c9aae10fc6..13e72983f04 100644 --- a/src/storage/storage.cpp +++ b/src/storage/storage.cpp @@ -213,12 +213,12 @@ void Storage::PopulateLayout(DataLayout &layout) { util::Log() << "load names from: " << config.names_data_path; // number of entries in name index - io::FileReader name_file(config.names_data_path, io::FileReader::HasNoFingerprint); + io::FileReader name_file(config.names_data_path, io::FileReader::VerifyFingerprint); layout.SetBlockSize(DataLayout::NAME_CHAR_DATA, name_file.GetSize()); } { - io::FileReader reader(config.turn_lane_description_path, io::FileReader::HasNoFingerprint); + io::FileReader reader(config.turn_lane_description_path, io::FileReader::VerifyFingerprint); auto num_offsets = reader.ReadVectorSize(); auto num_masks = reader.ReadVectorSize(); @@ -229,7 +229,7 @@ void Storage::PopulateLayout(DataLayout &layout) // Loading information for original edges { - io::FileReader edges_file(config.edges_data_path, io::FileReader::HasNoFingerprint); + io::FileReader edges_file(config.edges_data_path, io::FileReader::VerifyFingerprint); const auto number_of_original_edges = edges_file.ReadElementCount64(); // note: settings this all to the same size is correct, we extract them from the same struct @@ -272,7 +272,7 @@ void Storage::PopulateLayout(DataLayout &layout) // load rsearch tree size { - io::FileReader tree_node_file(config.ram_index_path, io::FileReader::HasNoFingerprint); + io::FileReader tree_node_file(config.ram_index_path, io::FileReader::VerifyFingerprint); const auto tree_size = tree_node_file.ReadElementCount64(); layout.SetBlockSize(DataLayout::R_SEARCH_TREE, tree_size); @@ -284,16 +284,16 @@ void Storage::PopulateLayout(DataLayout &layout) // read timestampsize { - io::FileReader timestamp_file(config.timestamp_path, io::FileReader::HasNoFingerprint); - const auto timestamp_size = timestamp_file.Size(); + io::FileReader timestamp_file(config.timestamp_path, io::FileReader::VerifyFingerprint); + const auto timestamp_size = timestamp_file.GetSize(); layout.SetBlockSize(DataLayout::TIMESTAMP, timestamp_size); } // load core marker size if (boost::filesystem::exists(config.core_data_path)) { - io::FileReader core_marker_file(config.core_data_path, io::FileReader::HasNoFingerprint); - const auto number_of_core_markers = core_marker_file.ReadElementCount32(); + io::FileReader core_marker_file(config.core_data_path, io::FileReader::VerifyFingerprint); + const auto number_of_core_markers = core_marker_file.ReadElementCount64(); layout.SetBlockSize(DataLayout::CH_CORE_MARKER, number_of_core_markers); } else @@ -304,7 +304,7 @@ void Storage::PopulateLayout(DataLayout &layout) // load turn weight penalties { io::FileReader turn_weight_penalties_file(config.turn_weight_penalties_path, - io::FileReader::HasNoFingerprint); + io::FileReader::VerifyFingerprint); const auto number_of_penalties = turn_weight_penalties_file.ReadElementCount64(); layout.SetBlockSize(DataLayout::TURN_WEIGHT_PENALTIES, number_of_penalties); } @@ -312,7 +312,7 @@ void Storage::PopulateLayout(DataLayout &layout) // load turn duration penalties { io::FileReader turn_duration_penalties_file(config.turn_duration_penalties_path, - io::FileReader::HasNoFingerprint); + io::FileReader::VerifyFingerprint); const auto number_of_penalties = turn_duration_penalties_file.ReadElementCount64(); layout.SetBlockSize(DataLayout::TURN_DURATION_PENALTIES, number_of_penalties); } @@ -331,7 +331,7 @@ void Storage::PopulateLayout(DataLayout &layout) // load geometries sizes { - io::FileReader reader(config.geometries_path, io::FileReader::HasNoFingerprint); + io::FileReader reader(config.geometries_path, io::FileReader::VerifyFingerprint); const auto number_of_geometries_indices = reader.ReadVectorSize(); layout.SetBlockSize(DataLayout::GEOMETRIES_INDEX, number_of_geometries_indices); @@ -366,7 +366,7 @@ void Storage::PopulateLayout(DataLayout &layout) layout.SetBlockSize(DataLayout::BEARING_CLASSID, bearing_class_id_table.size()); - const auto bearing_blocks = intersection_file.ReadElementCount32(); + const auto bearing_blocks = intersection_file.ReadElementCount64(); intersection_file.Skip(1); // sum_lengths layout.SetBlockSize(DataLayout::BEARING_OFFSETS, bearing_blocks); @@ -394,7 +394,8 @@ void Storage::PopulateLayout(DataLayout &layout) { // Loading turn lane data - io::FileReader lane_data_file(config.turn_lane_data_path, io::FileReader::HasNoFingerprint); + io::FileReader lane_data_file(config.turn_lane_data_path, + io::FileReader::VerifyFingerprint); const auto lane_tuple_count = lane_data_file.ReadElementCount64(); layout.SetBlockSize(DataLayout::TURN_LANE_DATA, lane_tuple_count); @@ -529,7 +530,7 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr) // Name data { - io::FileReader name_file(config.names_data_path, io::FileReader::HasNoFingerprint); + io::FileReader name_file(config.names_data_path, io::FileReader::VerifyFingerprint); std::size_t name_file_size = name_file.GetSize(); BOOST_ASSERT(name_file_size == layout.GetBlockSize(DataLayout::NAME_CHAR_DATA)); @@ -541,7 +542,8 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr) // Turn lane data { - io::FileReader lane_data_file(config.turn_lane_data_path, io::FileReader::HasNoFingerprint); + io::FileReader lane_data_file(config.turn_lane_data_path, + io::FileReader::VerifyFingerprint); const auto lane_tuple_count = lane_data_file.ReadElementCount64(); @@ -701,7 +703,7 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr) // load turn weight penalties { io::FileReader turn_weight_penalties_file(config.turn_weight_penalties_path, - io::FileReader::HasNoFingerprint); + io::FileReader::VerifyFingerprint); const auto number_of_penalties = turn_weight_penalties_file.ReadElementCount64(); const auto turn_weight_penalties_ptr = layout.GetBlockPtr(memory_ptr, DataLayout::TURN_WEIGHT_PENALTIES); @@ -711,7 +713,7 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr) // load turn duration penalties { io::FileReader turn_duration_penalties_file(config.turn_duration_penalties_path, - io::FileReader::HasNoFingerprint); + io::FileReader::VerifyFingerprint); const auto number_of_penalties = turn_duration_penalties_file.ReadElementCount64(); const auto turn_duration_penalties_ptr = layout.GetBlockPtr(memory_ptr, DataLayout::TURN_DURATION_PENALTIES); @@ -720,8 +722,8 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr) // store timestamp { - io::FileReader timestamp_file(config.timestamp_path, io::FileReader::HasNoFingerprint); - const auto timestamp_size = timestamp_file.Size(); + io::FileReader timestamp_file(config.timestamp_path, io::FileReader::VerifyFingerprint); + const auto timestamp_size = timestamp_file.GetSize(); const auto timestamp_ptr = layout.GetBlockPtr(memory_ptr, DataLayout::TIMESTAMP); @@ -731,7 +733,7 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr) // store search tree portion of rtree { - io::FileReader tree_node_file(config.ram_index_path, io::FileReader::HasNoFingerprint); + io::FileReader tree_node_file(config.ram_index_path, io::FileReader::VerifyFingerprint); // perform this read so that we're at the right stream position for the next // read. tree_node_file.Skip(1); @@ -743,8 +745,8 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr) if (boost::filesystem::exists(config.core_data_path)) { - io::FileReader core_marker_file(config.core_data_path, io::FileReader::HasNoFingerprint); - const auto number_of_core_markers = core_marker_file.ReadElementCount32(); + io::FileReader core_marker_file(config.core_data_path, io::FileReader::VerifyFingerprint); + const auto number_of_core_markers = core_marker_file.ReadElementCount64(); // load core markers std::vector unpacked_core_markers(number_of_core_markers); @@ -778,7 +780,7 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr) // load profile properties { io::FileReader profile_properties_file(config.properties_path, - io::FileReader::HasNoFingerprint); + io::FileReader::VerifyFingerprint); const auto profile_properties_ptr = layout.GetBlockPtr( memory_ptr, DataLayout::PROPERTIES); profile_properties_file.ReadInto(profile_properties_ptr, @@ -793,7 +795,7 @@ void Storage::PopulateData(const DataLayout &layout, char *memory_ptr) std::vector bearing_class_id_table; serialization::read(intersection_file, bearing_class_id_table); - const auto bearing_blocks = intersection_file.ReadElementCount32(); + const auto bearing_blocks = intersection_file.ReadElementCount64(); intersection_file.Skip(1); // sum_lengths std::vector bearing_offsets_data(bearing_blocks); diff --git a/src/updater/updater.cpp b/src/updater/updater.cpp index bbdc8521374..fc64c9be989 100644 --- a/src/updater/updater.cpp +++ b/src/updater/updater.cpp @@ -123,6 +123,7 @@ auto mmapFile(const std::string &filename, boost::interprocess::mode_t mode) try { const file_mapping mapping{filename.c_str(), mode}; + mapped_region region{mapping, mode}; region.advise(mapped_region::advice_sequential); return region; @@ -459,20 +460,20 @@ Updater::LoadAndUpdateEdgeExpandedGraph(std::vector &e const auto load_turn_weight_penalties = [&] { using storage::io::FileReader; - FileReader reader(config.turn_weight_penalties_path, FileReader::HasNoFingerprint); + FileReader reader(config.turn_weight_penalties_path, FileReader::VerifyFingerprint); storage::serialization::read(reader, turn_weight_penalties); }; const auto load_turn_duration_penalties = [&] { using storage::io::FileReader; - FileReader reader(config.turn_duration_penalties_path, FileReader::HasNoFingerprint); + FileReader reader(config.turn_duration_penalties_path, FileReader::VerifyFingerprint); storage::serialization::read(reader, turn_duration_penalties); }; const auto load_profile_properties = [&] { // Propagate profile properties to contractor configuration structure storage::io::FileReader profile_properties_file( - config.profile_properties_path, storage::io::FileReader::HasNoFingerprint); + config.profile_properties_path, storage::io::FileReader::VerifyFingerprint); profile_properties = profile_properties_file.ReadOne(); }; @@ -647,7 +648,7 @@ Updater::LoadAndUpdateEdgeExpandedGraph(std::vector &e if (update_turn_penalties) { const auto save_penalties = [](const auto &filename, const auto &data) -> void { - storage::io::FileWriter writer(filename, storage::io::FileWriter::HasNoFingerprint); + storage::io::FileWriter writer(filename, storage::io::FileWriter::GenerateFingerprint); storage::serialization::write(writer, data); }; diff --git a/src/util/name_table.cpp b/src/util/name_table.cpp index 6d080c2d70a..47101c2287f 100644 --- a/src/util/name_table.cpp +++ b/src/util/name_table.cpp @@ -11,7 +11,7 @@ NameTable::NameTable(const std::string &file_name) { using FileReader = storage::io::FileReader; - FileReader name_stream_file_reader(file_name, FileReader::HasNoFingerprint); + FileReader name_stream_file_reader(file_name, FileReader::VerifyFingerprint); const auto file_size = name_stream_file_reader.GetSize(); m_buffer = BufferType(static_cast(::operator new(file_size)),