From 7345dc686108aa59dff534a8d3023b324ef01868 Mon Sep 17 00:00:00 2001 From: Patrick Niklaus Date: Wed, 24 Jun 2015 01:55:30 +0200 Subject: [PATCH 001/122] Move graph compression code outside of EBGF --- contractor/edge_based_graph_factory.cpp | 233 ++---------------------- contractor/edge_based_graph_factory.hpp | 37 +--- contractor/graph_compressor.cpp | 187 +++++++++++++++++++ contractor/graph_compressor.hpp | 62 +++++++ contractor/processing_chain.cpp | 56 +++--- contractor/processing_chain.hpp | 12 +- contractor/speed_profile.hpp | 16 ++ data_structures/restriction_map.hpp | 2 +- 8 files changed, 335 insertions(+), 270 deletions(-) create mode 100644 contractor/graph_compressor.cpp create mode 100644 contractor/graph_compressor.hpp create mode 100644 contractor/speed_profile.hpp diff --git a/contractor/edge_based_graph_factory.cpp b/contractor/edge_based_graph_factory.cpp index dbd73a73c58..f4fb4467bb7 100644 --- a/contractor/edge_based_graph_factory.cpp +++ b/contractor/edge_based_graph_factory.cpp @@ -41,20 +41,20 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include EdgeBasedGraphFactory::EdgeBasedGraphFactory(std::shared_ptr node_based_graph, - std::shared_ptr restriction_map, - std::unique_ptr> barrier_node_list, - std::unique_ptr> traffic_light_node_list, - const std::vector &node_info_list, - const SpeedProfileProperties &speed_profile) - : speed_profile(speed_profile), - m_number_of_edge_based_nodes(std::numeric_limits::max()), - m_node_info_list(node_info_list), - m_node_based_graph(std::move(node_based_graph)), - m_restriction_map(std::move(restriction_map)), max_id(0), removed_node_count(0) + const GeometryCompressor& geometry_compressor, + const std::unordered_set& barrier_nodes, + const std::unordered_set& traffic_lights, + std::shared_ptr restriction_map, + const std::vector &node_info_list, + const SpeedProfileProperties &speed_profile) + : m_node_info_list(node_info_list), + m_node_based_graph(node_based_graph), + m_restriction_map(restriction_map), + m_barrier_nodes(barrier_nodes), + m_traffic_lights(traffic_lights), + m_geometry_compressor(geometry_compressor), + speed_profile(speed_profile) { - // insert into unordered sets for fast lookup - m_barrier_nodes.insert(barrier_node_list->begin(), barrier_node_list->end()); - m_traffic_lights.insert(traffic_light_node_list->begin(), traffic_light_node_list->end()); } void EdgeBasedGraphFactory::GetEdgeBasedEdges(DeallocatingVector &output_edge_list) @@ -146,15 +146,6 @@ void EdgeBasedGraphFactory::InsertEdgeBasedNode(const NodeID node_u, NodeID current_edge_source_coordinate_id = node_u; - if (SPECIAL_NODEID != forward_data.edgeBasedNodeID) - { - max_id = std::max(forward_data.edgeBasedNodeID, max_id); - } - if (SPECIAL_NODEID != reverse_data.edgeBasedNodeID) - { - max_id = std::max(reverse_data.edgeBasedNodeID, max_id); - } - // traverse arrays from start and end respectively for (const auto i : osrm::irange(0u, geometry_size)) { @@ -231,13 +222,8 @@ void EdgeBasedGraphFactory::FlushVectorToStream( } void EdgeBasedGraphFactory::Run(const std::string &original_edge_data_filename, - const std::string &geometry_filename, lua_State *lua_state) { - TIMER_START(geometry); - CompressGeometry(); - TIMER_STOP(geometry); - TIMER_START(renumber); RenumberEdges(); TIMER_STOP(renumber); @@ -250,189 +236,12 @@ void EdgeBasedGraphFactory::Run(const std::string &original_edge_data_filename, GenerateEdgeExpandedEdges(original_edge_data_filename, lua_state); TIMER_STOP(generate_edges); - m_geometry_compressor.SerializeInternalVector(geometry_filename); - SimpleLogger().Write() << "Timing statistics for edge-expanded graph:"; - SimpleLogger().Write() << "Geometry compression: " << TIMER_SEC(geometry) << "s"; SimpleLogger().Write() << "Renumbering edges: " << TIMER_SEC(renumber) << "s"; SimpleLogger().Write() << "Generating nodes: " << TIMER_SEC(generate_nodes) << "s"; SimpleLogger().Write() << "Generating edges: " << TIMER_SEC(generate_edges) << "s"; } -void EdgeBasedGraphFactory::CompressGeometry() -{ - SimpleLogger().Write() << "Removing graph geometry while preserving topology"; - - const unsigned original_number_of_nodes = m_node_based_graph->GetNumberOfNodes(); - const unsigned original_number_of_edges = m_node_based_graph->GetNumberOfEdges(); - - Percent progress(original_number_of_nodes); - - for (const NodeID node_v : osrm::irange(0u, original_number_of_nodes)) - { - progress.printStatus(node_v); - - // only contract degree 2 vertices - if (2 != m_node_based_graph->GetOutDegree(node_v)) - { - continue; - } - - // don't contract barrier node - if (m_barrier_nodes.end() != m_barrier_nodes.find(node_v)) - { - continue; - } - - // check if v is a via node for a turn restriction, i.e. a 'directed' barrier node - if (m_restriction_map->IsViaNode(node_v)) - { - continue; - } - - /* - * reverse_e2 forward_e2 - * u <---------- v -----------> w - * ----------> <----------- - * forward_e1 reverse_e1 - * - * Will be compressed to: - * - * reverse_e1 - * u <---------- w - * ----------> - * forward_e1 - * - * If the edges are compatible. - * - */ - - const bool reverse_edge_order = - !(m_node_based_graph->GetEdgeData(m_node_based_graph->BeginEdges(node_v)).forward); - const EdgeID forward_e2 = m_node_based_graph->BeginEdges(node_v) + reverse_edge_order; - BOOST_ASSERT(SPECIAL_EDGEID != forward_e2); - BOOST_ASSERT(forward_e2 >= m_node_based_graph->BeginEdges(node_v) && - forward_e2 < m_node_based_graph->EndEdges(node_v)); - const EdgeID reverse_e2 = m_node_based_graph->BeginEdges(node_v) + 1 - reverse_edge_order; - BOOST_ASSERT(SPECIAL_EDGEID != reverse_e2); - BOOST_ASSERT(reverse_e2 >= m_node_based_graph->BeginEdges(node_v) && - reverse_e2 < m_node_based_graph->EndEdges(node_v)); - - const EdgeData &fwd_edge_data2 = m_node_based_graph->GetEdgeData(forward_e2); - const EdgeData &rev_edge_data2 = m_node_based_graph->GetEdgeData(reverse_e2); - - const NodeID node_w = m_node_based_graph->GetTarget(forward_e2); - BOOST_ASSERT(SPECIAL_NODEID != node_w); - BOOST_ASSERT(node_v != node_w); - const NodeID node_u = m_node_based_graph->GetTarget(reverse_e2); - BOOST_ASSERT(SPECIAL_NODEID != node_u); - BOOST_ASSERT(node_u != node_v); - - const EdgeID forward_e1 = m_node_based_graph->FindEdge(node_u, node_v); - BOOST_ASSERT(SPECIAL_EDGEID != forward_e1); - BOOST_ASSERT(node_v == m_node_based_graph->GetTarget(forward_e1)); - const EdgeID reverse_e1 = m_node_based_graph->FindEdge(node_w, node_v); - BOOST_ASSERT(SPECIAL_EDGEID != reverse_e1); - BOOST_ASSERT(node_v == m_node_based_graph->GetTarget(reverse_e1)); - - const EdgeData &fwd_edge_data1 = m_node_based_graph->GetEdgeData(forward_e1); - const EdgeData &rev_edge_data1 = m_node_based_graph->GetEdgeData(reverse_e1); - - if (m_node_based_graph->FindEdgeInEitherDirection(node_u, node_w) != SPECIAL_EDGEID) - { - continue; - } - - // this case can happen if two ways with different names overlap - if (fwd_edge_data1.nameID != rev_edge_data1.nameID || - fwd_edge_data2.nameID != rev_edge_data2.nameID) - { - continue; - } - - if (fwd_edge_data1.IsCompatibleTo(fwd_edge_data2) && rev_edge_data1.IsCompatibleTo(rev_edge_data2)) - { - BOOST_ASSERT(m_node_based_graph->GetEdgeData(forward_e1).nameID == - m_node_based_graph->GetEdgeData(reverse_e1).nameID); - BOOST_ASSERT(m_node_based_graph->GetEdgeData(forward_e2).nameID == - m_node_based_graph->GetEdgeData(reverse_e2).nameID); - - // Get distances before graph is modified - const int forward_weight1 = m_node_based_graph->GetEdgeData(forward_e1).distance; - const int forward_weight2 = m_node_based_graph->GetEdgeData(forward_e2).distance; - - BOOST_ASSERT(0 != forward_weight1); - BOOST_ASSERT(0 != forward_weight2); - - const int reverse_weight1 = m_node_based_graph->GetEdgeData(reverse_e1).distance; - const int reverse_weight2 = m_node_based_graph->GetEdgeData(reverse_e2).distance; - - BOOST_ASSERT(0 != reverse_weight1); - BOOST_ASSERT(0 != reverse_weight2); - - const bool has_node_penalty = m_traffic_lights.find(node_v) != m_traffic_lights.end(); - - // add weight of e2's to e1 - m_node_based_graph->GetEdgeData(forward_e1).distance += fwd_edge_data2.distance; - m_node_based_graph->GetEdgeData(reverse_e1).distance += rev_edge_data2.distance; - if (has_node_penalty) - { - m_node_based_graph->GetEdgeData(forward_e1).distance += - speed_profile.traffic_signal_penalty; - m_node_based_graph->GetEdgeData(reverse_e1).distance += - speed_profile.traffic_signal_penalty; - } - - // extend e1's to targets of e2's - m_node_based_graph->SetTarget(forward_e1, node_w); - m_node_based_graph->SetTarget(reverse_e1, node_u); - - // remove e2's (if bidir, otherwise only one) - m_node_based_graph->DeleteEdge(node_v, forward_e2); - m_node_based_graph->DeleteEdge(node_v, reverse_e2); - - // update any involved turn restrictions - m_restriction_map->FixupStartingTurnRestriction(node_u, node_v, node_w); - m_restriction_map->FixupArrivingTurnRestriction(node_u, node_v, node_w, - *m_node_based_graph); - - m_restriction_map->FixupStartingTurnRestriction(node_w, node_v, node_u); - m_restriction_map->FixupArrivingTurnRestriction(node_w, node_v, node_u, - *m_node_based_graph); - - // store compressed geometry in container - m_geometry_compressor.CompressEdge( - forward_e1, forward_e2, node_v, node_w, - forward_weight1 + (has_node_penalty ? speed_profile.traffic_signal_penalty : 0), - forward_weight2); - m_geometry_compressor.CompressEdge( - reverse_e1, reverse_e2, node_v, node_u, reverse_weight1, - reverse_weight2 + (has_node_penalty ? speed_profile.traffic_signal_penalty : 0)); - ++removed_node_count; - - - } - } - SimpleLogger().Write() << "removed " << removed_node_count << " nodes"; - m_geometry_compressor.PrintStatistics(); - - unsigned new_node_count = 0; - unsigned new_edge_count = 0; - - for (const auto i : osrm::irange(0u, m_node_based_graph->GetNumberOfNodes())) - { - if (m_node_based_graph->GetOutDegree(i) > 0) - { - ++new_node_count; - new_edge_count += (m_node_based_graph->EndEdges(i) - m_node_based_graph->BeginEdges(i)); - } - } - SimpleLogger().Write() << "new nodes: " << new_node_count << ", edges " << new_edge_count; - SimpleLogger().Write() << "Node compression ratio: " - << new_node_count / (double)original_number_of_nodes; - SimpleLogger().Write() << "Edge compression ratio: " - << new_edge_count / (double)original_number_of_edges; -} /// Renumbers all _forward_ edges and sets the edgeBasedNodeID. /// A specific numbering is not important. Any unique ID will do. @@ -462,9 +271,7 @@ void EdgeBasedGraphFactory::RenumberEdges() m_number_of_edge_based_nodes = numbered_edges_count; } -/** - * Creates the nodes in the edge expanded graph from edges in the node-based graph. - */ +/// Creates the nodes in the edge expanded graph from edges in the node-based graph. void EdgeBasedGraphFactory::GenerateEdgeExpandedNodes() { SimpleLogger().Write() << "Identifying components of the (compressed) road network"; @@ -476,10 +283,10 @@ void EdgeBasedGraphFactory::GenerateEdgeExpandedNodes() component_explorer.run(); SimpleLogger().Write() << "identified: " - << component_explorer.get_number_of_components() - removed_node_count + << component_explorer.get_number_of_components() << " (compressed) components"; SimpleLogger().Write() << "identified " - << component_explorer.get_size_one_count() - removed_node_count + << component_explorer.get_size_one_count() << " (compressed) SCCs of size 1"; SimpleLogger().Write() << "generating edge-expanded nodes"; @@ -543,9 +350,7 @@ void EdgeBasedGraphFactory::GenerateEdgeExpandedNodes() << " nodes in edge-expanded graph"; } -/** - * Actually it also generates OriginalEdgeData and serializes them... - */ +/// Actually it also generates OriginalEdgeData and serializes them... void EdgeBasedGraphFactory::GenerateEdgeExpandedEdges( const std::string &original_edge_data_filename, lua_State *lua_state) { @@ -790,7 +595,3 @@ TurnInstruction EdgeBasedGraphFactory::AnalyzeTurn(const NodeID node_u, return TurnInstructionsClass::GetTurnDirectionOfInstruction(angle); } -unsigned EdgeBasedGraphFactory::GetNumberOfEdgeBasedNodes() const -{ - return m_number_of_edge_based_nodes; -} diff --git a/contractor/edge_based_graph_factory.hpp b/contractor/edge_based_graph_factory.hpp index a5274396f06..8e0ea260893 100644 --- a/contractor/edge_based_graph_factory.hpp +++ b/contractor/edge_based_graph_factory.hpp @@ -30,6 +30,7 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #ifndef EDGE_BASED_GRAPH_FACTORY_HPP_ #define EDGE_BASED_GRAPH_FACTORY_HPP_ +#include "speed_profile.hpp" #include "geometry_compressor.hpp" #include "../typedefs.h" #include "../data_structures/deallocating_vector.hpp" @@ -57,17 +58,16 @@ class EdgeBasedGraphFactory EdgeBasedGraphFactory() = delete; EdgeBasedGraphFactory(const EdgeBasedGraphFactory &) = delete; - struct SpeedProfileProperties; explicit EdgeBasedGraphFactory(std::shared_ptr node_based_graph, - std::shared_ptr restricion_map, - std::unique_ptr> barrier_node_list, - std::unique_ptr> traffic_light_node_list, + const GeometryCompressor& geometry_compressor, + const std::unordered_set& barrier_nodes, + const std::unordered_set& traffic_lights, + std::shared_ptr restriction_map, const std::vector &node_info_list, const SpeedProfileProperties &speed_profile); void Run(const std::string &original_edge_data_filename, - const std::string &geometry_filename, lua_State *lua_state); void GetEdgeBasedEdges(DeallocatingVector &edges); @@ -78,20 +78,6 @@ class EdgeBasedGraphFactory int GetTurnPenalty(double angle, lua_State *lua_state) const; - unsigned GetNumberOfEdgeBasedNodes() const; - - struct SpeedProfileProperties - { - SpeedProfileProperties() - : traffic_signal_penalty(0), u_turn_penalty(0), has_turn_penalty_function(false) - { - } - - int traffic_signal_penalty; - int u_turn_penalty; - bool has_turn_penalty_function; - } speed_profile; - private: using EdgeData = NodeBasedDynamicGraph::EdgeData; @@ -102,13 +88,13 @@ class EdgeBasedGraphFactory const std::vector& m_node_info_list; std::shared_ptr m_node_based_graph; - std::shared_ptr m_restriction_map; + std::shared_ptr m_restriction_map; - std::unordered_set m_barrier_nodes; - std::unordered_set m_traffic_lights; + const std::unordered_set& m_barrier_nodes; + const std::unordered_set& m_traffic_lights; + const GeometryCompressor& m_geometry_compressor; - - GeometryCompressor m_geometry_compressor; + SpeedProfileProperties speed_profile; void CompressGeometry(); void RenumberEdges(); @@ -121,9 +107,6 @@ class EdgeBasedGraphFactory void FlushVectorToStream(std::ofstream &edge_data_file, std::vector &original_edge_data_vector) const; - NodeID max_id; - std::size_t removed_node_count; - }; #endif /* EDGE_BASED_GRAPH_FACTORY_HPP_ */ diff --git a/contractor/graph_compressor.cpp b/contractor/graph_compressor.cpp new file mode 100644 index 00000000000..79fcbb937b9 --- /dev/null +++ b/contractor/graph_compressor.cpp @@ -0,0 +1,187 @@ +#include "graph_compressor.hpp" + +#include "geometry_compressor.hpp" +#include "../data_structures/dynamic_graph.hpp" +#include "../data_structures/node_based_graph.hpp" +#include "../data_structures/restriction_map.hpp" +#include "../data_structures/percent.hpp" + +GraphCompressor::GraphCompressor(const SpeedProfileProperties& speed_profile) + : speed_profile(speed_profile) +{ +} + + +void GraphCompressor::Compress(const std::unordered_set& barrier_nodes, + const std::unordered_set& traffic_lights, + RestrictionMap& restriction_map, + NodeBasedDynamicGraph& graph, + GeometryCompressor& geometry_compressor) +{ + const unsigned original_number_of_nodes = graph.GetNumberOfNodes(); + const unsigned original_number_of_edges = graph.GetNumberOfEdges(); + + Percent progress(original_number_of_nodes); + + for (const NodeID node_v : osrm::irange(0u, original_number_of_nodes)) + { + progress.printStatus(node_v); + + // only contract degree 2 vertices + if (2 != graph.GetOutDegree(node_v)) + { + continue; + } + + // don't contract barrier node + if (barrier_nodes.end() != barrier_nodes.find(node_v)) + { + continue; + } + + // check if v is a via node for a turn restriction, i.e. a 'directed' barrier node + if (restriction_map.IsViaNode(node_v)) + { + continue; + } + + // reverse_e2 forward_e2 + // u <---------- v -----------> w + // ----------> <----------- + // forward_e1 reverse_e1 + // + // Will be compressed to: + // + // reverse_e1 + // u <---------- w + // ----------> + // forward_e1 + // + // If the edges are compatible. + + const bool reverse_edge_order = + !(graph.GetEdgeData(graph.BeginEdges(node_v)).forward); + const EdgeID forward_e2 = graph.BeginEdges(node_v) + reverse_edge_order; + BOOST_ASSERT(SPECIAL_EDGEID != forward_e2); + BOOST_ASSERT(forward_e2 >= graph.BeginEdges(node_v) && + forward_e2 < graph.EndEdges(node_v)); + const EdgeID reverse_e2 = graph.BeginEdges(node_v) + 1 - reverse_edge_order; + BOOST_ASSERT(SPECIAL_EDGEID != reverse_e2); + BOOST_ASSERT(reverse_e2 >= graph.BeginEdges(node_v) && + reverse_e2 < graph.EndEdges(node_v)); + + const EdgeData &fwd_edge_data2 = graph.GetEdgeData(forward_e2); + const EdgeData &rev_edge_data2 = graph.GetEdgeData(reverse_e2); + + const NodeID node_w = graph.GetTarget(forward_e2); + BOOST_ASSERT(SPECIAL_NODEID != node_w); + BOOST_ASSERT(node_v != node_w); + const NodeID node_u = graph.GetTarget(reverse_e2); + BOOST_ASSERT(SPECIAL_NODEID != node_u); + BOOST_ASSERT(node_u != node_v); + + const EdgeID forward_e1 = graph.FindEdge(node_u, node_v); + BOOST_ASSERT(SPECIAL_EDGEID != forward_e1); + BOOST_ASSERT(node_v == graph.GetTarget(forward_e1)); + const EdgeID reverse_e1 = graph.FindEdge(node_w, node_v); + BOOST_ASSERT(SPECIAL_EDGEID != reverse_e1); + BOOST_ASSERT(node_v == graph.GetTarget(reverse_e1)); + + const EdgeData &fwd_edge_data1 = graph.GetEdgeData(forward_e1); + const EdgeData &rev_edge_data1 = graph.GetEdgeData(reverse_e1); + + if (graph.FindEdgeInEitherDirection(node_u, node_w) != SPECIAL_EDGEID) + { + continue; + } + + // this case can happen if two ways with different names overlap + if (fwd_edge_data1.nameID != rev_edge_data1.nameID || + fwd_edge_data2.nameID != rev_edge_data2.nameID) + { + continue; + } + + if (fwd_edge_data1.IsCompatibleTo(fwd_edge_data2) && rev_edge_data1.IsCompatibleTo(rev_edge_data2)) + { + BOOST_ASSERT(graph.GetEdgeData(forward_e1).nameID == + graph.GetEdgeData(reverse_e1).nameID); + BOOST_ASSERT(graph.GetEdgeData(forward_e2).nameID == + graph.GetEdgeData(reverse_e2).nameID); + + // Get distances before graph is modified + const int forward_weight1 = graph.GetEdgeData(forward_e1).distance; + const int forward_weight2 = graph.GetEdgeData(forward_e2).distance; + + BOOST_ASSERT(0 != forward_weight1); + BOOST_ASSERT(0 != forward_weight2); + + const int reverse_weight1 = graph.GetEdgeData(reverse_e1).distance; + const int reverse_weight2 = graph.GetEdgeData(reverse_e2).distance; + + BOOST_ASSERT(0 != reverse_weight1); + BOOST_ASSERT(0 != reverse_weight2); + + const bool has_node_penalty = traffic_lights.find(node_v) != traffic_lights.end(); + + // add weight of e2's to e1 + graph.GetEdgeData(forward_e1).distance += fwd_edge_data2.distance; + graph.GetEdgeData(reverse_e1).distance += rev_edge_data2.distance; + if (has_node_penalty) + { + graph.GetEdgeData(forward_e1).distance += + speed_profile.traffic_signal_penalty; + graph.GetEdgeData(reverse_e1).distance += + speed_profile.traffic_signal_penalty; + } + + // extend e1's to targets of e2's + graph.SetTarget(forward_e1, node_w); + graph.SetTarget(reverse_e1, node_u); + + // remove e2's (if bidir, otherwise only one) + graph.DeleteEdge(node_v, forward_e2); + graph.DeleteEdge(node_v, reverse_e2); + + // update any involved turn restrictions + restriction_map.FixupStartingTurnRestriction(node_u, node_v, node_w); + restriction_map.FixupArrivingTurnRestriction(node_u, node_v, node_w, graph); + + restriction_map.FixupStartingTurnRestriction(node_w, node_v, node_u); + restriction_map.FixupArrivingTurnRestriction(node_w, node_v, node_u, graph); + + // store compressed geometry in container + geometry_compressor.CompressEdge( + forward_e1, forward_e2, node_v, node_w, + forward_weight1 + (has_node_penalty ? speed_profile.traffic_signal_penalty : 0), + forward_weight2); + geometry_compressor.CompressEdge( + reverse_e1, reverse_e2, node_v, node_u, reverse_weight1, + reverse_weight2 + (has_node_penalty ? speed_profile.traffic_signal_penalty : 0)); + } + } + + PrintStatistics(original_number_of_nodes, original_number_of_edges, graph); +} + +void GraphCompressor::PrintStatistics(unsigned original_number_of_nodes, + unsigned original_number_of_edges, + const NodeBasedDynamicGraph& graph) const +{ + + unsigned new_node_count = 0; + unsigned new_edge_count = 0; + + for (const auto i : osrm::irange(0u, graph.GetNumberOfNodes())) + { + if (graph.GetOutDegree(i) > 0) + { + ++new_node_count; + new_edge_count += (graph.EndEdges(i) - graph.BeginEdges(i)); + } + } + SimpleLogger().Write() << "Node compression ratio: " + << new_node_count / (double)original_number_of_nodes; + SimpleLogger().Write() << "Edge compression ratio: " + << new_edge_count / (double)original_number_of_edges; +} diff --git a/contractor/graph_compressor.hpp b/contractor/graph_compressor.hpp new file mode 100644 index 00000000000..17b3c29a7a6 --- /dev/null +++ b/contractor/graph_compressor.hpp @@ -0,0 +1,62 @@ +/* + +Copyright (c) 2014, Project OSRM, Dennis Luxen, others +All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + +Redistributions of source code must retain the above copyright notice, this list +of conditions and the following disclaimer. +Redistributions in binary form must reproduce the above copyright notice, this +list of conditions and the following disclaimer in the documentation and/or +other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +*/ +#ifndef GEOMETRY_COMPRESSOR_HPP +#define GEOMETRY_COMPRESSOR_HPP + +#include "../typedefs.h" + +#include "speed_profile.hpp" +#include "../data_structures/node_based_graph.hpp" + +#include +#include + +class GeometryCompressor; +class RestrictionMap; + +class GraphCompressor +{ + using EdgeData = NodeBasedDynamicGraph::EdgeData; + +public: + GraphCompressor(const SpeedProfileProperties& speed_profile); + + void Compress(const std::unordered_set& barrier_nodes, + const std::unordered_set& traffic_lights, + RestrictionMap& restriction_map, + NodeBasedDynamicGraph& graph, + GeometryCompressor& geometry_compressor); +private: + + void PrintStatistics(unsigned original_number_of_nodes, + unsigned original_number_of_edges, + const NodeBasedDynamicGraph& graph) const; + + SpeedProfileProperties speed_profile; +}; + +#endif diff --git a/contractor/processing_chain.cpp b/contractor/processing_chain.cpp index a76bc657feb..2073818644f 100644 --- a/contractor/processing_chain.cpp +++ b/contractor/processing_chain.cpp @@ -28,6 +28,8 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "processing_chain.hpp" #include "contractor.hpp" +#include "geometry_compressor.hpp" +#include "graph_compressor.hpp" #include "../algorithms/crc32_processor.hpp" #include "../data_structures/deallocating_vector.hpp" @@ -263,7 +265,7 @@ unsigned Prepare::CalculateEdgeChecksum(std::unique_ptr Prepare::LoadRestrictionMap() \brief Load node based graph from .osrm file */ std::shared_ptr -Prepare::LoadNodeBasedGraph(std::vector &barrier_node_list, - std::vector &traffic_light_list, +Prepare::LoadNodeBasedGraph(std::unordered_set &barrier_nodes, + std::unordered_set &traffic_lights, std::vector& internal_to_external_node_map) { std::vector edge_list; boost::filesystem::ifstream input_stream(config.osrm_input_path, std::ios::in | std::ios::binary); + std::vector barrier_list; + std::vector traffic_light_list; NodeID number_of_node_based_nodes = loadNodesFromFile(input_stream, - barrier_node_list, traffic_light_list, + barrier_list, traffic_light_list, internal_to_external_node_map); - SimpleLogger().Write() << " - " << barrier_node_list.size() << " bollard nodes, " + SimpleLogger().Write() << " - " << barrier_list.size() << " bollard nodes, " << traffic_light_list.size() << " traffic lights"; + // insert into unordered sets for fast lookup + barrier_nodes.insert(barrier_list.begin(), barrier_list.end()); + traffic_lights.insert(traffic_light_list.begin(), traffic_light_list.end()); + + barrier_list.clear(); + barrier_list.shrink_to_fit(); + traffic_light_list.clear(); + traffic_light_list.shrink_to_fit(); + loadEdgesFromFile(input_stream, edge_list); if (edge_list.empty()) @@ -357,37 +370,38 @@ Prepare::BuildEdgeExpandedGraph(std::vector &internal_to_external_nod lua_State *lua_state = luaL_newstate(); luabind::open(lua_state); - EdgeBasedGraphFactory::SpeedProfileProperties speed_profile; - + SpeedProfileProperties speed_profile; SetupScriptingEnvironment(lua_state, speed_profile); - auto barrier_node_list = osrm::make_unique>(); - auto traffic_light_list = osrm::make_unique>(); + std::unordered_set barrier_nodes; + std::unordered_set traffic_lights; auto restriction_map = LoadRestrictionMap(); - auto node_based_graph = LoadNodeBasedGraph(*barrier_node_list, *traffic_light_list, internal_to_external_node_map); + auto node_based_graph = LoadNodeBasedGraph(barrier_nodes, traffic_lights, internal_to_external_node_map); - const std::size_t number_of_node_based_nodes = node_based_graph->GetNumberOfNodes(); + + GeometryCompressor geometry_compressor; + GraphCompressor graph_compressor(speed_profile); + graph_compressor.Compress(barrier_nodes, traffic_lights, *restriction_map, *node_based_graph, geometry_compressor); EdgeBasedGraphFactory edge_based_graph_factory(node_based_graph, - restriction_map, - std::move(barrier_node_list), - std::move(traffic_light_list), + geometry_compressor, + barrier_nodes, + traffic_lights, + std::const_pointer_cast(restriction_map), internal_to_external_node_map, speed_profile); - edge_based_graph_factory.Run(config.edge_output_path, config.geometry_output_path, lua_state); - lua_close(lua_state); - - const std::size_t number_of_edge_based_nodes = - edge_based_graph_factory.GetNumberOfEdgeBasedNodes(); + geometry_compressor.SerializeInternalVector(config.geometry_output_path); - BOOST_ASSERT(number_of_edge_based_nodes != std::numeric_limits::max()); + edge_based_graph_factory.Run(config.edge_output_path, lua_state); + lua_close(lua_state); edge_based_graph_factory.GetEdgeBasedEdges(edge_based_edge_list); edge_based_graph_factory.GetEdgeBasedNodes(node_based_edge_list); - return std::make_pair(number_of_node_based_nodes, number_of_edge_based_nodes); + const std::size_t number_of_node_based_nodes = node_based_graph->GetNumberOfNodes(); + return std::make_pair(number_of_node_based_nodes, node_based_edge_list.size()); } /** diff --git a/contractor/processing_chain.hpp b/contractor/processing_chain.hpp index 28176816604..a46b963ec96 100644 --- a/contractor/processing_chain.hpp +++ b/contractor/processing_chain.hpp @@ -33,6 +33,7 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "../data_structures/query_edge.hpp" #include "../data_structures/static_graph.hpp" +struct SpeedProfileProperties; struct EdgeBasedNode; struct lua_State; @@ -59,8 +60,7 @@ class Prepare protected: void SetupScriptingEnvironment(lua_State *myLuaState, - EdgeBasedGraphFactory::SpeedProfileProperties &speed_profile); - std::shared_ptr LoadRestrictionMap(); + SpeedProfileProperties &speed_profile); unsigned CalculateEdgeChecksum(std::unique_ptr> node_based_edge_list); void ContractGraph(const std::size_t number_of_edge_based_nodes, DeallocatingVector& edge_based_edge_list, @@ -68,9 +68,11 @@ class Prepare std::size_t WriteContractedGraph(unsigned number_of_edge_based_nodes, std::unique_ptr> node_based_edge_list, std::unique_ptr> contracted_edge_list); - std::shared_ptr LoadNodeBasedGraph(std::vector &barrier_node_list, - std::vector &traffic_light_list, - std::vector& internal_to_external_node_map); + std::shared_ptr LoadRestrictionMap(); + std::shared_ptr + LoadNodeBasedGraph(std::unordered_set &barrier_nodes, + std::unordered_set &traffic_lights, + std::vector& internal_to_external_node_map); std::pair BuildEdgeExpandedGraph(std::vector &internal_to_external_node_map, std::vector &node_based_edge_list, diff --git a/contractor/speed_profile.hpp b/contractor/speed_profile.hpp new file mode 100644 index 00000000000..534ccce34ac --- /dev/null +++ b/contractor/speed_profile.hpp @@ -0,0 +1,16 @@ +#ifndef SPEED_PROFILE_PROPERTIES_HPP +#define SPEED_PROFILE_PROPERTIES_HPP + +struct SpeedProfileProperties +{ + SpeedProfileProperties() + : traffic_signal_penalty(0), u_turn_penalty(0), has_turn_penalty_function(false) + { + } + + int traffic_signal_penalty; + int u_turn_penalty; + bool has_turn_penalty_function; +}; + +#endif diff --git a/data_structures/restriction_map.hpp b/data_structures/restriction_map.hpp index 98b360c2537..dbca35651ff 100644 --- a/data_structures/restriction_map.hpp +++ b/data_structures/restriction_map.hpp @@ -156,7 +156,7 @@ class RestrictionMap bool CheckIfTurnIsRestricted(const NodeID node_u, const NodeID node_v, const NodeID node_w) const; - std::size_t size() { return m_count; } + std::size_t size() const { return m_count; } private: // check of node is the start of any restriction From 3ef34fbb56836c9a7e2d0200ff72dae3663cd151 Mon Sep 17 00:00:00 2001 From: Patrick Niklaus Date: Wed, 24 Jun 2015 19:55:36 +0200 Subject: [PATCH 002/122] Rename GeometryCompressor and add unit tests --- CMakeLists.txt | 4 +- contractor/edge_based_graph_factory.cpp | 34 ++++---- contractor/edge_based_graph_factory.hpp | 6 +- contractor/graph_compressor.cpp | 4 +- contractor/graph_compressor.hpp | 4 +- contractor/processing_chain.cpp | 10 +-- .../compressed_edge_container.cpp | 30 ++++--- .../compressed_edge_container.hpp | 16 ++-- .../compressed_edge_container.cpp | 87 +++++++++++++++++++ 9 files changed, 142 insertions(+), 53 deletions(-) rename contractor/geometry_compressor.cpp => data_structures/compressed_edge_container.cpp (89%) rename contractor/geometry_compressor.hpp => data_structures/compressed_edge_container.hpp (86%) create mode 100644 unit_tests/data_structures/compressed_edge_container.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 80eb3d3fa9e..b1f627297ff 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -64,7 +64,7 @@ add_executable(osrm-extract ${ExtractorSources} $ $ $ $ $ $ $ $ $ $) @@ -76,7 +76,7 @@ file(GLOB CoordinateGlob data_structures/coordinate*.cpp) file(GLOB AlgorithmGlob algorithms/*.cpp) file(GLOB HttpGlob server/http/*.cpp) file(GLOB LibOSRMGlob library/*.cpp) -file(GLOB DataStructureTestsGlob unit_tests/data_structures/*.cpp data_structures/hilbert_value.cpp) +file(GLOB DataStructureTestsGlob unit_tests/data_structures/*.cpp data_structures/hilbert_value.cpp data_structures/compressed_edge_container.cpp) file(GLOB AlgorithmTestsGlob unit_tests/algorithms/*.cpp) set( diff --git a/contractor/edge_based_graph_factory.cpp b/contractor/edge_based_graph_factory.cpp index f4fb4467bb7..c5c31ac5b9c 100644 --- a/contractor/edge_based_graph_factory.cpp +++ b/contractor/edge_based_graph_factory.cpp @@ -41,7 +41,7 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include EdgeBasedGraphFactory::EdgeBasedGraphFactory(std::shared_ptr node_based_graph, - const GeometryCompressor& geometry_compressor, + const CompressedEdgeContainer& compressed_edge_container, const std::unordered_set& barrier_nodes, const std::unordered_set& traffic_lights, std::shared_ptr restriction_map, @@ -52,7 +52,7 @@ EdgeBasedGraphFactory::EdgeBasedGraphFactory(std::shared_ptr &forward_geometry = - m_geometry_compressor.GetBucketReference(edge_id_1); - const std::vector &reverse_geometry = - m_geometry_compressor.GetBucketReference(edge_id_2); + const auto& forward_geometry = m_compressed_edge_container.GetBucketReference(edge_id_1); + const auto& reverse_geometry = m_compressed_edge_container.GetBucketReference(edge_id_2); BOOST_ASSERT(forward_geometry.size() == reverse_geometry.size()); BOOST_ASSERT(0 != forward_geometry.size()); const unsigned geometry_size = static_cast(forward_geometry.size()); @@ -160,7 +158,7 @@ void EdgeBasedGraphFactory::InsertEdgeBasedNode(const NodeID node_u, current_edge_source_coordinate_id, current_edge_target_coordinate_id, forward_data.nameID, forward_geometry[i].second, reverse_geometry[geometry_size - 1 - i].second, forward_dist_prefix_sum[i], - reverse_dist_prefix_sum[i], m_geometry_compressor.GetPositionForID(edge_id_1), + reverse_dist_prefix_sum[i], m_compressed_edge_container.GetPositionForID(edge_id_1), component_id, i, forward_data.travel_mode, reverse_data.travel_mode); current_edge_source_coordinate_id = current_edge_target_coordinate_id; @@ -178,7 +176,7 @@ void EdgeBasedGraphFactory::InsertEdgeBasedNode(const NodeID node_u, } else { - BOOST_ASSERT(!m_geometry_compressor.HasEntryForID(edge_id_2)); + BOOST_ASSERT(!m_compressed_edge_container.HasEntryForID(edge_id_2)); if (forward_data.edgeBasedNodeID != SPECIAL_NODEID) { @@ -454,14 +452,14 @@ void EdgeBasedGraphFactory::GenerateEdgeExpandedEdges( // unpack last node of first segment if packed const auto first_coordinate = - m_node_info_list[(m_geometry_compressor.HasEntryForID(e1) - ? m_geometry_compressor.GetLastNodeIDOfBucket(e1) + m_node_info_list[(m_compressed_edge_container.HasEntryForID(e1) + ? m_compressed_edge_container.GetLastEdgeSourceID(e1) : node_u)]; // unpack first node of second segment if packed const auto third_coordinate = - m_node_info_list[(m_geometry_compressor.HasEntryForID(e2) - ? m_geometry_compressor.GetFirstNodeIDOfBucket(e2) + m_node_info_list[(m_compressed_edge_container.HasEntryForID(e2) + ? m_compressed_edge_container.GetFirstEdgeTargetID(e2) : node_w)]; const double turn_angle = ComputeAngle::OfThreeFixedPointCoordinates( @@ -475,7 +473,7 @@ void EdgeBasedGraphFactory::GenerateEdgeExpandedEdges( } distance += turn_penalty; - const bool edge_is_compressed = m_geometry_compressor.HasEntryForID(e1); + const bool edge_is_compressed = m_compressed_edge_container.HasEntryForID(e1); if (edge_is_compressed) { @@ -483,7 +481,7 @@ void EdgeBasedGraphFactory::GenerateEdgeExpandedEdges( } original_edge_data_vector.emplace_back( - (edge_is_compressed ? m_geometry_compressor.GetPositionForID(e1) : node_v), + (edge_is_compressed ? m_compressed_edge_container.GetPositionForID(e1) : node_v), edge_data1.nameID, turn_instruction, edge_is_compressed, edge_data2.travel_mode); diff --git a/contractor/edge_based_graph_factory.hpp b/contractor/edge_based_graph_factory.hpp index 8e0ea260893..20a3feda442 100644 --- a/contractor/edge_based_graph_factory.hpp +++ b/contractor/edge_based_graph_factory.hpp @@ -31,8 +31,8 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #define EDGE_BASED_GRAPH_FACTORY_HPP_ #include "speed_profile.hpp" -#include "geometry_compressor.hpp" #include "../typedefs.h" +#include "../data_structures/compressed_edge_container.hpp" #include "../data_structures/deallocating_vector.hpp" #include "../data_structures/edge_based_node.hpp" #include "../data_structures/original_edge_data.hpp" @@ -60,7 +60,7 @@ class EdgeBasedGraphFactory explicit EdgeBasedGraphFactory(std::shared_ptr node_based_graph, - const GeometryCompressor& geometry_compressor, + const CompressedEdgeContainer& compressed_edge_container, const std::unordered_set& barrier_nodes, const std::unordered_set& traffic_lights, std::shared_ptr restriction_map, @@ -92,7 +92,7 @@ class EdgeBasedGraphFactory const std::unordered_set& m_barrier_nodes; const std::unordered_set& m_traffic_lights; - const GeometryCompressor& m_geometry_compressor; + const CompressedEdgeContainer& m_compressed_edge_container; SpeedProfileProperties speed_profile; diff --git a/contractor/graph_compressor.cpp b/contractor/graph_compressor.cpp index 79fcbb937b9..e8ad384fc3f 100644 --- a/contractor/graph_compressor.cpp +++ b/contractor/graph_compressor.cpp @@ -1,6 +1,6 @@ #include "graph_compressor.hpp" -#include "geometry_compressor.hpp" +#include "../data_structures/compressed_edge_container.hpp" #include "../data_structures/dynamic_graph.hpp" #include "../data_structures/node_based_graph.hpp" #include "../data_structures/restriction_map.hpp" @@ -16,7 +16,7 @@ void GraphCompressor::Compress(const std::unordered_set& barrier_nodes, const std::unordered_set& traffic_lights, RestrictionMap& restriction_map, NodeBasedDynamicGraph& graph, - GeometryCompressor& geometry_compressor) + CompressedEdgeContainer& geometry_compressor) { const unsigned original_number_of_nodes = graph.GetNumberOfNodes(); const unsigned original_number_of_edges = graph.GetNumberOfEdges(); diff --git a/contractor/graph_compressor.hpp b/contractor/graph_compressor.hpp index 17b3c29a7a6..0e349423cb9 100644 --- a/contractor/graph_compressor.hpp +++ b/contractor/graph_compressor.hpp @@ -35,7 +35,7 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include #include -class GeometryCompressor; +class CompressedEdgeContainer; class RestrictionMap; class GraphCompressor @@ -49,7 +49,7 @@ class GraphCompressor const std::unordered_set& traffic_lights, RestrictionMap& restriction_map, NodeBasedDynamicGraph& graph, - GeometryCompressor& geometry_compressor); + CompressedEdgeContainer& geometry_compressor); private: void PrintStatistics(unsigned original_number_of_nodes, diff --git a/contractor/processing_chain.cpp b/contractor/processing_chain.cpp index 2073818644f..4a6a12c5b39 100644 --- a/contractor/processing_chain.cpp +++ b/contractor/processing_chain.cpp @@ -28,10 +28,10 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "processing_chain.hpp" #include "contractor.hpp" -#include "geometry_compressor.hpp" #include "graph_compressor.hpp" #include "../algorithms/crc32_processor.hpp" +#include "../data_structures/compressed_edge_container.hpp" #include "../data_structures/deallocating_vector.hpp" #include "../data_structures/static_rtree.hpp" #include "../data_structures/restriction_map.hpp" @@ -380,19 +380,19 @@ Prepare::BuildEdgeExpandedGraph(std::vector &internal_to_external_nod auto node_based_graph = LoadNodeBasedGraph(barrier_nodes, traffic_lights, internal_to_external_node_map); - GeometryCompressor geometry_compressor; + CompressedEdgeContainer compressed_edge_container; GraphCompressor graph_compressor(speed_profile); - graph_compressor.Compress(barrier_nodes, traffic_lights, *restriction_map, *node_based_graph, geometry_compressor); + graph_compressor.Compress(barrier_nodes, traffic_lights, *restriction_map, *node_based_graph, compressed_edge_container); EdgeBasedGraphFactory edge_based_graph_factory(node_based_graph, - geometry_compressor, + compressed_edge_container, barrier_nodes, traffic_lights, std::const_pointer_cast(restriction_map), internal_to_external_node_map, speed_profile); - geometry_compressor.SerializeInternalVector(config.geometry_output_path); + compressed_edge_container.SerializeInternalVector(config.geometry_output_path); edge_based_graph_factory.Run(config.edge_output_path, lua_state); lua_close(lua_state); diff --git a/contractor/geometry_compressor.cpp b/data_structures/compressed_edge_container.cpp similarity index 89% rename from contractor/geometry_compressor.cpp rename to data_structures/compressed_edge_container.cpp index 3997cdc99da..da916cee732 100644 --- a/contractor/geometry_compressor.cpp +++ b/data_structures/compressed_edge_container.cpp @@ -25,7 +25,7 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -#include "geometry_compressor.hpp" +#include "compressed_edge_container.hpp" #include "../util/simple_logger.hpp" #include @@ -35,13 +35,15 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include #include -GeometryCompressor::GeometryCompressor() +#include + +CompressedEdgeContainer::CompressedEdgeContainer() { m_free_list.reserve(100); IncreaseFreeList(); } -void GeometryCompressor::IncreaseFreeList() +void CompressedEdgeContainer::IncreaseFreeList() { m_compressed_geometries.resize(m_compressed_geometries.size() + 100); for (unsigned i = 100; i > 0; --i) @@ -51,13 +53,13 @@ void GeometryCompressor::IncreaseFreeList() } } -bool GeometryCompressor::HasEntryForID(const EdgeID edge_id) const +bool CompressedEdgeContainer::HasEntryForID(const EdgeID edge_id) const { auto iter = m_edge_id_to_list_index_map.find(edge_id); return iter != m_edge_id_to_list_index_map.end(); } -unsigned GeometryCompressor::GetPositionForID(const EdgeID edge_id) const +unsigned CompressedEdgeContainer::GetPositionForID(const EdgeID edge_id) const { auto map_iterator = m_edge_id_to_list_index_map.find(edge_id); BOOST_ASSERT(map_iterator != m_edge_id_to_list_index_map.end()); @@ -65,7 +67,7 @@ unsigned GeometryCompressor::GetPositionForID(const EdgeID edge_id) const return map_iterator->second; } -void GeometryCompressor::SerializeInternalVector(const std::string &path) const +void CompressedEdgeContainer::SerializeInternalVector(const std::string &path) const { boost::filesystem::fstream geometry_out_stream(path, std::ios::binary | std::ios::out); @@ -108,7 +110,7 @@ void GeometryCompressor::SerializeInternalVector(const std::string &path) const geometry_out_stream.close(); } -void GeometryCompressor::CompressEdge(const EdgeID edge_id_1, +void CompressedEdgeContainer::CompressEdge(const EdgeID edge_id_1, const EdgeID edge_id_2, const NodeID via_node_id, const NodeID target_node_id, @@ -153,6 +155,8 @@ void GeometryCompressor::CompressEdge(const EdgeID edge_id_1, std::vector &edge_bucket_list1 = m_compressed_geometries[edge_bucket_id1]; + // note we don't save the start coordinate: it is implicitly given by edge 1 + // weight1 is the distance to the (currently) last coordinate in the bucket if (edge_bucket_list1.empty()) { edge_bucket_list1.emplace_back(via_node_id, weight1); @@ -190,7 +194,7 @@ void GeometryCompressor::CompressEdge(const EdgeID edge_id_1, } } -void GeometryCompressor::PrintStatistics() const +void CompressedEdgeContainer::PrintStatistics() const { const uint64_t compressed_edges = m_compressed_geometries.size(); BOOST_ASSERT(0 == compressed_edges % 2); @@ -215,20 +219,20 @@ void GeometryCompressor::PrintStatistics() const std::max((uint64_t)1, compressed_edges); } -const std::vector & -GeometryCompressor::GetBucketReference(const EdgeID edge_id) const +const CompressedEdgeContainer::EdgeBucket& +CompressedEdgeContainer::GetBucketReference(const EdgeID edge_id) const { const unsigned index = m_edge_id_to_list_index_map.at(edge_id); return m_compressed_geometries.at(index); } -NodeID GeometryCompressor::GetFirstNodeIDOfBucket(const EdgeID edge_id) const +NodeID CompressedEdgeContainer::GetFirstEdgeTargetID(const EdgeID edge_id) const { const auto &bucket = GetBucketReference(edge_id); BOOST_ASSERT(bucket.size() >= 2); - return bucket[1].first; + return bucket.front().first; } -NodeID GeometryCompressor::GetLastNodeIDOfBucket(const EdgeID edge_id) const +NodeID CompressedEdgeContainer::GetLastEdgeSourceID(const EdgeID edge_id) const { const auto &bucket = GetBucketReference(edge_id); BOOST_ASSERT(bucket.size() >= 2); diff --git a/contractor/geometry_compressor.hpp b/data_structures/compressed_edge_container.hpp similarity index 86% rename from contractor/geometry_compressor.hpp rename to data_structures/compressed_edge_container.hpp index ca83fa44c9e..5d94ee6deb1 100644 --- a/contractor/geometry_compressor.hpp +++ b/data_structures/compressed_edge_container.hpp @@ -35,12 +35,13 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include #include -class GeometryCompressor +class CompressedEdgeContainer { public: using CompressedNode = std::pair; + using EdgeBucket = std::vector; - GeometryCompressor(); + CompressedEdgeContainer(); void CompressEdge(const EdgeID surviving_edge_id, const EdgeID removed_edge_id, const NodeID via_node_id, @@ -52,16 +53,15 @@ class GeometryCompressor void PrintStatistics() const; void SerializeInternalVector(const std::string &path) const; unsigned GetPositionForID(const EdgeID edge_id) const; - const std::vector & - GetBucketReference(const EdgeID edge_id) const; - NodeID GetFirstNodeIDOfBucket(const EdgeID edge_id) const; - NodeID GetLastNodeIDOfBucket(const EdgeID edge_id) const; + const EdgeBucket& GetBucketReference(const EdgeID edge_id) const; + NodeID GetFirstEdgeTargetID(const EdgeID edge_id) const; + NodeID GetLastEdgeSourceID(const EdgeID edge_id) const; private: int free_list_maximum = 0; - + void IncreaseFreeList(); - std::vector> m_compressed_geometries; + std::vector m_compressed_geometries; std::vector m_free_list; std::unordered_map m_edge_id_to_list_index_map; }; diff --git a/unit_tests/data_structures/compressed_edge_container.cpp b/unit_tests/data_structures/compressed_edge_container.cpp new file mode 100644 index 00000000000..12d9d3f4b04 --- /dev/null +++ b/unit_tests/data_structures/compressed_edge_container.cpp @@ -0,0 +1,87 @@ +#include "../../data_structures/compressed_edge_container.hpp" +#include "../../typedefs.h" + +#include +#include + +BOOST_AUTO_TEST_SUITE(compressed_edge_container) + +BOOST_AUTO_TEST_CASE(long_road_test) +{ + // 0 1 2 3 + // 0---1----2----3----4 + CompressedEdgeContainer container; + + // compress 0---1---2 to 0---2 + container.CompressEdge(0, 1, 1, 2, 1, 1); + BOOST_CHECK(container.HasEntryForID(0)); + BOOST_CHECK(!container.HasEntryForID(1)); + BOOST_CHECK(!container.HasEntryForID(2)); + BOOST_CHECK(!container.HasEntryForID(3)); + BOOST_CHECK_EQUAL(container.GetFirstEdgeTargetID(0), 1); + BOOST_CHECK_EQUAL(container.GetLastEdgeSourceID(0), 1); + + // compress 2---3---4 to 2---4 + container.CompressEdge(2, 3, 3, 4, 1, 1); + BOOST_CHECK(container.HasEntryForID(0)); + BOOST_CHECK(!container.HasEntryForID(1)); + BOOST_CHECK(container.HasEntryForID(2)); + BOOST_CHECK(!container.HasEntryForID(3)); + BOOST_CHECK_EQUAL(container.GetFirstEdgeTargetID(2), 3); + BOOST_CHECK_EQUAL(container.GetLastEdgeSourceID(2), 3); + + // compress 0---2---4 to 0---4 + container.CompressEdge(0, 2, 2, 4, 2, 2); + BOOST_CHECK(container.HasEntryForID(0)); + BOOST_CHECK(!container.HasEntryForID(1)); + BOOST_CHECK(!container.HasEntryForID(2)); + BOOST_CHECK(!container.HasEntryForID(3)); + BOOST_CHECK_EQUAL(container.GetFirstEdgeTargetID(0), 1); + BOOST_CHECK_EQUAL(container.GetLastEdgeSourceID(0), 3); +} + +BOOST_AUTO_TEST_CASE(t_crossing) +{ + // 0 1 2 3 + // 0---1---2---3---4 + // | 4 + // 5 + // | 5 + // 6 + CompressedEdgeContainer container; + + // compress 0---1---2 to 0---2 + container.CompressEdge(0, 1, 1, 2, 1, 1); + BOOST_CHECK(container.HasEntryForID(0)); + BOOST_CHECK(!container.HasEntryForID(1)); + BOOST_CHECK(!container.HasEntryForID(2)); + BOOST_CHECK(!container.HasEntryForID(3)); + BOOST_CHECK(!container.HasEntryForID(4)); + BOOST_CHECK(!container.HasEntryForID(5)); + BOOST_CHECK_EQUAL(container.GetFirstEdgeTargetID(0), 1); + BOOST_CHECK_EQUAL(container.GetLastEdgeSourceID(0), 1); + + // compress 2---5---6 to 2---6 + container.CompressEdge(4, 5, 5, 6, 1, 1); + BOOST_CHECK(container.HasEntryForID(0)); + BOOST_CHECK(!container.HasEntryForID(1)); + BOOST_CHECK(!container.HasEntryForID(2)); + BOOST_CHECK(!container.HasEntryForID(3)); + BOOST_CHECK(container.HasEntryForID(4)); + BOOST_CHECK(!container.HasEntryForID(5)); + BOOST_CHECK_EQUAL(container.GetFirstEdgeTargetID(4), 5); + BOOST_CHECK_EQUAL(container.GetLastEdgeSourceID(4), 5); + + // compress 2---3---4 to 2---4 + container.CompressEdge(2, 3, 3, 4, 1, 1); + BOOST_CHECK(container.HasEntryForID(0)); + BOOST_CHECK(!container.HasEntryForID(1)); + BOOST_CHECK(container.HasEntryForID(2)); + BOOST_CHECK(!container.HasEntryForID(3)); + BOOST_CHECK(container.HasEntryForID(4)); + BOOST_CHECK(!container.HasEntryForID(5)); + BOOST_CHECK_EQUAL(container.GetFirstEdgeTargetID(2), 3); + BOOST_CHECK_EQUAL(container.GetLastEdgeSourceID(2), 3); +} + +BOOST_AUTO_TEST_SUITE_END() From fd30e82836862d25775fc95b35db387e5b4fb465 Mon Sep 17 00:00:00 2001 From: Patrick Niklaus Date: Sun, 28 Jun 2015 15:30:40 +0200 Subject: [PATCH 003/122] Add graph compressor unit tests --- CMakeLists.txt | 4 +- .../graph_compressor.cpp | 0 .../graph_compressor.hpp | 2 +- contractor/edge_based_graph_factory.cpp | 1 - contractor/edge_based_graph_factory.hpp | 2 - contractor/processing_chain.cpp | 2 +- data_structures/dynamic_graph.hpp | 4 +- data_structures/node_based_graph.hpp | 10 + unit_tests/algorithms/graph_compressor.cpp | 232 ++++++++++++++++++ 9 files changed, 249 insertions(+), 8 deletions(-) rename {contractor => algorithms}/graph_compressor.cpp (100%) rename {contractor => algorithms}/graph_compressor.hpp (98%) create mode 100644 unit_tests/algorithms/graph_compressor.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index b1f627297ff..788d3547053 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -64,7 +64,7 @@ add_executable(osrm-extract ${ExtractorSources} $ $ $ $ $ $ $ $ $ $) @@ -77,7 +77,7 @@ file(GLOB AlgorithmGlob algorithms/*.cpp) file(GLOB HttpGlob server/http/*.cpp) file(GLOB LibOSRMGlob library/*.cpp) file(GLOB DataStructureTestsGlob unit_tests/data_structures/*.cpp data_structures/hilbert_value.cpp data_structures/compressed_edge_container.cpp) -file(GLOB AlgorithmTestsGlob unit_tests/algorithms/*.cpp) +file(GLOB AlgorithmTestsGlob unit_tests/algorithms/*.cpp algorithms/graph_compressor.cpp data_structures/compressed_edge_container.cpp data_structures/restriction_map.cpp) set( OSRMSources diff --git a/contractor/graph_compressor.cpp b/algorithms/graph_compressor.cpp similarity index 100% rename from contractor/graph_compressor.cpp rename to algorithms/graph_compressor.cpp diff --git a/contractor/graph_compressor.hpp b/algorithms/graph_compressor.hpp similarity index 98% rename from contractor/graph_compressor.hpp rename to algorithms/graph_compressor.hpp index 0e349423cb9..c1cca153e6c 100644 --- a/contractor/graph_compressor.hpp +++ b/algorithms/graph_compressor.hpp @@ -29,7 +29,7 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "../typedefs.h" -#include "speed_profile.hpp" +#include "../contractor/speed_profile.hpp" #include "../data_structures/node_based_graph.hpp" #include diff --git a/contractor/edge_based_graph_factory.cpp b/contractor/edge_based_graph_factory.cpp index c5c31ac5b9c..110311df6e8 100644 --- a/contractor/edge_based_graph_factory.cpp +++ b/contractor/edge_based_graph_factory.cpp @@ -266,7 +266,6 @@ void EdgeBasedGraphFactory::RenumberEdges() BOOST_ASSERT(SPECIAL_NODEID != edge_data.edgeBasedNodeID); } } - m_number_of_edge_based_nodes = numbered_edges_count; } /// Creates the nodes in the edge expanded graph from edges in the node-based graph. diff --git a/contractor/edge_based_graph_factory.hpp b/contractor/edge_based_graph_factory.hpp index 20a3feda442..e4829737ca9 100644 --- a/contractor/edge_based_graph_factory.hpp +++ b/contractor/edge_based_graph_factory.hpp @@ -81,8 +81,6 @@ class EdgeBasedGraphFactory private: using EdgeData = NodeBasedDynamicGraph::EdgeData; - unsigned m_number_of_edge_based_nodes; - std::vector m_edge_based_node_list; DeallocatingVector m_edge_based_edge_list; diff --git a/contractor/processing_chain.cpp b/contractor/processing_chain.cpp index 4a6a12c5b39..461d38afe88 100644 --- a/contractor/processing_chain.cpp +++ b/contractor/processing_chain.cpp @@ -28,7 +28,7 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "processing_chain.hpp" #include "contractor.hpp" -#include "graph_compressor.hpp" +#include "../algorithms/graph_compressor.hpp" #include "../algorithms/crc32_processor.hpp" #include "../data_structures/compressed_edge_container.hpp" diff --git a/data_structures/dynamic_graph.hpp b/data_structures/dynamic_graph.hpp index b18204c6143..f96eb19927d 100644 --- a/data_structures/dynamic_graph.hpp +++ b/data_structures/dynamic_graph.hpp @@ -90,9 +90,11 @@ template class DynamicGraph */ template DynamicGraph(const NodeIterator nodes, const ContainerT &graph) { + // we need to cast here because DeallocatingVector does not have a valid const iterator + BOOST_ASSERT(std::is_sorted(const_cast(graph).begin(), const_cast(graph).end())); + number_of_nodes = nodes; number_of_edges = static_cast(graph.size()); - // node_array.reserve(number_of_nodes + 1); node_array.resize(number_of_nodes + 1); EdgeIterator edge = 0; EdgeIterator position = 0; diff --git a/data_structures/node_based_graph.hpp b/data_structures/node_based_graph.hpp index b69053a979c..c44f92b532a 100644 --- a/data_structures/node_based_graph.hpp +++ b/data_structures/node_based_graph.hpp @@ -46,6 +46,16 @@ struct NodeBasedEdgeData { } + NodeBasedEdgeData(int distance, unsigned edgeBasedNodeID, unsigned nameID, + bool isAccessRestricted, bool shortcut, bool forward, bool backward, + bool roundabout, bool ignore_in_grid, TravelMode travel_mode) + : distance(distance), edgeBasedNodeID(edgeBasedNodeID), + nameID(nameID), isAccessRestricted(isAccessRestricted), shortcut(shortcut), + forward(forward), backward(backward), roundabout(roundabout), ignore_in_grid(ignore_in_grid), + travel_mode(travel_mode) + { + } + int distance; unsigned edgeBasedNodeID; unsigned nameID; diff --git a/unit_tests/algorithms/graph_compressor.cpp b/unit_tests/algorithms/graph_compressor.cpp new file mode 100644 index 00000000000..0b09752a6d5 --- /dev/null +++ b/unit_tests/algorithms/graph_compressor.cpp @@ -0,0 +1,232 @@ +#include "../../algorithms/graph_compressor.hpp" +#include "../../data_structures/compressed_edge_container.hpp" +#include "../../data_structures/restriction_map.hpp" +#include "../../data_structures/node_based_graph.hpp" +#include "../../contractor/speed_profile.hpp" +#include "../../typedefs.h" + +#include +#include + +#include + +BOOST_AUTO_TEST_SUITE(graph_compressor) + +void dumpGraph(const NodeBasedDynamicGraph& graph) +{ + for (auto i = 0u; i < graph.GetNumberOfNodes(); ++i) + { + std::cout << "## node " << i << " degree: " << graph.GetOutDegree(i) << std::endl; + for (auto e = graph.BeginEdges(i); e < graph.EndEdges(i); ++e) + { + const auto& data = graph.GetEdgeData(e); + auto target = graph.GetTarget(e); + if (data.forward && !data.backward) + std::cout << i << "->" << target << std::endl; + if (!data.forward && data.backward) + std::cout << i << "<-" << target << std::endl; + if (data.forward && data.backward) + std::cout << i << "--" << target << std::endl; + } + } +} + +BOOST_AUTO_TEST_CASE(long_road_test) +{ + // + // 0---1---2---3---4 + // + SpeedProfileProperties speed_profile; + GraphCompressor compressor(speed_profile); + + std::unordered_set barrier_nodes; + std::unordered_set traffic_lights; + RestrictionMap map; + CompressedEdgeContainer container; + + using InputEdge = NodeBasedDynamicGraph::InputEdge; + std::vector edges = { + // source, target, distance, edgeBasedNodeID, nameID, isAccessRestricted, shortcut, forward, backward, roundabout, ignore_in_grid, travel_mode + {0, 1, 1, SPECIAL_EDGEID, 0, false, false, true, true, false, false, TRAVEL_MODE_DEFAULT}, + {1, 0, 1, SPECIAL_EDGEID, 0, false, false, true, true, false, false, TRAVEL_MODE_DEFAULT}, + {1, 2, 1, SPECIAL_EDGEID, 0, false, false, true, true, false, false, TRAVEL_MODE_DEFAULT}, + {2, 1, 1, SPECIAL_EDGEID, 0, false, false, true, true, false, false, TRAVEL_MODE_DEFAULT}, + {2, 3, 1, SPECIAL_EDGEID, 0, false, false, true, true, false, false, TRAVEL_MODE_DEFAULT}, + {3, 2, 1, SPECIAL_EDGEID, 0, false, false, true, true, false, false, TRAVEL_MODE_DEFAULT}, + {3, 4, 1, SPECIAL_EDGEID, 0, false, false, true, true, false, false, TRAVEL_MODE_DEFAULT}, + {4, 3, 1, SPECIAL_EDGEID, 0, false, false, true, true, false, false, TRAVEL_MODE_DEFAULT} + }; + + BOOST_ASSERT(edges[0].data.IsCompatibleTo(edges[2].data)); + BOOST_ASSERT(edges[2].data.IsCompatibleTo(edges[4].data)); + BOOST_ASSERT(edges[4].data.IsCompatibleTo(edges[6].data)); + + NodeBasedDynamicGraph graph(5, edges); + compressor.Compress(barrier_nodes, traffic_lights, map, graph, container); + + BOOST_CHECK_EQUAL(graph.FindEdge(0, 1), SPECIAL_EDGEID); + BOOST_CHECK_EQUAL(graph.FindEdge(1, 2), SPECIAL_EDGEID); + BOOST_CHECK_EQUAL(graph.FindEdge(2, 3), SPECIAL_EDGEID); + BOOST_CHECK_EQUAL(graph.FindEdge(3, 4), SPECIAL_EDGEID); + BOOST_CHECK(graph.FindEdge(0, 4) != SPECIAL_EDGEID); +} + +BOOST_AUTO_TEST_CASE(loop_test) +{ + // + // 0---1---2 + // | | + // 5---4---3 + // + SpeedProfileProperties speed_profile; + GraphCompressor compressor(speed_profile); + + std::unordered_set barrier_nodes; + std::unordered_set traffic_lights; + RestrictionMap map; + CompressedEdgeContainer container; + + using InputEdge = NodeBasedDynamicGraph::InputEdge; + std::vector edges = { + // source, target, distance, edgeBasedNodeID, nameID, isAccessRestricted, shortcut, forward, backward, roundabout, ignore_in_grid, travel_mode + {0, 1, 1, SPECIAL_EDGEID, 0, false, false, true, true, false, false, TRAVEL_MODE_DEFAULT}, + {0, 5, 1, SPECIAL_EDGEID, 0, false, false, true, true, false, false, TRAVEL_MODE_DEFAULT}, + {1, 0, 1, SPECIAL_EDGEID, 0, false, false, true, true, false, false, TRAVEL_MODE_DEFAULT}, + {1, 2, 1, SPECIAL_EDGEID, 0, false, false, true, true, false, false, TRAVEL_MODE_DEFAULT}, + {2, 1, 1, SPECIAL_EDGEID, 0, false, false, true, true, false, false, TRAVEL_MODE_DEFAULT}, + {2, 3, 1, SPECIAL_EDGEID, 0, false, false, true, true, false, false, TRAVEL_MODE_DEFAULT}, + {3, 2, 1, SPECIAL_EDGEID, 0, false, false, true, true, false, false, TRAVEL_MODE_DEFAULT}, + {3, 4, 1, SPECIAL_EDGEID, 0, false, false, true, true, false, false, TRAVEL_MODE_DEFAULT}, + {4, 3, 1, SPECIAL_EDGEID, 0, false, false, true, true, false, false, TRAVEL_MODE_DEFAULT}, + {4, 5, 1, SPECIAL_EDGEID, 0, false, false, true, true, false, false, TRAVEL_MODE_DEFAULT}, + {5, 0, 1, SPECIAL_EDGEID, 0, false, false, true, true, false, false, TRAVEL_MODE_DEFAULT}, + {5, 4, 1, SPECIAL_EDGEID, 0, false, false, true, true, false, false, TRAVEL_MODE_DEFAULT}, + }; + + BOOST_ASSERT(edges.size() == 12); + BOOST_ASSERT(edges[0].data.IsCompatibleTo(edges[1].data)); + BOOST_ASSERT(edges[1].data.IsCompatibleTo(edges[2].data)); + BOOST_ASSERT(edges[2].data.IsCompatibleTo(edges[3].data)); + BOOST_ASSERT(edges[3].data.IsCompatibleTo(edges[4].data)); + BOOST_ASSERT(edges[4].data.IsCompatibleTo(edges[5].data)); + BOOST_ASSERT(edges[5].data.IsCompatibleTo(edges[6].data)); + BOOST_ASSERT(edges[6].data.IsCompatibleTo(edges[7].data)); + BOOST_ASSERT(edges[7].data.IsCompatibleTo(edges[8].data)); + BOOST_ASSERT(edges[8].data.IsCompatibleTo(edges[9].data)); + BOOST_ASSERT(edges[9].data.IsCompatibleTo(edges[10].data)); + BOOST_ASSERT(edges[10].data.IsCompatibleTo(edges[11].data)); + + NodeBasedDynamicGraph graph(6, edges); + compressor.Compress(barrier_nodes, traffic_lights, map, graph, container); + + BOOST_CHECK_EQUAL(graph.FindEdge(5, 0), SPECIAL_EDGEID); + BOOST_CHECK_EQUAL(graph.FindEdge(0, 1), SPECIAL_EDGEID); + BOOST_CHECK_EQUAL(graph.FindEdge(1, 2), SPECIAL_EDGEID); + BOOST_CHECK_EQUAL(graph.FindEdge(2, 3), SPECIAL_EDGEID); + BOOST_CHECK(graph.FindEdge(5, 3) != SPECIAL_EDGEID); + BOOST_CHECK(graph.FindEdge(3, 4) != SPECIAL_EDGEID); + BOOST_CHECK(graph.FindEdge(4, 5) != SPECIAL_EDGEID); +} + +BOOST_AUTO_TEST_CASE(t_intersection) +{ + // + // 0---1---2 + // | + // 3 + // + SpeedProfileProperties speed_profile; + GraphCompressor compressor(speed_profile); + + std::unordered_set barrier_nodes; + std::unordered_set traffic_lights; + RestrictionMap map; + CompressedEdgeContainer container; + + using InputEdge = NodeBasedDynamicGraph::InputEdge; + std::vector edges = { + // source, target, distance, edgeBasedNodeID, nameID, isAccessRestricted, shortcut, forward, backward, roundabout, ignore_in_grid, travel_mode + {0, 1, 1, SPECIAL_EDGEID, 0, false, false, true, true, false, false, TRAVEL_MODE_DEFAULT}, + {1, 0, 1, SPECIAL_EDGEID, 0, false, false, true, true, false, false, TRAVEL_MODE_DEFAULT}, + {1, 2, 1, SPECIAL_EDGEID, 0, false, false, true, true, false, false, TRAVEL_MODE_DEFAULT}, + {1, 3, 1, SPECIAL_EDGEID, 0, false, false, true, true, false, false, TRAVEL_MODE_DEFAULT}, + {2, 1, 1, SPECIAL_EDGEID, 0, false, false, true, true, false, false, TRAVEL_MODE_DEFAULT}, + {3, 1, 1, SPECIAL_EDGEID, 0, false, false, true, true, false, false, TRAVEL_MODE_DEFAULT}, + }; + + BOOST_ASSERT(edges[0].data.IsCompatibleTo(edges[1].data)); + BOOST_ASSERT(edges[1].data.IsCompatibleTo(edges[2].data)); + BOOST_ASSERT(edges[2].data.IsCompatibleTo(edges[3].data)); + BOOST_ASSERT(edges[3].data.IsCompatibleTo(edges[4].data)); + BOOST_ASSERT(edges[4].data.IsCompatibleTo(edges[5].data)); + + NodeBasedDynamicGraph graph(4, edges); + compressor.Compress(barrier_nodes, traffic_lights, map, graph, container); + + BOOST_CHECK(graph.FindEdge(0, 1) != SPECIAL_EDGEID); + BOOST_CHECK(graph.FindEdge(1, 2) != SPECIAL_EDGEID); + BOOST_CHECK(graph.FindEdge(1, 3) != SPECIAL_EDGEID); +} + +BOOST_AUTO_TEST_CASE(street_name_changes) +{ + // + // 0---1---2 + // + SpeedProfileProperties speed_profile; + GraphCompressor compressor(speed_profile); + + std::unordered_set barrier_nodes; + std::unordered_set traffic_lights; + RestrictionMap map; + CompressedEdgeContainer container; + + using InputEdge = NodeBasedDynamicGraph::InputEdge; + std::vector edges = { + // source, target, distance, edgeBasedNodeID, nameID, isAccessRestricted, shortcut, forward, backward, roundabout, ignore_in_grid, travel_mode + {0, 1, 1, SPECIAL_EDGEID, 0, false, false, true, true, false, false, TRAVEL_MODE_DEFAULT}, + {1, 0, 1, SPECIAL_EDGEID, 0, false, false, true, true, false, false, TRAVEL_MODE_DEFAULT}, + {1, 2, 1, SPECIAL_EDGEID, 1, false, false, true, true, false, false, TRAVEL_MODE_DEFAULT}, + {2, 1, 1, SPECIAL_EDGEID, 1, false, false, true, true, false, false, TRAVEL_MODE_DEFAULT}, + }; + + BOOST_ASSERT(edges[0].data.IsCompatibleTo(edges[1].data)); + BOOST_ASSERT(edges[2].data.IsCompatibleTo(edges[3].data)); + + NodeBasedDynamicGraph graph(5, edges); + compressor.Compress(barrier_nodes, traffic_lights, map, graph, container); + + BOOST_CHECK(graph.FindEdge(0, 1) != SPECIAL_EDGEID); + BOOST_CHECK(graph.FindEdge(1, 2) != SPECIAL_EDGEID); +} + +BOOST_AUTO_TEST_CASE(direction_changes) +{ + // + // 0-->1---2 + // + SpeedProfileProperties speed_profile; + GraphCompressor compressor(speed_profile); + + std::unordered_set barrier_nodes; + std::unordered_set traffic_lights; + RestrictionMap map; + CompressedEdgeContainer container; + + using InputEdge = NodeBasedDynamicGraph::InputEdge; + std::vector edges = { + // source, target, distance, edgeBasedNodeID, nameID, isAccessRestricted, shortcut, forward, backward, roundabout, ignore_in_grid, travel_mode + {0, 1, 1, SPECIAL_EDGEID, 0, false, false, true, false, false, false, TRAVEL_MODE_DEFAULT}, + {1, 0, 1, SPECIAL_EDGEID, 0, false, false, false, true, false, false, TRAVEL_MODE_DEFAULT}, + {1, 2, 1, SPECIAL_EDGEID, 0, false, false, true, true, false, false, TRAVEL_MODE_DEFAULT}, + {2, 1, 1, SPECIAL_EDGEID, 0, false, false, true, true, false, false, TRAVEL_MODE_DEFAULT}, + }; + + NodeBasedDynamicGraph graph(5, edges); + compressor.Compress(barrier_nodes, traffic_lights, map, graph, container); + + BOOST_CHECK(graph.FindEdge(0, 1) != SPECIAL_EDGEID); + BOOST_CHECK(graph.FindEdge(1, 2) != SPECIAL_EDGEID); +} + +BOOST_AUTO_TEST_SUITE_END() From faa880d60a162728f8a9dd06c87373b8924d0837 Mon Sep 17 00:00:00 2001 From: Patrick Niklaus Date: Sun, 28 Jun 2015 22:13:54 +0200 Subject: [PATCH 004/122] Remove unused memebers and rename to currrent style convention --- algorithms/graph_compressor.cpp | 12 ++-- contractor/edge_based_graph_factory.cpp | 47 +++++++------ data_structures/edge_based_node.hpp | 2 + data_structures/import_edge.cpp | 5 +- data_structures/import_edge.hpp | 2 - data_structures/node_based_graph.hpp | 37 +++++----- extractor/extraction_way.hpp | 2 - extractor/extractor_callbacks.cpp | 12 ++-- extractor/internal_extractor_edge.hpp | 9 ++- extractor/scripting_environment.cpp | 1 - profiles/car.lua | 6 -- unit_tests/algorithms/graph_compressor.cpp | 78 +++++++++++----------- 12 files changed, 97 insertions(+), 116 deletions(-) diff --git a/algorithms/graph_compressor.cpp b/algorithms/graph_compressor.cpp index e8ad384fc3f..c924360b653 100644 --- a/algorithms/graph_compressor.cpp +++ b/algorithms/graph_compressor.cpp @@ -96,18 +96,18 @@ void GraphCompressor::Compress(const std::unordered_set& barrier_nodes, } // this case can happen if two ways with different names overlap - if (fwd_edge_data1.nameID != rev_edge_data1.nameID || - fwd_edge_data2.nameID != rev_edge_data2.nameID) + if (fwd_edge_data1.name_id != rev_edge_data1.name_id || + fwd_edge_data2.name_id != rev_edge_data2.name_id) { continue; } if (fwd_edge_data1.IsCompatibleTo(fwd_edge_data2) && rev_edge_data1.IsCompatibleTo(rev_edge_data2)) { - BOOST_ASSERT(graph.GetEdgeData(forward_e1).nameID == - graph.GetEdgeData(reverse_e1).nameID); - BOOST_ASSERT(graph.GetEdgeData(forward_e2).nameID == - graph.GetEdgeData(reverse_e2).nameID); + BOOST_ASSERT(graph.GetEdgeData(forward_e1).name_id == + graph.GetEdgeData(reverse_e1).name_id); + BOOST_ASSERT(graph.GetEdgeData(forward_e2).name_id == + graph.GetEdgeData(reverse_e2).name_id); // Get distances before graph is modified const int forward_weight1 = graph.GetEdgeData(forward_e1).distance; diff --git a/contractor/edge_based_graph_factory.cpp b/contractor/edge_based_graph_factory.cpp index 110311df6e8..76880ce9e21 100644 --- a/contractor/edge_based_graph_factory.cpp +++ b/contractor/edge_based_graph_factory.cpp @@ -97,8 +97,8 @@ void EdgeBasedGraphFactory::InsertEdgeBasedNode(const NodeID node_u, const EdgeData &reverse_data = m_node_based_graph->GetEdgeData(edge_id_2); - if (forward_data.edgeBasedNodeID == SPECIAL_NODEID && - reverse_data.edgeBasedNodeID == SPECIAL_NODEID) + if (forward_data.edge_id == SPECIAL_NODEID && + reverse_data.edge_id == SPECIAL_NODEID) { return; } @@ -154,9 +154,9 @@ void EdgeBasedGraphFactory::InsertEdgeBasedNode(const NodeID node_u, // build edges m_edge_based_node_list.emplace_back( - forward_data.edgeBasedNodeID, reverse_data.edgeBasedNodeID, + forward_data.edge_id, reverse_data.edge_id, current_edge_source_coordinate_id, current_edge_target_coordinate_id, - forward_data.nameID, forward_geometry[i].second, + forward_data.name_id, forward_geometry[i].second, reverse_geometry[geometry_size - 1 - i].second, forward_dist_prefix_sum[i], reverse_dist_prefix_sum[i], m_compressed_edge_container.GetPositionForID(edge_id_1), component_id, i, forward_data.travel_mode, reverse_data.travel_mode); @@ -178,7 +178,7 @@ void EdgeBasedGraphFactory::InsertEdgeBasedNode(const NodeID node_u, { BOOST_ASSERT(!m_compressed_edge_container.HasEntryForID(edge_id_2)); - if (forward_data.edgeBasedNodeID != SPECIAL_NODEID) + if (forward_data.edge_id != SPECIAL_NODEID) { BOOST_ASSERT(forward_data.forward); } @@ -187,7 +187,7 @@ void EdgeBasedGraphFactory::InsertEdgeBasedNode(const NodeID node_u, BOOST_ASSERT(!forward_data.forward); } - if (reverse_data.edgeBasedNodeID != SPECIAL_NODEID) + if (reverse_data.edge_id != SPECIAL_NODEID) { BOOST_ASSERT(reverse_data.forward); } @@ -196,12 +196,12 @@ void EdgeBasedGraphFactory::InsertEdgeBasedNode(const NodeID node_u, BOOST_ASSERT(!reverse_data.forward); } - BOOST_ASSERT(forward_data.edgeBasedNodeID != SPECIAL_NODEID || - reverse_data.edgeBasedNodeID != SPECIAL_NODEID); + BOOST_ASSERT(forward_data.edge_id != SPECIAL_NODEID || + reverse_data.edge_id != SPECIAL_NODEID); m_edge_based_node_list.emplace_back( - forward_data.edgeBasedNodeID, reverse_data.edgeBasedNodeID, node_u, node_v, - forward_data.nameID, forward_data.distance, reverse_data.distance, 0, 0, SPECIAL_EDGEID, + forward_data.edge_id, reverse_data.edge_id, node_u, node_v, + forward_data.name_id, forward_data.distance, reverse_data.distance, 0, 0, SPECIAL_EDGEID, component_id, 0, forward_data.travel_mode, reverse_data.travel_mode); BOOST_ASSERT(!m_edge_based_node_list.back().IsCompressed()); } @@ -241,7 +241,7 @@ void EdgeBasedGraphFactory::Run(const std::string &original_edge_data_filename, } -/// Renumbers all _forward_ edges and sets the edgeBasedNodeID. +/// Renumbers all _forward_ edges and sets the edge_id. /// A specific numbering is not important. Any unique ID will do. void EdgeBasedGraphFactory::RenumberEdges() { @@ -260,10 +260,10 @@ void EdgeBasedGraphFactory::RenumberEdges() } BOOST_ASSERT(numbered_edges_count < m_node_based_graph->GetNumberOfEdges()); - edge_data.edgeBasedNodeID = numbered_edges_count; + edge_data.edge_id = numbered_edges_count; ++numbered_edges_count; - BOOST_ASSERT(SPECIAL_NODEID != edge_data.edgeBasedNodeID); + BOOST_ASSERT(SPECIAL_NODEID != edge_data.edge_id); } } } @@ -329,8 +329,8 @@ void EdgeBasedGraphFactory::GenerateEdgeExpandedNodes() const bool component_is_tiny = size_of_component < 1000; - // we only set edgeBasedNodeID for forward edges - if (edge_data.edgeBasedNodeID == SPECIAL_NODEID) + // we only set edge_id for forward edges + if (edge_data.edge_id == SPECIAL_NODEID) { InsertEdgeBasedNode(node_v, node_u, (component_is_tiny ? id_of_smaller_component + 1 : 0)); @@ -438,7 +438,7 @@ void EdgeBasedGraphFactory::GenerateEdgeExpandedEdges( const EdgeData &edge_data1 = m_node_based_graph->GetEdgeData(e1); const EdgeData &edge_data2 = m_node_based_graph->GetEdgeData(e2); - BOOST_ASSERT(edge_data1.edgeBasedNodeID != edge_data2.edgeBasedNodeID); + BOOST_ASSERT(edge_data1.edge_id != edge_data2.edge_id); BOOST_ASSERT(edge_data1.forward); BOOST_ASSERT(edge_data2.forward); @@ -481,7 +481,7 @@ void EdgeBasedGraphFactory::GenerateEdgeExpandedEdges( original_edge_data_vector.emplace_back( (edge_is_compressed ? m_compressed_edge_container.GetPositionForID(e1) : node_v), - edge_data1.nameID, turn_instruction, edge_is_compressed, + edge_data1.name_id, turn_instruction, edge_is_compressed, edge_data2.travel_mode); ++original_edges_counter; @@ -491,12 +491,11 @@ void EdgeBasedGraphFactory::GenerateEdgeExpandedEdges( FlushVectorToStream(edge_data_file, original_edge_data_vector); } - BOOST_ASSERT(SPECIAL_NODEID != edge_data1.edgeBasedNodeID); - BOOST_ASSERT(SPECIAL_NODEID != edge_data2.edgeBasedNodeID); + BOOST_ASSERT(SPECIAL_NODEID != edge_data1.edge_id); + BOOST_ASSERT(SPECIAL_NODEID != edge_data2.edge_id); - m_edge_based_edge_list.emplace_back( - EdgeBasedEdge(edge_data1.edgeBasedNodeID, edge_data2.edgeBasedNodeID, - m_edge_based_edge_list.size(), distance, true, false)); + m_edge_based_edge_list.emplace_back(edge_data1.edge_id, edge_data2.edge_id, + m_edge_based_edge_list.size(), distance, true, false); } } } @@ -579,11 +578,11 @@ TurnInstruction EdgeBasedGraphFactory::AnalyzeTurn(const NodeID node_u, // If street names stay the same and if we are certain that it is not a // a segment of a roundabout, we skip it. - if (data1.nameID == data2.nameID) + if (data1.name_id == data2.name_id) { // TODO: Here we should also do a small graph exploration to check for // more complex situations - if (0 != data1.nameID || m_node_based_graph->GetOutDegree(node_v) <= 2) + if (0 != data1.name_id || m_node_based_graph->GetOutDegree(node_v) <= 2) { return TurnInstruction::NoTurn; } diff --git a/data_structures/edge_based_node.hpp b/data_structures/edge_based_node.hpp index 72a585a1397..027644d7b6a 100644 --- a/data_structures/edge_based_node.hpp +++ b/data_structures/edge_based_node.hpp @@ -37,6 +37,8 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include +/// This is what StaticRTree serialized and stores on disk +/// It is generated in EdgeBasedGraphFactory. struct EdgeBasedNode { EdgeBasedNode() diff --git a/data_structures/import_edge.cpp b/data_structures/import_edge.cpp index 3f2146ca3ad..15394f1e3ea 100644 --- a/data_structures/import_edge.cpp +++ b/data_structures/import_edge.cpp @@ -49,7 +49,7 @@ bool NodeBasedEdge::operator<(const NodeBasedEdge &other) const NodeBasedEdge::NodeBasedEdge() : source(SPECIAL_NODEID), target(SPECIAL_NODEID), name_id(0), weight(0), forward(false), - backward(false), roundabout(false), in_tiny_cc(false), + backward(false), roundabout(false), access_restricted(false), is_split(false), travel_mode(false) { } @@ -61,12 +61,11 @@ NodeBasedEdge::NodeBasedEdge(NodeID source, bool forward, bool backward, bool roundabout, - bool in_tiny_cc, bool access_restricted, TravelMode travel_mode, bool is_split) : source(source), target(target), name_id(name_id), weight(weight), forward(forward), - backward(backward), roundabout(roundabout), in_tiny_cc(in_tiny_cc), + backward(backward), roundabout(roundabout), access_restricted(access_restricted), is_split(is_split), travel_mode(travel_mode) { } diff --git a/data_structures/import_edge.hpp b/data_structures/import_edge.hpp index e4898507366..8c5325cfac6 100644 --- a/data_structures/import_edge.hpp +++ b/data_structures/import_edge.hpp @@ -43,7 +43,6 @@ struct NodeBasedEdge bool forward, bool backward, bool roundabout, - bool in_tiny_cc, bool access_restricted, TravelMode travel_mode, bool is_split); @@ -55,7 +54,6 @@ struct NodeBasedEdge bool forward : 1; bool backward : 1; bool roundabout : 1; - bool in_tiny_cc : 1; bool access_restricted : 1; bool is_split : 1; TravelMode travel_mode : 4; diff --git a/data_structures/node_based_graph.hpp b/data_structures/node_based_graph.hpp index c44f92b532a..60d05d15bb6 100644 --- a/data_structures/node_based_graph.hpp +++ b/data_structures/node_based_graph.hpp @@ -39,32 +39,28 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. struct NodeBasedEdgeData { NodeBasedEdgeData() - : distance(INVALID_EDGE_WEIGHT), edgeBasedNodeID(SPECIAL_NODEID), - nameID(std::numeric_limits::max()), isAccessRestricted(false), shortcut(false), - forward(false), backward(false), roundabout(false), ignore_in_grid(false), - travel_mode(TRAVEL_MODE_INACCESSIBLE) + : distance(INVALID_EDGE_WEIGHT), edge_id(SPECIAL_NODEID), + name_id(std::numeric_limits::max()), access_restricted(false), + forward(false), backward(false), roundabout(false), travel_mode(TRAVEL_MODE_INACCESSIBLE) { } - NodeBasedEdgeData(int distance, unsigned edgeBasedNodeID, unsigned nameID, - bool isAccessRestricted, bool shortcut, bool forward, bool backward, - bool roundabout, bool ignore_in_grid, TravelMode travel_mode) - : distance(distance), edgeBasedNodeID(edgeBasedNodeID), - nameID(nameID), isAccessRestricted(isAccessRestricted), shortcut(shortcut), - forward(forward), backward(backward), roundabout(roundabout), ignore_in_grid(ignore_in_grid), - travel_mode(travel_mode) + NodeBasedEdgeData(int distance, unsigned edge_id, unsigned name_id, + bool access_restricted, bool forward, bool backward, + bool roundabout, TravelMode travel_mode) + : distance(distance), edge_id(edge_id), + name_id(name_id), access_restricted(access_restricted), + forward(forward), backward(backward), roundabout(roundabout), travel_mode(travel_mode) { } int distance; - unsigned edgeBasedNodeID; - unsigned nameID; - bool isAccessRestricted : 1; - bool shortcut : 1; + unsigned edge_id; + unsigned name_id; + bool access_restricted : 1; bool forward : 1; bool backward : 1; bool roundabout : 1; - bool ignore_in_grid : 1; TravelMode travel_mode : 4; void SwapDirectionFlags() @@ -77,8 +73,7 @@ struct NodeBasedEdgeData bool IsCompatibleTo(const NodeBasedEdgeData &other) const { return (forward == other.forward) && (backward == other.backward) && - (nameID == other.nameID) && (ignore_in_grid == other.ignore_in_grid) && - (travel_mode == other.travel_mode); + (name_id == other.name_id) && (travel_mode == other.travel_mode); } }; @@ -211,11 +206,9 @@ NodeBasedDynamicGraphFromImportEdges(int number_of_nodes, std::vector(import_edge.weight); BOOST_ASSERT(edge.data.distance > 0); - edge.data.shortcut = false; edge.data.roundabout = import_edge.roundabout; - edge.data.ignore_in_grid = import_edge.in_tiny_cc; - edge.data.nameID = import_edge.name_id; - edge.data.isAccessRestricted = import_edge.access_restricted; + edge.data.name_id = import_edge.name_id; + edge.data.access_restricted = import_edge.access_restricted; edge.data.travel_mode = import_edge.travel_mode; edges_list.push_back(edge); diff --git a/extractor/extraction_way.hpp b/extractor/extraction_way.hpp index d344e366529..23fd50a474b 100644 --- a/extractor/extraction_way.hpp +++ b/extractor/extraction_way.hpp @@ -51,7 +51,6 @@ struct ExtractionWay duration = -1; roundabout = false; is_access_restricted = false; - ignore_in_grid = false; name.clear(); forward_travel_mode = TRAVEL_MODE_DEFAULT; backward_travel_mode = TRAVEL_MODE_DEFAULT; @@ -121,7 +120,6 @@ struct ExtractionWay std::string name; bool roundabout; bool is_access_restricted; - bool ignore_in_grid; TravelMode forward_travel_mode : 4; TravelMode backward_travel_mode : 4; }; diff --git a/extractor/extractor_callbacks.cpp b/extractor/extractor_callbacks.cpp index ed9087adb7f..6a50f16c111 100644 --- a/extractor/extractor_callbacks.cpp +++ b/extractor/extractor_callbacks.cpp @@ -183,8 +183,8 @@ void ExtractorCallbacks::ProcessWay(const osmium::Way &input_way, const Extracti { external_memory.all_edges_list.push_back(InternalExtractorEdge( first_node.ref(), last_node.ref(), name_id, backward_weight_data, - true, false, parsed_way.roundabout, parsed_way.ignore_in_grid, - parsed_way.is_access_restricted, parsed_way.backward_travel_mode, false)); + true, false, parsed_way.roundabout, parsed_way.is_access_restricted, + parsed_way.backward_travel_mode, false)); }); external_memory.way_start_end_id_list.push_back( @@ -205,8 +205,8 @@ void ExtractorCallbacks::ProcessWay(const osmium::Way &input_way, const Extracti { external_memory.all_edges_list.push_back(InternalExtractorEdge( first_node.ref(), last_node.ref(), name_id, forward_weight_data, - true, !forward_only, parsed_way.roundabout, parsed_way.ignore_in_grid, - parsed_way.is_access_restricted, parsed_way.forward_travel_mode, split_edge)); + true, !forward_only, parsed_way.roundabout, parsed_way.is_access_restricted, + parsed_way.forward_travel_mode, split_edge)); }); if (split_edge) { @@ -216,8 +216,8 @@ void ExtractorCallbacks::ProcessWay(const osmium::Way &input_way, const Extracti { external_memory.all_edges_list.push_back(InternalExtractorEdge( first_node.ref(), last_node.ref(), name_id, backward_weight_data, - false, true, parsed_way.roundabout, parsed_way.ignore_in_grid, - parsed_way.is_access_restricted, parsed_way.backward_travel_mode, true)); + false, true, parsed_way.roundabout, parsed_way.is_access_restricted, + parsed_way.backward_travel_mode, true)); }); } diff --git a/extractor/internal_extractor_edge.hpp b/extractor/internal_extractor_edge.hpp index 15f3c0d66cd..462bd3531d8 100644 --- a/extractor/internal_extractor_edge.hpp +++ b/extractor/internal_extractor_edge.hpp @@ -62,7 +62,7 @@ struct InternalExtractorEdge }; explicit InternalExtractorEdge() - : result(0, 0, 0, 0, false, false, false, false, false, + : result(0, 0, 0, 0, false, false, false, false, TRAVEL_MODE_INACCESSIBLE, false) { } @@ -74,12 +74,11 @@ struct InternalExtractorEdge bool forward, bool backward, bool roundabout, - bool in_tiny_cc, bool access_restricted, TravelMode travel_mode, bool is_split) : result(source, target, name_id, 0, forward, backward, roundabout, - in_tiny_cc, access_restricted, travel_mode, is_split), + access_restricted, travel_mode, is_split), weight_data(weight_data) { } @@ -96,11 +95,11 @@ struct InternalExtractorEdge static InternalExtractorEdge min_value() { return InternalExtractorEdge(0, 0, 0, WeightData(), false, false, false, - false, false, TRAVEL_MODE_INACCESSIBLE, false); + false, TRAVEL_MODE_INACCESSIBLE, false); } static InternalExtractorEdge max_value() { - return InternalExtractorEdge(SPECIAL_NODEID, SPECIAL_NODEID, 0, WeightData(), false, false, + return InternalExtractorEdge(SPECIAL_NODEID, SPECIAL_NODEID, 0, WeightData(), false, false, false, false, TRAVEL_MODE_INACCESSIBLE, false); } }; diff --git a/extractor/scripting_environment.cpp b/extractor/scripting_environment.cpp index b32ab4b2591..e98f688c6c7 100644 --- a/extractor/scripting_environment.cpp +++ b/extractor/scripting_environment.cpp @@ -107,7 +107,6 @@ void ScriptingEnvironment::init_lua_state(lua_State *lua_state) .def_readwrite("name", &ExtractionWay::name) .def_readwrite("roundabout", &ExtractionWay::roundabout) .def_readwrite("is_access_restricted", &ExtractionWay::is_access_restricted) - .def_readwrite("ignore_in_index", &ExtractionWay::ignore_in_grid) .def_readwrite("duration", &ExtractionWay::duration) .property("forward_mode", &ExtractionWay::get_forward_mode, &ExtractionWay::set_forward_mode) diff --git a/profiles/car.lua b/profiles/car.lua index 6c2b5b7ddbd..52b11fd2fd4 100644 --- a/profiles/car.lua +++ b/profiles/car.lua @@ -10,7 +10,6 @@ access_tag_restricted = { ["destination"] = true, ["delivery"] = true } access_tags = { "motorcar", "motor_vehicle", "vehicle" } access_tags_hierachy = { "motorcar", "motor_vehicle", "vehicle", "access" } service_tag_restricted = { ["parking_aisle"] = true } -ignore_in_grid = { ["ferry"] = true } restriction_exception_tags = { "motorcar", "motor_vehicle", "vehicle" } speed_profile = { @@ -393,11 +392,6 @@ function way_function (way, result) result.backward_speed = maxspeed_backward end - -- Override general direction settings of there is a specific one for our mode of travel - if ignore_in_grid[highway] then - result.ignore_in_grid = true - end - local width = math.huge local lanes = math.huge if result.forward_speed > 0 or result.backward_speed > 0 then diff --git a/unit_tests/algorithms/graph_compressor.cpp b/unit_tests/algorithms/graph_compressor.cpp index 0b09752a6d5..54063187989 100644 --- a/unit_tests/algorithms/graph_compressor.cpp +++ b/unit_tests/algorithms/graph_compressor.cpp @@ -46,15 +46,15 @@ BOOST_AUTO_TEST_CASE(long_road_test) using InputEdge = NodeBasedDynamicGraph::InputEdge; std::vector edges = { - // source, target, distance, edgeBasedNodeID, nameID, isAccessRestricted, shortcut, forward, backward, roundabout, ignore_in_grid, travel_mode - {0, 1, 1, SPECIAL_EDGEID, 0, false, false, true, true, false, false, TRAVEL_MODE_DEFAULT}, - {1, 0, 1, SPECIAL_EDGEID, 0, false, false, true, true, false, false, TRAVEL_MODE_DEFAULT}, - {1, 2, 1, SPECIAL_EDGEID, 0, false, false, true, true, false, false, TRAVEL_MODE_DEFAULT}, - {2, 1, 1, SPECIAL_EDGEID, 0, false, false, true, true, false, false, TRAVEL_MODE_DEFAULT}, - {2, 3, 1, SPECIAL_EDGEID, 0, false, false, true, true, false, false, TRAVEL_MODE_DEFAULT}, - {3, 2, 1, SPECIAL_EDGEID, 0, false, false, true, true, false, false, TRAVEL_MODE_DEFAULT}, - {3, 4, 1, SPECIAL_EDGEID, 0, false, false, true, true, false, false, TRAVEL_MODE_DEFAULT}, - {4, 3, 1, SPECIAL_EDGEID, 0, false, false, true, true, false, false, TRAVEL_MODE_DEFAULT} + // source, target, distance, edge_id, name_id, access_restricted, forward, backward, roundabout, travel_mode + {0, 1, 1, SPECIAL_EDGEID, 0, false, true, true, false, TRAVEL_MODE_DEFAULT}, + {1, 0, 1, SPECIAL_EDGEID, 0, false, true, true, false, TRAVEL_MODE_DEFAULT}, + {1, 2, 1, SPECIAL_EDGEID, 0, false, true, true, false, TRAVEL_MODE_DEFAULT}, + {2, 1, 1, SPECIAL_EDGEID, 0, false, true, true, false, TRAVEL_MODE_DEFAULT}, + {2, 3, 1, SPECIAL_EDGEID, 0, false, true, true, false, TRAVEL_MODE_DEFAULT}, + {3, 2, 1, SPECIAL_EDGEID, 0, false, true, true, false, TRAVEL_MODE_DEFAULT}, + {3, 4, 1, SPECIAL_EDGEID, 0, false, true, true, false, TRAVEL_MODE_DEFAULT}, + {4, 3, 1, SPECIAL_EDGEID, 0, false, true, true, false, TRAVEL_MODE_DEFAULT} }; BOOST_ASSERT(edges[0].data.IsCompatibleTo(edges[2].data)); @@ -88,19 +88,19 @@ BOOST_AUTO_TEST_CASE(loop_test) using InputEdge = NodeBasedDynamicGraph::InputEdge; std::vector edges = { - // source, target, distance, edgeBasedNodeID, nameID, isAccessRestricted, shortcut, forward, backward, roundabout, ignore_in_grid, travel_mode - {0, 1, 1, SPECIAL_EDGEID, 0, false, false, true, true, false, false, TRAVEL_MODE_DEFAULT}, - {0, 5, 1, SPECIAL_EDGEID, 0, false, false, true, true, false, false, TRAVEL_MODE_DEFAULT}, - {1, 0, 1, SPECIAL_EDGEID, 0, false, false, true, true, false, false, TRAVEL_MODE_DEFAULT}, - {1, 2, 1, SPECIAL_EDGEID, 0, false, false, true, true, false, false, TRAVEL_MODE_DEFAULT}, - {2, 1, 1, SPECIAL_EDGEID, 0, false, false, true, true, false, false, TRAVEL_MODE_DEFAULT}, - {2, 3, 1, SPECIAL_EDGEID, 0, false, false, true, true, false, false, TRAVEL_MODE_DEFAULT}, - {3, 2, 1, SPECIAL_EDGEID, 0, false, false, true, true, false, false, TRAVEL_MODE_DEFAULT}, - {3, 4, 1, SPECIAL_EDGEID, 0, false, false, true, true, false, false, TRAVEL_MODE_DEFAULT}, - {4, 3, 1, SPECIAL_EDGEID, 0, false, false, true, true, false, false, TRAVEL_MODE_DEFAULT}, - {4, 5, 1, SPECIAL_EDGEID, 0, false, false, true, true, false, false, TRAVEL_MODE_DEFAULT}, - {5, 0, 1, SPECIAL_EDGEID, 0, false, false, true, true, false, false, TRAVEL_MODE_DEFAULT}, - {5, 4, 1, SPECIAL_EDGEID, 0, false, false, true, true, false, false, TRAVEL_MODE_DEFAULT}, + // source, target, distance, edge_id, name_id, access_restricted, forward, backward, roundabout, travel_mode + {0, 1, 1, SPECIAL_EDGEID, 0, false, true, true, false, TRAVEL_MODE_DEFAULT}, + {0, 5, 1, SPECIAL_EDGEID, 0, false, true, true, false, TRAVEL_MODE_DEFAULT}, + {1, 0, 1, SPECIAL_EDGEID, 0, false, true, true, false, TRAVEL_MODE_DEFAULT}, + {1, 2, 1, SPECIAL_EDGEID, 0, false, true, true, false, TRAVEL_MODE_DEFAULT}, + {2, 1, 1, SPECIAL_EDGEID, 0, false, true, true, false, TRAVEL_MODE_DEFAULT}, + {2, 3, 1, SPECIAL_EDGEID, 0, false, true, true, false, TRAVEL_MODE_DEFAULT}, + {3, 2, 1, SPECIAL_EDGEID, 0, false, true, true, false, TRAVEL_MODE_DEFAULT}, + {3, 4, 1, SPECIAL_EDGEID, 0, false, true, true, false, TRAVEL_MODE_DEFAULT}, + {4, 3, 1, SPECIAL_EDGEID, 0, false, true, true, false, TRAVEL_MODE_DEFAULT}, + {4, 5, 1, SPECIAL_EDGEID, 0, false, true, true, false, TRAVEL_MODE_DEFAULT}, + {5, 0, 1, SPECIAL_EDGEID, 0, false, true, true, false, TRAVEL_MODE_DEFAULT}, + {5, 4, 1, SPECIAL_EDGEID, 0, false, true, true, false, TRAVEL_MODE_DEFAULT}, }; BOOST_ASSERT(edges.size() == 12); @@ -145,13 +145,13 @@ BOOST_AUTO_TEST_CASE(t_intersection) using InputEdge = NodeBasedDynamicGraph::InputEdge; std::vector edges = { - // source, target, distance, edgeBasedNodeID, nameID, isAccessRestricted, shortcut, forward, backward, roundabout, ignore_in_grid, travel_mode - {0, 1, 1, SPECIAL_EDGEID, 0, false, false, true, true, false, false, TRAVEL_MODE_DEFAULT}, - {1, 0, 1, SPECIAL_EDGEID, 0, false, false, true, true, false, false, TRAVEL_MODE_DEFAULT}, - {1, 2, 1, SPECIAL_EDGEID, 0, false, false, true, true, false, false, TRAVEL_MODE_DEFAULT}, - {1, 3, 1, SPECIAL_EDGEID, 0, false, false, true, true, false, false, TRAVEL_MODE_DEFAULT}, - {2, 1, 1, SPECIAL_EDGEID, 0, false, false, true, true, false, false, TRAVEL_MODE_DEFAULT}, - {3, 1, 1, SPECIAL_EDGEID, 0, false, false, true, true, false, false, TRAVEL_MODE_DEFAULT}, + // source, target, distance, edge_id, name_id, access_restricted, forward, backward, roundabout, travel_mode + {0, 1, 1, SPECIAL_EDGEID, 0, false, true, true, false, TRAVEL_MODE_DEFAULT}, + {1, 0, 1, SPECIAL_EDGEID, 0, false, true, true, false, TRAVEL_MODE_DEFAULT}, + {1, 2, 1, SPECIAL_EDGEID, 0, false, true, true, false, TRAVEL_MODE_DEFAULT}, + {1, 3, 1, SPECIAL_EDGEID, 0, false, true, true, false, TRAVEL_MODE_DEFAULT}, + {2, 1, 1, SPECIAL_EDGEID, 0, false, true, true, false, TRAVEL_MODE_DEFAULT}, + {3, 1, 1, SPECIAL_EDGEID, 0, false, true, true, false, TRAVEL_MODE_DEFAULT}, }; BOOST_ASSERT(edges[0].data.IsCompatibleTo(edges[1].data)); @@ -183,11 +183,11 @@ BOOST_AUTO_TEST_CASE(street_name_changes) using InputEdge = NodeBasedDynamicGraph::InputEdge; std::vector edges = { - // source, target, distance, edgeBasedNodeID, nameID, isAccessRestricted, shortcut, forward, backward, roundabout, ignore_in_grid, travel_mode - {0, 1, 1, SPECIAL_EDGEID, 0, false, false, true, true, false, false, TRAVEL_MODE_DEFAULT}, - {1, 0, 1, SPECIAL_EDGEID, 0, false, false, true, true, false, false, TRAVEL_MODE_DEFAULT}, - {1, 2, 1, SPECIAL_EDGEID, 1, false, false, true, true, false, false, TRAVEL_MODE_DEFAULT}, - {2, 1, 1, SPECIAL_EDGEID, 1, false, false, true, true, false, false, TRAVEL_MODE_DEFAULT}, + // source, target, distance, edge_id, name_id, access_restricted, forward, backward, roundabout, travel_mode + {0, 1, 1, SPECIAL_EDGEID, 0, false, true, true, false, TRAVEL_MODE_DEFAULT}, + {1, 0, 1, SPECIAL_EDGEID, 0, false, true, true, false, TRAVEL_MODE_DEFAULT}, + {1, 2, 1, SPECIAL_EDGEID, 1, false, true, true, false, TRAVEL_MODE_DEFAULT}, + {2, 1, 1, SPECIAL_EDGEID, 1, false, true, true, false, TRAVEL_MODE_DEFAULT}, }; BOOST_ASSERT(edges[0].data.IsCompatibleTo(edges[1].data)); @@ -215,11 +215,11 @@ BOOST_AUTO_TEST_CASE(direction_changes) using InputEdge = NodeBasedDynamicGraph::InputEdge; std::vector edges = { - // source, target, distance, edgeBasedNodeID, nameID, isAccessRestricted, shortcut, forward, backward, roundabout, ignore_in_grid, travel_mode - {0, 1, 1, SPECIAL_EDGEID, 0, false, false, true, false, false, false, TRAVEL_MODE_DEFAULT}, - {1, 0, 1, SPECIAL_EDGEID, 0, false, false, false, true, false, false, TRAVEL_MODE_DEFAULT}, - {1, 2, 1, SPECIAL_EDGEID, 0, false, false, true, true, false, false, TRAVEL_MODE_DEFAULT}, - {2, 1, 1, SPECIAL_EDGEID, 0, false, false, true, true, false, false, TRAVEL_MODE_DEFAULT}, + // source, target, distance, edge_id, name_id, access_restricted, forward, backward, roundabout, travel_mode + {0, 1, 1, SPECIAL_EDGEID, 0, false, true, false, false, TRAVEL_MODE_DEFAULT}, + {1, 0, 1, SPECIAL_EDGEID, 0, false, false, true, false, TRAVEL_MODE_DEFAULT}, + {1, 2, 1, SPECIAL_EDGEID, 0, false, true, true, false, TRAVEL_MODE_DEFAULT}, + {2, 1, 1, SPECIAL_EDGEID, 0, false, true, true, false, TRAVEL_MODE_DEFAULT}, }; NodeBasedDynamicGraph graph(5, edges); From 4a7451682bc84a5b373600a6df658c490ea96ed3 Mon Sep 17 00:00:00 2001 From: Patrick Niklaus Date: Mon, 29 Jun 2015 00:34:15 +0200 Subject: [PATCH 005/122] Fix data_structure test thanks to new assertion --- unit_tests/data_structures/dynamic_graph.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/unit_tests/data_structures/dynamic_graph.cpp b/unit_tests/data_structures/dynamic_graph.cpp index 27dc7dc68f2..5082278f0c0 100644 --- a/unit_tests/data_structures/dynamic_graph.cpp +++ b/unit_tests/data_structures/dynamic_graph.cpp @@ -59,9 +59,9 @@ BOOST_AUTO_TEST_CASE(find_test) std::vector input_edges = { TestInputEdge{0, 1, TestData{1}}, TestInputEdge{3, 0, TestData{2}}, + TestInputEdge{3, 0, TestData{5}}, TestInputEdge{3, 4, TestData{3}}, - TestInputEdge{4, 3, TestData{4}}, - TestInputEdge{3, 0, TestData{5}} + TestInputEdge{4, 3, TestData{4}} }; TestDynamicGraph simple_graph(5, input_edges); From 021a1c7a39fc37e0e49e3427e3786d0dc14dd06f Mon Sep 17 00:00:00 2001 From: Patrick Niklaus Date: Mon, 29 Jun 2015 00:42:22 +0200 Subject: [PATCH 006/122] Restructure the construction of the undirected graph --- algorithms/graph_compressor.cpp | 5 +- contractor/edge_based_graph_factory.cpp | 20 +-- contractor/processing_chain.cpp | 2 +- data_structures/dynamic_graph.hpp | 2 +- data_structures/node_based_graph.hpp | 181 +++------------------ unit_tests/algorithms/graph_compressor.cpp | 93 +++++------ util/graph_utils.hpp | 94 +++++++++++ 7 files changed, 168 insertions(+), 229 deletions(-) create mode 100644 util/graph_utils.hpp diff --git a/algorithms/graph_compressor.cpp b/algorithms/graph_compressor.cpp index c924360b653..d375a588bde 100644 --- a/algorithms/graph_compressor.cpp +++ b/algorithms/graph_compressor.cpp @@ -6,6 +6,8 @@ #include "../data_structures/restriction_map.hpp" #include "../data_structures/percent.hpp" +#include "../util/simple_logger.hpp" + GraphCompressor::GraphCompressor(const SpeedProfileProperties& speed_profile) : speed_profile(speed_profile) { @@ -59,8 +61,7 @@ void GraphCompressor::Compress(const std::unordered_set& barrier_nodes, // // If the edges are compatible. - const bool reverse_edge_order = - !(graph.GetEdgeData(graph.BeginEdges(node_v)).forward); + const bool reverse_edge_order = graph.GetEdgeData(graph.BeginEdges(node_v)).reversed; const EdgeID forward_e2 = graph.BeginEdges(node_v) + reverse_edge_order; BOOST_ASSERT(SPECIAL_EDGEID != forward_e2); BOOST_ASSERT(forward_e2 >= graph.BeginEdges(node_v) && diff --git a/contractor/edge_based_graph_factory.cpp b/contractor/edge_based_graph_factory.cpp index 76880ce9e21..35f2f30742e 100644 --- a/contractor/edge_based_graph_factory.cpp +++ b/contractor/edge_based_graph_factory.cpp @@ -180,20 +180,20 @@ void EdgeBasedGraphFactory::InsertEdgeBasedNode(const NodeID node_u, if (forward_data.edge_id != SPECIAL_NODEID) { - BOOST_ASSERT(forward_data.forward); + BOOST_ASSERT(!forward_data.reversed); } else { - BOOST_ASSERT(!forward_data.forward); + BOOST_ASSERT(forward_data.reversed); } if (reverse_data.edge_id != SPECIAL_NODEID) { - BOOST_ASSERT(reverse_data.forward); + BOOST_ASSERT(!reverse_data.reversed); } else { - BOOST_ASSERT(!reverse_data.forward); + BOOST_ASSERT(reverse_data.reversed); } BOOST_ASSERT(forward_data.edge_id != SPECIAL_NODEID || @@ -253,8 +253,8 @@ void EdgeBasedGraphFactory::RenumberEdges() { EdgeData &edge_data = m_node_based_graph->GetEdgeData(current_edge); - // this edge is an incoming edge - if (!edge_data.forward) + // only number incoming edges + if (edge_data.reversed) { continue; } @@ -379,7 +379,7 @@ void EdgeBasedGraphFactory::GenerateEdgeExpandedEdges( progress.printStatus(node_u); for (const EdgeID e1 : m_node_based_graph->GetAdjacentEdgeRange(node_u)) { - if (!m_node_based_graph->GetEdgeData(e1).forward) + if (m_node_based_graph->GetEdgeData(e1).reversed) { continue; } @@ -392,7 +392,7 @@ void EdgeBasedGraphFactory::GenerateEdgeExpandedEdges( for (const EdgeID e2 : m_node_based_graph->GetAdjacentEdgeRange(node_v)) { - if (!m_node_based_graph->GetEdgeData(e2).forward) + if (m_node_based_graph->GetEdgeData(e2).reversed) { continue; } @@ -439,8 +439,8 @@ void EdgeBasedGraphFactory::GenerateEdgeExpandedEdges( const EdgeData &edge_data2 = m_node_based_graph->GetEdgeData(e2); BOOST_ASSERT(edge_data1.edge_id != edge_data2.edge_id); - BOOST_ASSERT(edge_data1.forward); - BOOST_ASSERT(edge_data2.forward); + BOOST_ASSERT(!edge_data1.reversed); + BOOST_ASSERT(!edge_data2.reversed); // the following is the core of the loop. unsigned distance = edge_data1.distance; diff --git a/contractor/processing_chain.cpp b/contractor/processing_chain.cpp index 461d38afe88..f9ea4c1b3b4 100644 --- a/contractor/processing_chain.cpp +++ b/contractor/processing_chain.cpp @@ -355,7 +355,7 @@ Prepare::LoadNodeBasedGraph(std::unordered_set &barrier_nodes, return std::shared_ptr(); } - return NodeBasedDynamicGraphFromImportEdges(number_of_node_based_nodes, edge_list); + return NodeBasedDynamicGraphFromEdges(number_of_node_based_nodes, edge_list); } diff --git a/data_structures/dynamic_graph.hpp b/data_structures/dynamic_graph.hpp index f96eb19927d..b8c3e42a80f 100644 --- a/data_structures/dynamic_graph.hpp +++ b/data_structures/dynamic_graph.hpp @@ -138,7 +138,7 @@ template class DynamicGraph unsigned degree = 0; for (const auto edge : osrm::irange(BeginEdges(n), EndEdges(n))) { - if (GetEdgeData(edge).forward) + if (!GetEdgeData(edge).reversed) { ++degree; } diff --git a/data_structures/node_based_graph.hpp b/data_structures/node_based_graph.hpp index 60d05d15bb6..efdacb9984f 100644 --- a/data_structures/node_based_graph.hpp +++ b/data_structures/node_based_graph.hpp @@ -30,7 +30,7 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "dynamic_graph.hpp" #include "import_edge.hpp" -#include "../util/simple_logger.hpp" +#include "../util/graph_utils.hpp" #include @@ -41,16 +41,16 @@ struct NodeBasedEdgeData NodeBasedEdgeData() : distance(INVALID_EDGE_WEIGHT), edge_id(SPECIAL_NODEID), name_id(std::numeric_limits::max()), access_restricted(false), - forward(false), backward(false), roundabout(false), travel_mode(TRAVEL_MODE_INACCESSIBLE) + reversed(false), roundabout(false), travel_mode(TRAVEL_MODE_INACCESSIBLE) { } NodeBasedEdgeData(int distance, unsigned edge_id, unsigned name_id, - bool access_restricted, bool forward, bool backward, + bool access_restricted, bool reversed, bool roundabout, TravelMode travel_mode) - : distance(distance), edge_id(edge_id), - name_id(name_id), access_restricted(access_restricted), - forward(forward), backward(backward), roundabout(roundabout), travel_mode(travel_mode) + : distance(distance), edge_id(edge_id), name_id(name_id), + access_restricted(access_restricted), reversed(reversed), + roundabout(roundabout), travel_mode(travel_mode) { } @@ -58,180 +58,43 @@ struct NodeBasedEdgeData unsigned edge_id; unsigned name_id; bool access_restricted : 1; - bool forward : 1; - bool backward : 1; + bool reversed : 1; bool roundabout : 1; TravelMode travel_mode : 4; - void SwapDirectionFlags() - { - bool temp_flag = forward; - forward = backward; - backward = temp_flag; - } - bool IsCompatibleTo(const NodeBasedEdgeData &other) const { - return (forward == other.forward) && (backward == other.backward) && - (name_id == other.name_id) && (travel_mode == other.travel_mode); + return (reversed == other.reversed) && (name_id == other.name_id) && + (travel_mode == other.travel_mode); } }; using NodeBasedDynamicGraph = DynamicGraph; -inline bool validateNeighborHood(const NodeBasedDynamicGraph& graph, const NodeID source) -{ - for (auto edge = graph.BeginEdges(source); edge < graph.EndEdges(source); ++edge) - { - const auto& data = graph.GetEdgeData(edge); - if (!data.forward && !data.backward) - { - SimpleLogger().Write(logWARNING) << "Invalid edge directions"; - return false; - } - - auto target = graph.GetTarget(edge); - if (target == SPECIAL_NODEID) - { - SimpleLogger().Write(logWARNING) << "Invalid edge target"; - return false; - } - - bool found_reverse = false; - for (auto rev_edge = graph.BeginEdges(target); rev_edge < graph.EndEdges(target); ++rev_edge) - { - auto rev_target = graph.GetTarget(rev_edge); - if (rev_target == SPECIAL_NODEID) - { - SimpleLogger().Write(logWARNING) << "Invalid reverse edge target"; - return false; - } - - if (rev_target != source) - { - continue; - } - - if (found_reverse) - { - SimpleLogger().Write(logWARNING) << "Found more than one reverse edge"; - return false; - } - - const auto& rev_data = graph.GetEdgeData(rev_edge); - - // edge is incoming, this must be an outgoing edge - if (data.backward && !rev_data.forward) - { - SimpleLogger().Write(logWARNING) << "Found no outgoing edge to an incoming edge!"; - return false; - } - - // edge is bi-directional, reverse must be as well - if (data.forward && data.backward && (!rev_data.forward || !rev_data.backward)) - { - SimpleLogger().Write(logWARNING) << "Found bi-directional edge that is not bi-directional to both ends"; - return false; - } - - found_reverse = true; - - } - - if (!found_reverse) - { - SimpleLogger().Write(logWARNING) << "Could not find reverse edge"; - return false; - } - } - - return true; -} - -// This function checks if the overal graph is undirected (has an edge in each direction). -inline bool validateNodeBasedGraph(const NodeBasedDynamicGraph& graph) -{ - for (auto source = 0u; source < graph.GetNumberOfNodes(); ++source) - { - if (!validateNeighborHood(graph, source)) - { - return false; - } - } - - return true; -} - -// Factory method to create NodeBasedDynamicGraph from NodeBasedEdges -// The since DynamicGraph expects directed edges, we need to insert -// two edges for undirected edges. +/// Factory method to create NodeBasedDynamicGraph from NodeBasedEdges +/// The since DynamicGraph expects directed edges, we need to insert +/// two edges for undirected edges. inline std::shared_ptr -NodeBasedDynamicGraphFromImportEdges(int number_of_nodes, std::vector &input_edge_list) +NodeBasedDynamicGraphFromEdges(int number_of_nodes, const std::vector &input_edge_list) { - static_assert(sizeof(NodeBasedEdgeData) == 16, - "changing node based edge data size changes memory consumption"); - - DeallocatingVector edges_list; - NodeBasedDynamicGraph::InputEdge edge; - - // Since DynamicGraph assumes directed edges we have to make sure we transformed - // the compressed edge format into single directed edges. We do this to make sure - // every node also knows its incoming edges, not only its outgoing edges and use the backward=true - // flag to indicate which is which. - // - // We do the transformation in the following way: - // - // if the edge (a, b) is split: - // 1. this edge must be in only one direction, so its a --> b - // 2. there must be another directed edge b --> a somewhere in the data - // if the edge (a, b) is not split: - // 1. this edge be on of a --> b od a <-> b - // (a <-- b gets reducted to b --> a) - // 2. a --> b will be transformed to a --> b and b <-- a - // 3. a <-> b will be transformed to a <-> b and b <-> a (I think a --> b and b <-- a would work as well though) - for (const NodeBasedEdge &import_edge : input_edge_list) - { - // edges that are not forward get converted by flipping the end points - BOOST_ASSERT(import_edge.forward); - - if (import_edge.forward) + auto edges_list = directedEdgesFromCompressed(input_edge_list, + [](NodeBasedDynamicGraph::InputEdge& output_edge, const NodeBasedEdge& input_edge) { - edge.source = import_edge.source; - edge.target = import_edge.target; - edge.data.forward = import_edge.forward; - edge.data.backward = import_edge.backward; - } - - BOOST_ASSERT(edge.source != edge.target); - - edge.data.distance = static_cast(import_edge.weight); - BOOST_ASSERT(edge.data.distance > 0); - edge.data.roundabout = import_edge.roundabout; - edge.data.name_id = import_edge.name_id; - edge.data.access_restricted = import_edge.access_restricted; - edge.data.travel_mode = import_edge.travel_mode; - - edges_list.push_back(edge); + output_edge.data.distance = static_cast(input_edge.weight); + BOOST_ASSERT(output_edge.data.distance > 0); - if (!import_edge.is_split) - { - using std::swap; // enable ADL - swap(edge.source, edge.target); - edge.data.SwapDirectionFlags(); - edges_list.push_back(edge); + output_edge.data.roundabout = input_edge.roundabout; + output_edge.data.name_id = input_edge.name_id; + output_edge.data.access_restricted = input_edge.access_restricted; + output_edge.data.travel_mode = input_edge.travel_mode; } - } + ); tbb::parallel_sort(edges_list.begin(), edges_list.end()); auto graph = std::make_shared( static_cast(number_of_nodes), edges_list); - -#ifndef NDEBUG - BOOST_ASSERT(validateNodeBasedGraph(*graph)); -#endif - return graph; } diff --git a/unit_tests/algorithms/graph_compressor.cpp b/unit_tests/algorithms/graph_compressor.cpp index 54063187989..8f500eba521 100644 --- a/unit_tests/algorithms/graph_compressor.cpp +++ b/unit_tests/algorithms/graph_compressor.cpp @@ -12,25 +12,6 @@ BOOST_AUTO_TEST_SUITE(graph_compressor) -void dumpGraph(const NodeBasedDynamicGraph& graph) -{ - for (auto i = 0u; i < graph.GetNumberOfNodes(); ++i) - { - std::cout << "## node " << i << " degree: " << graph.GetOutDegree(i) << std::endl; - for (auto e = graph.BeginEdges(i); e < graph.EndEdges(i); ++e) - { - const auto& data = graph.GetEdgeData(e); - auto target = graph.GetTarget(e); - if (data.forward && !data.backward) - std::cout << i << "->" << target << std::endl; - if (!data.forward && data.backward) - std::cout << i << "<-" << target << std::endl; - if (data.forward && data.backward) - std::cout << i << "--" << target << std::endl; - } - } -} - BOOST_AUTO_TEST_CASE(long_road_test) { // @@ -46,15 +27,15 @@ BOOST_AUTO_TEST_CASE(long_road_test) using InputEdge = NodeBasedDynamicGraph::InputEdge; std::vector edges = { - // source, target, distance, edge_id, name_id, access_restricted, forward, backward, roundabout, travel_mode - {0, 1, 1, SPECIAL_EDGEID, 0, false, true, true, false, TRAVEL_MODE_DEFAULT}, - {1, 0, 1, SPECIAL_EDGEID, 0, false, true, true, false, TRAVEL_MODE_DEFAULT}, - {1, 2, 1, SPECIAL_EDGEID, 0, false, true, true, false, TRAVEL_MODE_DEFAULT}, - {2, 1, 1, SPECIAL_EDGEID, 0, false, true, true, false, TRAVEL_MODE_DEFAULT}, - {2, 3, 1, SPECIAL_EDGEID, 0, false, true, true, false, TRAVEL_MODE_DEFAULT}, - {3, 2, 1, SPECIAL_EDGEID, 0, false, true, true, false, TRAVEL_MODE_DEFAULT}, - {3, 4, 1, SPECIAL_EDGEID, 0, false, true, true, false, TRAVEL_MODE_DEFAULT}, - {4, 3, 1, SPECIAL_EDGEID, 0, false, true, true, false, TRAVEL_MODE_DEFAULT} + // source, target, distance, edge_id, name_id, access_restricted, reversed, roundabout, travel_mode + {0, 1, 1, SPECIAL_EDGEID, 0, false, false, false, TRAVEL_MODE_DEFAULT}, + {1, 0, 1, SPECIAL_EDGEID, 0, false, false, false, TRAVEL_MODE_DEFAULT}, + {1, 2, 1, SPECIAL_EDGEID, 0, false, false, false, TRAVEL_MODE_DEFAULT}, + {2, 1, 1, SPECIAL_EDGEID, 0, false, false, false, TRAVEL_MODE_DEFAULT}, + {2, 3, 1, SPECIAL_EDGEID, 0, false, false, false, TRAVEL_MODE_DEFAULT}, + {3, 2, 1, SPECIAL_EDGEID, 0, false, false, false, TRAVEL_MODE_DEFAULT}, + {3, 4, 1, SPECIAL_EDGEID, 0, false, false, false, TRAVEL_MODE_DEFAULT}, + {4, 3, 1, SPECIAL_EDGEID, 0, false, false, false, TRAVEL_MODE_DEFAULT} }; BOOST_ASSERT(edges[0].data.IsCompatibleTo(edges[2].data)); @@ -89,18 +70,18 @@ BOOST_AUTO_TEST_CASE(loop_test) using InputEdge = NodeBasedDynamicGraph::InputEdge; std::vector edges = { // source, target, distance, edge_id, name_id, access_restricted, forward, backward, roundabout, travel_mode - {0, 1, 1, SPECIAL_EDGEID, 0, false, true, true, false, TRAVEL_MODE_DEFAULT}, - {0, 5, 1, SPECIAL_EDGEID, 0, false, true, true, false, TRAVEL_MODE_DEFAULT}, - {1, 0, 1, SPECIAL_EDGEID, 0, false, true, true, false, TRAVEL_MODE_DEFAULT}, - {1, 2, 1, SPECIAL_EDGEID, 0, false, true, true, false, TRAVEL_MODE_DEFAULT}, - {2, 1, 1, SPECIAL_EDGEID, 0, false, true, true, false, TRAVEL_MODE_DEFAULT}, - {2, 3, 1, SPECIAL_EDGEID, 0, false, true, true, false, TRAVEL_MODE_DEFAULT}, - {3, 2, 1, SPECIAL_EDGEID, 0, false, true, true, false, TRAVEL_MODE_DEFAULT}, - {3, 4, 1, SPECIAL_EDGEID, 0, false, true, true, false, TRAVEL_MODE_DEFAULT}, - {4, 3, 1, SPECIAL_EDGEID, 0, false, true, true, false, TRAVEL_MODE_DEFAULT}, - {4, 5, 1, SPECIAL_EDGEID, 0, false, true, true, false, TRAVEL_MODE_DEFAULT}, - {5, 0, 1, SPECIAL_EDGEID, 0, false, true, true, false, TRAVEL_MODE_DEFAULT}, - {5, 4, 1, SPECIAL_EDGEID, 0, false, true, true, false, TRAVEL_MODE_DEFAULT}, + {0, 1, 1, SPECIAL_EDGEID, 0, false, false, false, TRAVEL_MODE_DEFAULT}, + {0, 5, 1, SPECIAL_EDGEID, 0, false, false, false, TRAVEL_MODE_DEFAULT}, + {1, 0, 1, SPECIAL_EDGEID, 0, false, false, false, TRAVEL_MODE_DEFAULT}, + {1, 2, 1, SPECIAL_EDGEID, 0, false, false, false, TRAVEL_MODE_DEFAULT}, + {2, 1, 1, SPECIAL_EDGEID, 0, false, false, false, TRAVEL_MODE_DEFAULT}, + {2, 3, 1, SPECIAL_EDGEID, 0, false, false, false, TRAVEL_MODE_DEFAULT}, + {3, 2, 1, SPECIAL_EDGEID, 0, false, false, false, TRAVEL_MODE_DEFAULT}, + {3, 4, 1, SPECIAL_EDGEID, 0, false, false, false, TRAVEL_MODE_DEFAULT}, + {4, 3, 1, SPECIAL_EDGEID, 0, false, false, false, TRAVEL_MODE_DEFAULT}, + {4, 5, 1, SPECIAL_EDGEID, 0, false, false, false, TRAVEL_MODE_DEFAULT}, + {5, 0, 1, SPECIAL_EDGEID, 0, false, false, false, TRAVEL_MODE_DEFAULT}, + {5, 4, 1, SPECIAL_EDGEID, 0, false, false, false, TRAVEL_MODE_DEFAULT}, }; BOOST_ASSERT(edges.size() == 12); @@ -145,13 +126,13 @@ BOOST_AUTO_TEST_CASE(t_intersection) using InputEdge = NodeBasedDynamicGraph::InputEdge; std::vector edges = { - // source, target, distance, edge_id, name_id, access_restricted, forward, backward, roundabout, travel_mode - {0, 1, 1, SPECIAL_EDGEID, 0, false, true, true, false, TRAVEL_MODE_DEFAULT}, - {1, 0, 1, SPECIAL_EDGEID, 0, false, true, true, false, TRAVEL_MODE_DEFAULT}, - {1, 2, 1, SPECIAL_EDGEID, 0, false, true, true, false, TRAVEL_MODE_DEFAULT}, - {1, 3, 1, SPECIAL_EDGEID, 0, false, true, true, false, TRAVEL_MODE_DEFAULT}, - {2, 1, 1, SPECIAL_EDGEID, 0, false, true, true, false, TRAVEL_MODE_DEFAULT}, - {3, 1, 1, SPECIAL_EDGEID, 0, false, true, true, false, TRAVEL_MODE_DEFAULT}, + // source, target, distance, edge_id, name_id, access_restricted, reversed, roundabout, travel_mode + {0, 1, 1, SPECIAL_EDGEID, 0, false, false, false, TRAVEL_MODE_DEFAULT}, + {1, 0, 1, SPECIAL_EDGEID, 0, false, false, false, TRAVEL_MODE_DEFAULT}, + {1, 2, 1, SPECIAL_EDGEID, 0, false, false, false, TRAVEL_MODE_DEFAULT}, + {1, 3, 1, SPECIAL_EDGEID, 0, false, false, false, TRAVEL_MODE_DEFAULT}, + {2, 1, 1, SPECIAL_EDGEID, 0, false, false, false, TRAVEL_MODE_DEFAULT}, + {3, 1, 1, SPECIAL_EDGEID, 0, false, false, false, TRAVEL_MODE_DEFAULT}, }; BOOST_ASSERT(edges[0].data.IsCompatibleTo(edges[1].data)); @@ -184,10 +165,10 @@ BOOST_AUTO_TEST_CASE(street_name_changes) using InputEdge = NodeBasedDynamicGraph::InputEdge; std::vector edges = { // source, target, distance, edge_id, name_id, access_restricted, forward, backward, roundabout, travel_mode - {0, 1, 1, SPECIAL_EDGEID, 0, false, true, true, false, TRAVEL_MODE_DEFAULT}, - {1, 0, 1, SPECIAL_EDGEID, 0, false, true, true, false, TRAVEL_MODE_DEFAULT}, - {1, 2, 1, SPECIAL_EDGEID, 1, false, true, true, false, TRAVEL_MODE_DEFAULT}, - {2, 1, 1, SPECIAL_EDGEID, 1, false, true, true, false, TRAVEL_MODE_DEFAULT}, + {0, 1, 1, SPECIAL_EDGEID, 0, false, false, false, TRAVEL_MODE_DEFAULT}, + {1, 0, 1, SPECIAL_EDGEID, 0, false, false, false, TRAVEL_MODE_DEFAULT}, + {1, 2, 1, SPECIAL_EDGEID, 1, false, false, false, TRAVEL_MODE_DEFAULT}, + {2, 1, 1, SPECIAL_EDGEID, 1, false, false, false, TRAVEL_MODE_DEFAULT}, }; BOOST_ASSERT(edges[0].data.IsCompatibleTo(edges[1].data)); @@ -215,11 +196,11 @@ BOOST_AUTO_TEST_CASE(direction_changes) using InputEdge = NodeBasedDynamicGraph::InputEdge; std::vector edges = { - // source, target, distance, edge_id, name_id, access_restricted, forward, backward, roundabout, travel_mode - {0, 1, 1, SPECIAL_EDGEID, 0, false, true, false, false, TRAVEL_MODE_DEFAULT}, - {1, 0, 1, SPECIAL_EDGEID, 0, false, false, true, false, TRAVEL_MODE_DEFAULT}, - {1, 2, 1, SPECIAL_EDGEID, 0, false, true, true, false, TRAVEL_MODE_DEFAULT}, - {2, 1, 1, SPECIAL_EDGEID, 0, false, true, true, false, TRAVEL_MODE_DEFAULT}, + // source, target, distance, edge_id, name_id, access_restricted, reverse, roundabout, travel_mode + {0, 1, 1, SPECIAL_EDGEID, 0, false, false, false, TRAVEL_MODE_DEFAULT}, + {1, 0, 1, SPECIAL_EDGEID, 0, false, true, false, TRAVEL_MODE_DEFAULT}, + {1, 2, 1, SPECIAL_EDGEID, 0, false, false, false, TRAVEL_MODE_DEFAULT}, + {2, 1, 1, SPECIAL_EDGEID, 0, false, false, false, TRAVEL_MODE_DEFAULT}, }; NodeBasedDynamicGraph graph(5, edges); diff --git a/util/graph_utils.hpp b/util/graph_utils.hpp new file mode 100644 index 00000000000..9af8dbc077f --- /dev/null +++ b/util/graph_utils.hpp @@ -0,0 +1,94 @@ +#ifndef GRAPH_UTILS_HPP +#define GRAPH_UTILS_HPP + +#include "../typedefs.h" + +#include +#include + +/// This function checks if the graph (consisting of directed edges) is undirected +template +bool isUndirectedGraph(const GraphT& graph) +{ + for (auto source = 0u; source < graph.GetNumberOfNodes(); ++source) + { + for (auto edge = graph.BeginEdges(source); edge < graph.EndEdges(source); ++edge) + { + const auto& data = graph.GetEdgeData(edge); + + auto target = graph.GetTarget(edge); + BOOST_ASSERT(target != SPECIAL_NODEID); + + bool found_reverse = false; + for (auto rev_edge = graph.BeginEdges(target); rev_edge < graph.EndEdges(target); ++rev_edge) + { + auto rev_target = graph.GetTarget(rev_edge); + BOOST_ASSERT(rev_target != SPECIAL_NODEID); + + if (rev_target != source) + { + continue; + } + + BOOST_ASSERT_MSG(!found_reverse, "Found more than one reverse edge"); + found_reverse = true; + } + + if (!found_reverse) + { + return false; + } + } + } + + return true; +} + + +/// Since DynamicGraph assumes directed edges we have to make sure we transformed +/// the compressed edge format into single directed edges. We do this to make sure +/// every node also knows its incoming edges, not only its outgoing edges and use the reversed=true +/// flag to indicate which is which. +/// +/// We do the transformation in the following way: +/// +/// if the edge (a, b) is split: +/// 1. this edge must be in only one direction, so its a --> b +/// 2. there must be another directed edge b --> a somewhere in the data +/// if the edge (a, b) is not split: +/// 1. this edge be on of a --> b od a <-> b +/// (a <-- b gets reducted to b --> a) +/// 2. a --> b will be transformed to a --> b and b <-- a +/// 3. a <-> b will be transformed to a --> b and b --> a +template +std::vector directedEdgesFromCompressed(const std::vector& input_edge_list, FunctorT copy_data) +{ + std::vector output_edge_list; + + OutputEdgeT edge; + for (const auto& input_edge : input_edge_list) + { + // edges that are not forward get converted by flipping the end points + BOOST_ASSERT(input_edge.forward); + + edge.source = input_edge.source; + edge.target = input_edge.target; + edge.data.reversed = false; + + BOOST_ASSERT(edge.source != edge.target); + + copy_data(edge, input_edge); + + output_edge_list.push_back(edge); + + if (!input_edge.is_split) + { + std::swap(edge.source, edge.target); + edge.data.reversed = !input_edge.backward; + output_edge_list.push_back(edge); + } + } + + return output_edge_list; +} +#endif From 922e8a491258ffc5f07ff912a83b85eceddd12ab Mon Sep 17 00:00:00 2001 From: Patrick Niklaus Date: Wed, 1 Jul 2015 17:55:54 +0200 Subject: [PATCH 007/122] Return the correct size --- contractor/edge_based_graph_factory.cpp | 12 +++++++-- contractor/edge_based_graph_factory.hpp | 5 +++- contractor/processing_chain.cpp | 34 +++++++++++++------------ contractor/processing_chain.hpp | 2 +- 4 files changed, 33 insertions(+), 20 deletions(-) diff --git a/contractor/edge_based_graph_factory.cpp b/contractor/edge_based_graph_factory.cpp index 35f2f30742e..714ca4bdfce 100644 --- a/contractor/edge_based_graph_factory.cpp +++ b/contractor/edge_based_graph_factory.cpp @@ -77,6 +77,11 @@ void EdgeBasedGraphFactory::GetEdgeBasedNodes(std::vector &nodes) nodes.swap(m_edge_based_node_list); } +unsigned EdgeBasedGraphFactory::GetHighestEdgeID() +{ + return m_max_edge_id; +} + void EdgeBasedGraphFactory::InsertEdgeBasedNode(const NodeID node_u, const NodeID node_v, const unsigned component_id) @@ -223,7 +228,7 @@ void EdgeBasedGraphFactory::Run(const std::string &original_edge_data_filename, lua_State *lua_state) { TIMER_START(renumber); - RenumberEdges(); + m_max_edge_id = RenumberEdges() - 1; TIMER_STOP(renumber); TIMER_START(generate_nodes); @@ -243,7 +248,8 @@ void EdgeBasedGraphFactory::Run(const std::string &original_edge_data_filename, /// Renumbers all _forward_ edges and sets the edge_id. /// A specific numbering is not important. Any unique ID will do. -void EdgeBasedGraphFactory::RenumberEdges() +/// Returns the number of edge based nodes. +unsigned EdgeBasedGraphFactory::RenumberEdges() { // renumber edge based node of outgoing edges unsigned numbered_edges_count = 0; @@ -266,6 +272,8 @@ void EdgeBasedGraphFactory::RenumberEdges() BOOST_ASSERT(SPECIAL_NODEID != edge_data.edge_id); } } + + return numbered_edges_count; } /// Creates the nodes in the edge expanded graph from edges in the node-based graph. diff --git a/contractor/edge_based_graph_factory.hpp b/contractor/edge_based_graph_factory.hpp index e4829737ca9..29f8bf77c81 100644 --- a/contractor/edge_based_graph_factory.hpp +++ b/contractor/edge_based_graph_factory.hpp @@ -74,6 +74,8 @@ class EdgeBasedGraphFactory void GetEdgeBasedNodes(std::vector &nodes); + unsigned GetHighestEdgeID(); + TurnInstruction AnalyzeTurn(const NodeID u, const NodeID v, const NodeID w, const double angle) const; int GetTurnPenalty(double angle, lua_State *lua_state) const; @@ -83,6 +85,7 @@ class EdgeBasedGraphFactory std::vector m_edge_based_node_list; DeallocatingVector m_edge_based_edge_list; + unsigned m_max_edge_id; const std::vector& m_node_info_list; std::shared_ptr m_node_based_graph; @@ -95,7 +98,7 @@ class EdgeBasedGraphFactory SpeedProfileProperties speed_profile; void CompressGeometry(); - void RenumberEdges(); + unsigned RenumberEdges(); void GenerateEdgeExpandedNodes(); void GenerateEdgeExpandedEdges(const std::string &original_edge_data_filename, lua_State *lua_state); diff --git a/contractor/processing_chain.cpp b/contractor/processing_chain.cpp index f9ea4c1b3b4..7c6db539025 100644 --- a/contractor/processing_chain.cpp +++ b/contractor/processing_chain.cpp @@ -87,7 +87,7 @@ int Prepare::Run() *node_based_edge_list, edge_based_edge_list); auto number_of_node_based_nodes = graph_size.first; - auto number_of_edge_based_nodes = graph_size.second; + auto max_edge_id = graph_size.second; TIMER_STOP(expansion); @@ -105,12 +105,12 @@ int Prepare::Run() TIMER_START(contraction); auto contracted_edge_list = osrm::make_unique>(); - ContractGraph(number_of_edge_based_nodes, edge_based_edge_list, *contracted_edge_list); + ContractGraph(max_edge_id, edge_based_edge_list, *contracted_edge_list); TIMER_STOP(contraction); SimpleLogger().Write() << "Contraction took " << TIMER_SEC(contraction) << " sec"; - std::size_t number_of_used_edges = WriteContractedGraph(number_of_edge_based_nodes, + std::size_t number_of_used_edges = WriteContractedGraph(max_edge_id, std::move(node_based_edge_list), std::move(contracted_edge_list)); @@ -119,10 +119,10 @@ int Prepare::Run() SimpleLogger().Write() << "Preprocessing : " << TIMER_SEC(preparing) << " seconds"; SimpleLogger().Write() << "Expansion : " << (number_of_node_based_nodes / TIMER_SEC(expansion)) << " nodes/sec and " - << (number_of_edge_based_nodes / TIMER_SEC(expansion)) << " edges/sec"; + << ((max_edge_id+1) / TIMER_SEC(expansion)) << " edges/sec"; SimpleLogger().Write() << "Contraction: " - << (number_of_edge_based_nodes / TIMER_SEC(contraction)) + << ((max_edge_id+1) / TIMER_SEC(contraction)) << " nodes/sec and " << number_of_used_edges / TIMER_SEC(contraction) << " edges/sec"; @@ -131,7 +131,7 @@ int Prepare::Run() return 0; } -std::size_t Prepare::WriteContractedGraph(unsigned number_of_edge_based_nodes, +std::size_t Prepare::WriteContractedGraph(unsigned max_node_id, std::unique_ptr> node_based_edge_list, std::unique_ptr> contracted_edge_list) { @@ -146,7 +146,7 @@ std::size_t Prepare::WriteContractedGraph(unsigned number_of_edge_based_nodes, const FingerPrint fingerprint = FingerPrint::GetValid(); boost::filesystem::ofstream hsgr_output_stream(config.graph_output_path, std::ios::binary); hsgr_output_stream.write((char *)&fingerprint, sizeof(FingerPrint)); - const unsigned max_used_node_id = 1 + [&contracted_edge_list] + const unsigned max_used_node_id = [&contracted_edge_list] { unsigned tmp_max = 0; for (const QueryEdge &edge : *contracted_edge_list) @@ -159,11 +159,12 @@ std::size_t Prepare::WriteContractedGraph(unsigned number_of_edge_based_nodes, return tmp_max; }(); - SimpleLogger().Write(logDEBUG) << "input graph has " << number_of_edge_based_nodes << " nodes"; - SimpleLogger().Write(logDEBUG) << "contracted graph has " << max_used_node_id << " nodes"; + SimpleLogger().Write(logDEBUG) << "input graph has " << (max_node_id+1) << " nodes"; + SimpleLogger().Write(logDEBUG) << "contracted graph has " << (max_used_node_id+1) << " nodes"; std::vector::NodeArrayEntry> node_array; - node_array.resize(number_of_edge_based_nodes + 1); + // make sure we have at least one sentinel + node_array.resize(max_node_id + 2); SimpleLogger().Write() << "Building node array"; StaticGraph::EdgeIterator edge = 0; @@ -171,7 +172,7 @@ std::size_t Prepare::WriteContractedGraph(unsigned number_of_edge_based_nodes, StaticGraph::EdgeIterator last_edge = edge; // initializing 'first_edge'-field of nodes: - for (const auto node : osrm::irange(0u, max_used_node_id)) + for (const auto node : osrm::irange(0u, max_used_node_id+1)) { last_edge = edge; while ((edge < contracted_edge_count) && ((*contracted_edge_list)[edge].source == node)) @@ -182,7 +183,7 @@ std::size_t Prepare::WriteContractedGraph(unsigned number_of_edge_based_nodes, position += edge - last_edge; // remove } - for (const auto sentinel_counter : osrm::irange(max_used_node_id, node_array.size())) + for (const auto sentinel_counter : osrm::irange(max_used_node_id+1, node_array.size())) { // sentinel element, guarded against underflow node_array[sentinel_counter].first_edge = contracted_edge_count; @@ -218,7 +219,7 @@ std::size_t Prepare::WriteContractedGraph(unsigned number_of_edge_based_nodes, current_edge.data = (*contracted_edge_list)[edge].data; // every target needs to be valid - BOOST_ASSERT(current_edge.target < max_used_node_id); + BOOST_ASSERT(current_edge.target <= max_used_node_id); #ifndef NDEBUG if (current_edge.data.distance <= 0) { @@ -399,19 +400,20 @@ Prepare::BuildEdgeExpandedGraph(std::vector &internal_to_external_nod edge_based_graph_factory.GetEdgeBasedEdges(edge_based_edge_list); edge_based_graph_factory.GetEdgeBasedNodes(node_based_edge_list); + auto max_edge_id = edge_based_graph_factory.GetHighestEdgeID(); const std::size_t number_of_node_based_nodes = node_based_graph->GetNumberOfNodes(); - return std::make_pair(number_of_node_based_nodes, node_based_edge_list.size()); + return std::make_pair(number_of_node_based_nodes, max_edge_id); } /** \brief Build contracted graph. */ -void Prepare::ContractGraph(const std::size_t number_of_edge_based_nodes, +void Prepare::ContractGraph(const unsigned max_edge_id, DeallocatingVector& edge_based_edge_list, DeallocatingVector& contracted_edge_list) { - Contractor contractor(number_of_edge_based_nodes, edge_based_edge_list); + Contractor contractor(max_edge_id + 1, edge_based_edge_list); contractor.Run(); contractor.GetEdges(contracted_edge_list); } diff --git a/contractor/processing_chain.hpp b/contractor/processing_chain.hpp index a46b963ec96..d6ef7ec0fd8 100644 --- a/contractor/processing_chain.hpp +++ b/contractor/processing_chain.hpp @@ -62,7 +62,7 @@ class Prepare void SetupScriptingEnvironment(lua_State *myLuaState, SpeedProfileProperties &speed_profile); unsigned CalculateEdgeChecksum(std::unique_ptr> node_based_edge_list); - void ContractGraph(const std::size_t number_of_edge_based_nodes, + void ContractGraph(const unsigned max_edge_id, DeallocatingVector& edge_based_edge_list, DeallocatingVector& contracted_edge_list); std::size_t WriteContractedGraph(unsigned number_of_edge_based_nodes, From f0389c0b2f092d799acb0385059d8aebbe4f3679 Mon Sep 17 00:00:00 2001 From: Patrick Niklaus Date: Wed, 8 Jul 2015 18:26:25 +0200 Subject: [PATCH 008/122] Restructure CMakeFile to fix shared library linking errors --- CMakeLists.txt | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 788d3547053..d605f5a72bd 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -63,21 +63,22 @@ set(ExtractorSources extract.cpp ${ExtractorGlob}) add_executable(osrm-extract ${ExtractorSources} $ $ $ $ $ $ $) add_library(RESTRICTION OBJECT data_structures/restriction_map.cpp) +add_library(COMPRESSEDEDGE OBJECT data_structures/compressed_edge_container.cpp) +add_library(GRAPHCOMPRESSOR OBJECT algorithms/graph_compressor.cpp) -file(GLOB PrepareGlob contractor/*.cpp data_structures/hilbert_value.cpp data_structures/compressed_edge_container.cpp algorithms/graph_compressor.cpp {RestrictionMapGlob}) +file(GLOB PrepareGlob contractor/*.cpp data_structures/hilbert_value.cpp {RestrictionMapGlob}) set(PrepareSources prepare.cpp ${PrepareGlob}) -add_executable(osrm-prepare ${PrepareSources} $ $ $ $ $ $ $ $ $) +add_executable(osrm-prepare ${PrepareSources} $ $ $ $ $ $ $ $ $ $ $) file(GLOB ServerGlob server/*.cpp) file(GLOB DescriptorGlob descriptors/*.cpp) file(GLOB DatastructureGlob data_structures/search_engine_data.cpp data_structures/route_parameters.cpp util/bearing.cpp) -list(REMOVE_ITEM DatastructureGlob data_structures/Coordinate.cpp) file(GLOB CoordinateGlob data_structures/coordinate*.cpp) -file(GLOB AlgorithmGlob algorithms/*.cpp) +file(GLOB AlgorithmGlob algorithms/polyline_compressor.cpp algorithms/polyline_formatter.cpp algorithms/douglas_peucker.cpp) file(GLOB HttpGlob server/http/*.cpp) file(GLOB LibOSRMGlob library/*.cpp) -file(GLOB DataStructureTestsGlob unit_tests/data_structures/*.cpp data_structures/hilbert_value.cpp data_structures/compressed_edge_container.cpp) -file(GLOB AlgorithmTestsGlob unit_tests/algorithms/*.cpp algorithms/graph_compressor.cpp data_structures/compressed_edge_container.cpp data_structures/restriction_map.cpp) +file(GLOB DataStructureTestsGlob unit_tests/data_structures/*.cpp data_structures/hilbert_value.cpp) +file(GLOB AlgorithmTestsGlob unit_tests/algorithms/*.cpp algorithms/graph_compressor.cpp) set( OSRMSources @@ -101,8 +102,8 @@ add_executable(osrm-routed routed.cpp ${ServerGlob} $) add_executable(osrm-datastore datastore.cpp $ $ $ $ $ $) # Unit tests -add_executable(datastructure-tests EXCLUDE_FROM_ALL unit_tests/datastructure_tests.cpp ${DataStructureTestsGlob} $ $ $ $ $) -add_executable(algorithm-tests EXCLUDE_FROM_ALL unit_tests/algorithm_tests.cpp ${AlgorithmTestsGlob} $ $ $ $) +add_executable(datastructure-tests EXCLUDE_FROM_ALL unit_tests/datastructure_tests.cpp ${DataStructureTestsGlob} $ $ $ $ $ $ $) +add_executable(algorithm-tests EXCLUDE_FROM_ALL unit_tests/algorithm_tests.cpp ${AlgorithmTestsGlob} $ $ $ $ $ $) # Benchmarks add_executable(rtree-bench EXCLUDE_FROM_ALL benchmarks/static_rtree.cpp $ $ $ $ $) From 8f4e332409e51dc9638f5108b66cc4a76d68ef63 Mon Sep 17 00:00:00 2001 From: Patrick Niklaus Date: Wed, 8 Jul 2015 20:26:54 +0200 Subject: [PATCH 009/122] Link restrictions to datastore test --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index d605f5a72bd..66b3fe829dc 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -102,7 +102,7 @@ add_executable(osrm-routed routed.cpp ${ServerGlob} $) add_executable(osrm-datastore datastore.cpp $ $ $ $ $ $) # Unit tests -add_executable(datastructure-tests EXCLUDE_FROM_ALL unit_tests/datastructure_tests.cpp ${DataStructureTestsGlob} $ $ $ $ $ $ $) +add_executable(datastructure-tests EXCLUDE_FROM_ALL unit_tests/datastructure_tests.cpp ${DataStructureTestsGlob} $ $ $ $ $ $ $ $) add_executable(algorithm-tests EXCLUDE_FROM_ALL unit_tests/algorithm_tests.cpp ${AlgorithmTestsGlob} $ $ $ $ $ $) # Benchmarks From 486d7b6d62865d9eead3185b6532a020531ab549 Mon Sep 17 00:00:00 2001 From: Patrick Niklaus Date: Thu, 9 Jul 2015 21:24:07 +0200 Subject: [PATCH 010/122] Fix typo in foot profile that removed traffic lights --- profiles/foot.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/profiles/foot.lua b/profiles/foot.lua index 4b3bc1cca78..566425f3b65 100644 --- a/profiles/foot.lua +++ b/profiles/foot.lua @@ -85,7 +85,7 @@ function node_function (node, result) -- flag node if it carries a traffic light if traffic_signal and traffic_signal == "traffic_signals" then - result.traffic_light = true + result.traffic_lights = true end -- parse access and barrier tags From 0cd3f37e1bf32cd6fdd1b086f8df4c0ab3e07d29 Mon Sep 17 00:00:00 2001 From: bergwerkgis Date: Mon, 13 Jul 2015 16:03:18 +0200 Subject: [PATCH 011/122] AppVeyor: create artifacts --- appveyor-build.bat | 45 +++++++++++++++------------------------------ build-local.bat | 3 +++ 2 files changed, 18 insertions(+), 30 deletions(-) diff --git a/appveyor-build.bat b/appveyor-build.bat index eef29b1bae2..cd7527caecb 100644 --- a/appveyor-build.bat +++ b/appveyor-build.bat @@ -7,7 +7,7 @@ ECHO platform^: %platform% SET DEPSPKG=osrm-deps-win-x64-14.0.7z :: local development -IF "%computername%"=="MB" GOTO SKIPDL +IF "%computername%"=="MBX" GOTO SKIPDL IF EXIST %DEPSPKG% DEL %DEPSPKG% IF %ERRORLEVEL% NEQ 0 GOTO ERROR @@ -60,8 +60,6 @@ msbuild OSRM.sln ^ /flp2:logfile=build_warnings.txt;warningsonly IF %ERRORLEVEL% NEQ 0 GOTO ERROR -ECHO ========= TODO^: CREATE PACKAGES ========== - CD c:\projects\osrm\build\%Configuration% IF %ERRORLEVEL% NEQ 0 GOTO ERROR @@ -74,6 +72,20 @@ ECHO running algorithm-tests.exe ... algorithm-tests.exe IF %ERRORLEVEL% NEQ 0 GOTO ERROR +IF NOT "%APPVEYOR_REPO_BRANCH%"=="develop" GOTO DONE +ECHO ========= CREATING PACKAGES ========== + +SET P=c:/projects/osrm +7z a %P%/osrm_%Configuration%.zip *.exe *.pdb %P%/libs/bin/*.dll -tzip +IF %ERRORLEVEL% NEQ 0 GOTO ERROR + +CD ..\..\profiles +IF %ERRORLEVEL% NEQ 0 GOTO ERROR + +ECHO disk=c:\temp\stxxl,10000,wincall > .stxxl.txt +7z a %P%/osrm_%Configuration%.zip * -tzip +IF %ERRORLEVEL% NEQ 0 GOTO ERROR + GOTO DONE :ERROR @@ -84,30 +96,3 @@ ECHO ============== ERROR =============== ECHO ============= DONE =============== CD C:\projects\osrm EXIT /b %EL% - - - - - - cd c:/projects/osrm - - mkdir build - - cd build - - echo Running cmake... - - call "%VS120COMNTOOLS%\..\..\VC\vcvarsall.bat" x86_amd64 - - SET PATH=C:\Program Files (x86)\MSBuild\12.0\bin\;%PATH% - - SET P=c:/projects/osrm - - set TBB_INSTALL_DIR=%P%/tbb - - set TBB_ARCH_PLATFORM=intel64/vc12 - - cmake .. -G "Visual Studio 14 Win64" -DCMAKE_BUILD_TYPE=%Configuration% -DCMAKE_INSTALL_PREFIX=%P%/libs -DBOOST_ROOT=%P%/boost_min -DBoost_ADDITIONAL_VERSIONS=1.57 -DBoost_USE_STATIC_LIBS=ON - - SET PLATFORM_TOOLSET=v140 - - SET TOOLS_VERSION=14.0 - - msbuild /p:Platform=x64 /clp:Verbosity=minimal /toolsversion:%TOOLS_VERSION% /p:PlatformToolset=%PLATFORM_TOOLSET% /nologo OSRM.sln - - msbuild /p:Platform=x64 /clp:Verbosity=minimal /toolsversion:%TOOLS_VERSION% /p:PlatformToolset=%PLATFORM_TOOLSET% /nologo tests.vcxproj - - cd %Configuration% - - if "%APPVEYOR_REPO_BRANCH%"=="develop" (7z a %P%/osrm_%Configuration%.zip *.exe *.pdb %P%/libs/bin/*.dll -tzip) - - cd ..\..\profiles - - echo disk=c:\temp\stxxl,10000,wincall > .stxxl.txt - - if "%APPVEYOR_REPO_BRANCH%"=="develop" (7z a %P%/osrm_%Configuration%.zip * -tzip) - - set PATH=%PATH%;c:/projects/osrm/libs/bin - - cd c:/projects/osrm/build/%Configuration% - - datastructure-tests.exe - - algorithm-tests.exe diff --git a/build-local.bat b/build-local.bat index 47a1fb3a61e..df18fc4447f 100644 --- a/build-local.bat +++ b/build-local.bat @@ -6,6 +6,9 @@ SET CONFIGURATION=Release WHERE msbuild IF %ERRORLEVEL% EQU 0 GOTO RUNBUILD +FOR /F "tokens=*" %%i in ('git rev-parse --abbrev-ref HEAD') do SET APPVEYOR_REPO_BRANCH=%%i +ECHO APPVEYOR_REPO_BRANCH^: %APPVEYOR_REPO_BRANCH% + SET PATH=C:\mb\windows-builds-64\tmp-bin\cmake-3.1.0-win32-x86\bin;%PATH% SET PATH=C:\Program Files\7-Zip;%PATH% ECHO activating VS command prompt ... From 0352d9c99eb22b834921b9a7ebdb980436ecb0b3 Mon Sep 17 00:00:00 2001 From: bergwerkgis Date: Mon, 13 Jul 2015 14:49:30 +0000 Subject: [PATCH 012/122] AppVeyor: wrong paths when creating artifacts --- appveyor-build.bat | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/appveyor-build.bat b/appveyor-build.bat index cd7527caecb..41f15c3fd53 100644 --- a/appveyor-build.bat +++ b/appveyor-build.bat @@ -7,7 +7,7 @@ ECHO platform^: %platform% SET DEPSPKG=osrm-deps-win-x64-14.0.7z :: local development -IF "%computername%"=="MBX" GOTO SKIPDL +IF "%computername%"=="WIN-2G2NPH4S5B8" GOTO SKIPDL IF EXIST %DEPSPKG% DEL %DEPSPKG% IF %ERRORLEVEL% NEQ 0 GOTO ERROR @@ -75,15 +75,19 @@ IF %ERRORLEVEL% NEQ 0 GOTO ERROR IF NOT "%APPVEYOR_REPO_BRANCH%"=="develop" GOTO DONE ECHO ========= CREATING PACKAGES ========== -SET P=c:/projects/osrm -7z a %P%/osrm_%Configuration%.zip *.exe *.pdb %P%/libs/bin/*.dll -tzip +SET P=c:\projects\osrm +SET ZIP= %P%\osrm_%Configuration%.zip +IF EXIST %ZIP% ECHO deleting %ZIP% && DEL /F /Q %ZIP% +IF %ERRORLEVEL% NEQ 0 ECHO deleting %ZIP% FAILED && GOTO ERROR + +7z a %ZIP% *.exe *.pdb %P%/osrm-deps/libs/bin/*.dll -tzip IF %ERRORLEVEL% NEQ 0 GOTO ERROR CD ..\..\profiles IF %ERRORLEVEL% NEQ 0 GOTO ERROR ECHO disk=c:\temp\stxxl,10000,wincall > .stxxl.txt -7z a %P%/osrm_%Configuration%.zip * -tzip +7z a %ZIP% * -tzip IF %ERRORLEVEL% NEQ 0 GOTO ERROR GOTO DONE @@ -93,6 +97,6 @@ SET EL=%ERRORLEVEL% ECHO ============== ERROR =============== :DONE -ECHO ============= DONE =============== +ECHO ============== DONE ================ CD C:\projects\osrm EXIT /b %EL% From 94f44e1d5ded389a8ebd609f22078cfb40787679 Mon Sep 17 00:00:00 2001 From: Patrick Niklaus Date: Sat, 1 Aug 2015 17:46:47 +0200 Subject: [PATCH 013/122] Make sure to capture floating point return values from lua --- contractor/edge_based_graph_factory.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/contractor/edge_based_graph_factory.cpp b/contractor/edge_based_graph_factory.cpp index 714ca4bdfce..2926944e6f7 100644 --- a/contractor/edge_based_graph_factory.cpp +++ b/contractor/edge_based_graph_factory.cpp @@ -532,7 +532,8 @@ int EdgeBasedGraphFactory::GetTurnPenalty(double angle, lua_State *lua_state) co try { // call lua profile to compute turn penalty - return luabind::call_function(lua_state, "turn_function", 180. - angle); + double penalty = luabind::call_function(lua_state, "turn_function", 180. - angle); + return static_cast(penalty); } catch (const luabind::error &er) { From b526cadebd8d01ff029f4fae81676bff99107d2d Mon Sep 17 00:00:00 2001 From: MoKob Date: Mon, 13 Jul 2015 21:09:19 +0200 Subject: [PATCH 014/122] Initial version of core ch This improves preprocessing times in favour of worse query performance. Core size can be set over the --core parameater, default is the old behaviour to fully contract the graph. --- contractor/contractor.hpp | 10 +- contractor/contractor_options.cpp | 6 +- contractor/contractor_options.hpp | 6 + contractor/processing_chain.cpp | 2 +- data_structures/binary_heap.hpp | 5 + data_structures/search_engine.hpp | 3 + features/options/prepare/help.feature | 9 +- plugins/viaroute.hpp | 15 +- routing_algorithms/direct_shortest_path.hpp | 165 ++++++++++++++++++++ unit_tests/data_structures/binary_heap.cpp | 2 + 10 files changed, 211 insertions(+), 12 deletions(-) create mode 100644 routing_algorithms/direct_shortest_path.hpp diff --git a/contractor/contractor.hpp b/contractor/contractor.hpp index 4468f5e4969..cd5f8eecb0f 100644 --- a/contractor/contractor.hpp +++ b/contractor/contractor.hpp @@ -284,7 +284,7 @@ class Contractor ~Contractor() {} - void Run() + void Run( double core_factor = 1.0 ) { // for the preperation we can use a big grain size, which is much faster (probably cache) constexpr size_t InitGrainSize = 100000; @@ -333,9 +333,9 @@ class Contractor << std::flush; bool flushed_contractor = false; - while (number_of_nodes > 2 && number_of_contracted_nodes < number_of_nodes) + while (number_of_nodes > 2 && number_of_contracted_nodes < static_cast(number_of_nodes * core_factor) ) { - if (!flushed_contractor && (number_of_contracted_nodes > (number_of_nodes * 0.65))) + if (!flushed_contractor && (number_of_contracted_nodes > static_cast(number_of_nodes * 0.65 * core_factor))) { DeallocatingVector new_edge_set; // this one is not explicitely // cleared since it goes out of @@ -524,7 +524,7 @@ class Contractor // unsigned quaddegree = 0; // // for(unsigned i = 0; i < remaining_nodes.size(); ++i) { - // unsigned degree = contractor_graph->EndEdges(remaining_nodes[i].first) + // unsigned degree = contractor_graph->EndEdges(remaining_nodes[i].id) // - // contractor_graph->BeginEdges(remaining_nodes[i].first); // if(degree > maxdegree) @@ -546,6 +546,8 @@ class Contractor p.printStatus(number_of_contracted_nodes); } + SimpleLogger().Write() << "[core] " << remaining_nodes.size() << " nodes " << contractor_graph->GetNumberOfEdges() << " edges." << std::endl; + thread_data_list.data.clear(); } diff --git a/contractor/contractor_options.cpp b/contractor/contractor_options.cpp index eaa1ba997f8..cd304bc3c77 100644 --- a/contractor/contractor_options.cpp +++ b/contractor/contractor_options.cpp @@ -56,7 +56,11 @@ ContractorOptions::ParseArguments(int argc, char *argv[], ContractorConfig &cont "Path to LUA routing profile")( "threads,t", boost::program_options::value(&contractor_config.requested_num_threads) ->default_value(tbb::task_scheduler_init::default_num_threads()), - "Number of threads to use"); + "Number of threads to use")( + "core,k", boost::program_options::value(&contractor_config.core_factor) + ->default_value(1.0),"Percentage of the graph (in vertices) to contract [0.1]"); + + // hidden options, will be allowed both on command line and in config file, but will not be // shown to the user diff --git a/contractor/contractor_options.hpp b/contractor/contractor_options.hpp index 87541af9d88..248659527c8 100644 --- a/contractor/contractor_options.hpp +++ b/contractor/contractor_options.hpp @@ -56,6 +56,12 @@ struct ContractorConfig std::string rtree_leafs_output_path; unsigned requested_num_threads; + + //A percentage of vertices that will be contracted for the hierarchy. + //Offers a trade-off between preprocessing and query time. + //The remaining vertices form the core of the hierarchy + //(e.g. 0.8 contracts 80 percent of the hierarchy, leaving a core of 20%) + double core_factor; }; struct ContractorOptions diff --git a/contractor/processing_chain.cpp b/contractor/processing_chain.cpp index 7c6db539025..39005462d6a 100644 --- a/contractor/processing_chain.cpp +++ b/contractor/processing_chain.cpp @@ -414,7 +414,7 @@ void Prepare::ContractGraph(const unsigned max_edge_id, DeallocatingVector& contracted_edge_list) { Contractor contractor(max_edge_id + 1, edge_based_edge_list); - contractor.Run(); + contractor.Run(config.core_factor); contractor.GetEdges(contracted_edge_list); } diff --git a/data_structures/binary_heap.hpp b/data_structures/binary_heap.hpp index a23a6b0f516..b237486aa7a 100644 --- a/data_structures/binary_heap.hpp +++ b/data_structures/binary_heap.hpp @@ -189,6 +189,11 @@ class BinaryHeap return inserted_nodes[heap[1].index].node; } + Weight MinKey() const { + BOOST_ASSERT(heap.size() > 1); + return heap[1].weight; + } + NodeID DeleteMin() { BOOST_ASSERT(heap.size() > 1); diff --git a/data_structures/search_engine.hpp b/data_structures/search_engine.hpp index 7e47b1f5df1..5af734e77f9 100644 --- a/data_structures/search_engine.hpp +++ b/data_structures/search_engine.hpp @@ -33,6 +33,7 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "../routing_algorithms/many_to_many.hpp" #include "../routing_algorithms/map_matching.hpp" #include "../routing_algorithms/shortest_path.hpp" +#include "../routing_algorithms/direct_shortest_path.hpp" #include @@ -44,6 +45,7 @@ template class SearchEngine public: ShortestPathRouting shortest_path; + DirectShortestPathRouting direct_shortest_path; AlternativeRouting alternative_path; ManyToManyRouting distance_table; MapMatching map_matching; @@ -51,6 +53,7 @@ template class SearchEngine explicit SearchEngine(DataFacadeT *facade) : facade(facade), shortest_path(facade, engine_working_data), + direct_shortest_path(facade, engine_working_data), alternative_path(facade, engine_working_data), distance_table(facade, engine_working_data), map_matching(facade, engine_working_data) diff --git a/features/options/prepare/help.feature b/features/options/prepare/help.feature index dae522d5614..59265e968ec 100644 --- a/features/options/prepare/help.feature +++ b/features/options/prepare/help.feature @@ -16,7 +16,8 @@ Feature: osrm-prepare command line options: help And stdout should contain "--restrictions" And stdout should contain "--profile" And stdout should contain "--threads" - And stdout should contain 15 lines + And stdout should contain "--core" + And stdout should contain 17 lines And it should exit with code 1 Scenario: osrm-prepare - Help, short @@ -31,7 +32,8 @@ Feature: osrm-prepare command line options: help And stdout should contain "--restrictions" And stdout should contain "--profile" And stdout should contain "--threads" - And stdout should contain 15 lines + And stdout should contain "--core" + And stdout should contain 17 lines And it should exit with code 0 Scenario: osrm-prepare - Help, long @@ -46,5 +48,6 @@ Feature: osrm-prepare command line options: help And stdout should contain "--restrictions" And stdout should contain "--profile" And stdout should contain "--threads" - And stdout should contain 15 lines + And stdout should contain "--core" + And stdout should contain 17 lines And it should exit with code 0 diff --git a/plugins/viaroute.hpp b/plugins/viaroute.hpp index 335bda1afbd..ceaeebf2851 100644 --- a/plugins/viaroute.hpp +++ b/plugins/viaroute.hpp @@ -39,6 +39,7 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "../util/json_renderer.hpp" #include "../util/make_unique.hpp" #include "../util/simple_logger.hpp" +#include "../util/timing_util.hpp" #include @@ -153,10 +154,18 @@ template class ViaRoutePlugin final : public BasePlugin }; osrm::for_each_pair(phantom_node_pair_list, build_phantom_pairs); - if (route_parameters.alternate_route && 1 == raw_route.segment_end_coordinates.size()) + if (1 == raw_route.segment_end_coordinates.size()) { - search_engine_ptr->alternative_path(raw_route.segment_end_coordinates.front(), - raw_route); + if (route_parameters.alternate_route) + { + search_engine_ptr->alternative_path(raw_route.segment_end_coordinates.front(), + raw_route); + } + else + { + search_engine_ptr->direct_shortest_path(raw_route.segment_end_coordinates, + route_parameters.uturns, raw_route); + } } else { diff --git a/routing_algorithms/direct_shortest_path.hpp b/routing_algorithms/direct_shortest_path.hpp new file mode 100644 index 00000000000..45515f2616a --- /dev/null +++ b/routing_algorithms/direct_shortest_path.hpp @@ -0,0 +1,165 @@ +/* + +Copyright (c) 2015, Project OSRM contributors +All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + +Redistributions of source code must retain the above copyright notice, this list +of conditions and the following disclaimer. +Redistributions in binary form must reproduce the above copyright notice, this +list of conditions and the following disclaimer in the documentation and/or +other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +*/ + +#ifndef DIRECT_SHORTEST_PATH_HPP +#define DIRECT_SHORTEST_PATH_HPP + +#include + +#include "routing_base.hpp" +#include "../data_structures/search_engine_data.hpp" +#include "../util/integer_range.hpp" +#include "../util/timing_util.hpp" +#include "../typedefs.h" + +/// This is a striped down version of the general shortest path algorithm. +/// The general algorithm always computes two queries for each leg. This is only +/// necessary in case of vias, where the directions of the start node is constrainted +/// by the previous route. +/// This variation is only an optimazation for graphs with slow queries, for example +/// not fully contracted graphs. +template +class DirectShortestPathRouting final + : public BasicRoutingInterface> +{ + using super = BasicRoutingInterface>; + using QueryHeap = SearchEngineData::QueryHeap; + SearchEngineData &engine_working_data; + + public: + DirectShortestPathRouting(DataFacadeT *facade, SearchEngineData &engine_working_data) + : super(facade), engine_working_data(engine_working_data) + { + } + + ~DirectShortestPathRouting() {} + + void operator()(const std::vector &phantom_nodes_vector, + const std::vector &uturn_indicators, + InternalRouteResult &raw_route_data) const + { + engine_working_data.InitializeOrClearFirstThreadLocalStorage( + super::facade->GetNumberOfNodes()); + + QueryHeap &forward_heap = *(engine_working_data.forward_heap_1); + QueryHeap &reverse_heap = *(engine_working_data.reverse_heap_1); + + // Get distance to next pair of target nodes. + BOOST_ASSERT_MSG(1 == phantom_nodes_vector.size(), + "Direct Shortest Path Query only accepts a single source and target pair. Multiple ones have been specified."); + + const auto& phantom_node_pair = phantom_nodes_vector.front(); + + forward_heap.Clear(); + reverse_heap.Clear(); + int distance = INVALID_EDGE_WEIGHT; + NodeID middle = SPECIAL_NODEID; + + const EdgeWeight min_edge_offset = + std::min(-phantom_node_pair.source_phantom.GetForwardWeightPlusOffset(), + -phantom_node_pair.source_phantom.GetReverseWeightPlusOffset()); + + // insert new starting nodes into forward heap, adjusted by previous distances. + if (phantom_node_pair.source_phantom.forward_node_id != SPECIAL_NODEID) + { + forward_heap.Insert( + phantom_node_pair.source_phantom.forward_node_id, + -phantom_node_pair.source_phantom.GetForwardWeightPlusOffset(), + phantom_node_pair.source_phantom.forward_node_id); + } + if ( phantom_node_pair.source_phantom.reverse_node_id != SPECIAL_NODEID) + { + forward_heap.Insert( + phantom_node_pair.source_phantom.reverse_node_id, + -phantom_node_pair.source_phantom.GetReverseWeightPlusOffset(), + phantom_node_pair.source_phantom.reverse_node_id); + } + + // insert new backward nodes into backward heap, unadjusted. + if (phantom_node_pair.target_phantom.forward_node_id != SPECIAL_NODEID) + { + reverse_heap.Insert(phantom_node_pair.target_phantom.forward_node_id, + phantom_node_pair.target_phantom.GetForwardWeightPlusOffset(), + phantom_node_pair.target_phantom.forward_node_id); + } + + if (phantom_node_pair.target_phantom.reverse_node_id != SPECIAL_NODEID) + { + reverse_heap.Insert(phantom_node_pair.target_phantom.reverse_node_id, + phantom_node_pair.target_phantom.GetReverseWeightPlusOffset(), + phantom_node_pair.target_phantom.reverse_node_id); + } + + // run two-Target Dijkstra routing step. + while (0 < (forward_heap.Size() + reverse_heap.Size()) ) + { + if (!forward_heap.Empty()) + { + super::RoutingStep(forward_heap, reverse_heap, &middle, &distance, + min_edge_offset, true); + } + if (!reverse_heap.Empty()) + { + super::RoutingStep(reverse_heap, forward_heap, &middle, &distance, + min_edge_offset, false); + } + } + + // No path found for both target nodes? + if ((INVALID_EDGE_WEIGHT == distance)) + { + raw_route_data.shortest_path_length = INVALID_EDGE_WEIGHT; + raw_route_data.alternative_path_length = INVALID_EDGE_WEIGHT; + return; + } + + // Was a paths over one of the forward/reverse nodes not found? + BOOST_ASSERT_MSG((SPECIAL_NODEID == middle || INVALID_EDGE_WEIGHT != distance), + "no path found"); + + // Unpack paths if they exist + std::vector packed_leg; + if (INVALID_EDGE_WEIGHT != distance) + { + super::RetrievePackedPathFromHeap(forward_heap, reverse_heap, middle, packed_leg); + + BOOST_ASSERT_MSG(!packed_leg.empty(), "packed path empty"); + + raw_route_data.unpacked_path_segments.resize(1); + raw_route_data.source_traversed_in_reverse.push_back( + (packed_leg.front() != phantom_node_pair.source_phantom.forward_node_id)); + raw_route_data.target_traversed_in_reverse.push_back( + (packed_leg.back() != phantom_node_pair.target_phantom.forward_node_id)); + + super::UnpackPath(packed_leg, phantom_node_pair, raw_route_data.unpacked_path_segments.front()); + } + + raw_route_data.shortest_path_length = distance; + } +}; + +#endif /* DIRECT_SHORTEST_PATH_HPP */ diff --git a/unit_tests/data_structures/binary_heap.cpp b/unit_tests/data_structures/binary_heap.cpp index d039710c032..300d8984ed7 100644 --- a/unit_tests/data_structures/binary_heap.cpp +++ b/unit_tests/data_structures/binary_heap.cpp @@ -165,6 +165,7 @@ BOOST_FIXTURE_TEST_CASE_TEMPLATE(decrease_key_test, T, storage_types, RandomData { heap.DecreaseKey(id, weights[id]); BOOST_CHECK_EQUAL(heap.Min(), min_id); + BOOST_CHECK_EQUAL(heap.MinKey(), min_weight); weights[id]--; } @@ -172,6 +173,7 @@ BOOST_FIXTURE_TEST_CASE_TEMPLATE(decrease_key_test, T, storage_types, RandomData weights[id] -= 2; heap.DecreaseKey(id, weights[id]); BOOST_CHECK_EQUAL(heap.Min(), id); + BOOST_CHECK_EQUAL(heap.MinKey(), weights[id]); } } From c43c043521ecf4fa20d9ff48961c36943a4df2ab Mon Sep 17 00:00:00 2001 From: Patrick Niklaus Date: Fri, 13 Mar 2015 15:53:40 +0100 Subject: [PATCH 015/122] Add docker port of build instructions --- docker/Dockerfile | 22 ++++++++++++++++++++++ docker/run-gcc.sh | 15 +++++++++++++++ docker/test.sh | 24 ++++++++++++++++++++++++ 3 files changed, 61 insertions(+) create mode 100644 docker/Dockerfile create mode 100755 docker/run-gcc.sh create mode 100755 docker/test.sh diff --git a/docker/Dockerfile b/docker/Dockerfile new file mode 100644 index 00000000000..6ddded45513 --- /dev/null +++ b/docker/Dockerfile @@ -0,0 +1,22 @@ +FROM ubuntu:14.04 + +RUN apt-get update -y && \ +apt-get install -y build-essential git-core python-pip python-software-properties software-properties-common && \ +rm -rf /var/lib/apt/lists/* + +RUN apt-get -y install gcc-4.8 g++-4.8 curl automake gdb libtool make cmake pkg-config libboost1.55-dev llvm-3.4 \ +protobuf-compiler libprotoc-dev libprotobuf7 libprotobuf-dev libbz2-dev libstxxl-dev libstxxl1 libxml2-dev \ +libzip-dev lua5.1 liblua5.1-0-dev rubygems libtbb-dev libgdal-dev && \ +pip install awscli + +# luabind +RUN curl https://gist.githubusercontent.com/DennisOSRM/f2eb7b948e6fe1ae319e/raw/install-luabind.sh | sudo bash +# osmosis +RUN curl -s https://gist.githubusercontent.com/DennisOSRM/803a64a9178ec375069f/raw/ | sudo bash +# osmpbf library +RUN curl -s https://gist.githubusercontent.com/DennisOSRM/13b1b4fe38a57ead850e/raw/install_osmpbf.sh | sudo bash + +RUN useradd -ms /bin/bash mapbox +USER mapbox +ENV HOME /home/mapbox +WORKDIR /home/mapbox diff --git a/docker/run-gcc.sh b/docker/run-gcc.sh new file mode 100755 index 00000000000..ea07f670981 --- /dev/null +++ b/docker/run-gcc.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash + +set -e +set -o pipefail + +docker build \ + -t mapbox/osrm:linux \ + docker/ + +docker run \ + -i \ + -e "CXX=g++" \ + -v `pwd`:/home/mapbox/build \ + -t mapbox/osrm:linux \ + build/docker/test.sh diff --git a/docker/test.sh b/docker/test.sh new file mode 100755 index 00000000000..b0b6cb6b514 --- /dev/null +++ b/docker/test.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash + +set -e +set -o pipefail + +export TRAVIS_OS_NAME=linux +export CMAKEOPTIONS="-DCMAKE_BUILD_TYPE=Release -DCMAKE_CXX_COMPILER=g++-4.8" +export OSRM_PORT=5000 +export OSRM_TIMEOUT=60 + +rvm use 1.9.3 +gem install bundler +bundle install + +mkdir -p build +cd build +cmake .. $CMAKEOPTIONS -DBUILD_TOOLS=1 + +make -j`nproc` +make tests -j`nproc` +./datastructure-tests +./algorithm-tests +cd .. +cucumber -p verify From 1acde593b5e284ba0f0ce992fb6f08d876e0b280 Mon Sep 17 00:00:00 2001 From: Patrick Niklaus Date: Mon, 16 Mar 2015 12:38:52 +0100 Subject: [PATCH 016/122] Fix docker run step --- docker/Dockerfile | 16 +++++++++------- docker/run-gcc.sh | 4 ++-- docker/test.sh | 12 +++++++----- 3 files changed, 18 insertions(+), 14 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 6ddded45513..ca3028522ca 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,13 +1,15 @@ FROM ubuntu:14.04 -RUN apt-get update -y && \ -apt-get install -y build-essential git-core python-pip python-software-properties software-properties-common && \ -rm -rf /var/lib/apt/lists/* +RUN apt-get update -y +RUN apt-get install -y build-essential git-core python-pip python-software-properties software-properties-common -RUN apt-get -y install gcc-4.8 g++-4.8 curl automake gdb libtool make cmake pkg-config libboost1.55-dev llvm-3.4 \ -protobuf-compiler libprotoc-dev libprotobuf7 libprotobuf-dev libbz2-dev libstxxl-dev libstxxl1 libxml2-dev \ -libzip-dev lua5.1 liblua5.1-0-dev rubygems libtbb-dev libgdal-dev && \ -pip install awscli +RUN apt-get -y install gcc-4.8 g++-4.8 libboost1.55-all-dev llvm-3.4 +RUN apt-get -y install protobuf-compiler libprotoc-dev libprotobuf-dev +RUN apt-get -y install libbz2-dev libstxxl-dev libstxxl1 libxml2-dev +RUN apt-get -y install libzip-dev lua5.1 liblua5.1-0-dev libtbb-dev libgdal-dev ruby1.9 +RUN apt-get -y install curl cmake + +RUN pip install awscli # luabind RUN curl https://gist.githubusercontent.com/DennisOSRM/f2eb7b948e6fe1ae319e/raw/install-luabind.sh | sudo bash diff --git a/docker/run-gcc.sh b/docker/run-gcc.sh index ea07f670981..df45b42f534 100755 --- a/docker/run-gcc.sh +++ b/docker/run-gcc.sh @@ -10,6 +10,6 @@ docker build \ docker run \ -i \ -e "CXX=g++" \ - -v `pwd`:/home/mapbox/build \ + -v `pwd`:/home/mapbox/osrm-backend \ -t mapbox/osrm:linux \ - build/docker/test.sh + osrm-backend/docker/test.sh diff --git a/docker/test.sh b/docker/test.sh index b0b6cb6b514..8bbb7b20a5b 100755 --- a/docker/test.sh +++ b/docker/test.sh @@ -7,11 +7,13 @@ export TRAVIS_OS_NAME=linux export CMAKEOPTIONS="-DCMAKE_BUILD_TYPE=Release -DCMAKE_CXX_COMPILER=g++-4.8" export OSRM_PORT=5000 export OSRM_TIMEOUT=60 +export PATH=$PATH:/home/mapbox/.gem/ruby/1.9.1/bin:/home/mapbox/osrm-backend/vendor/bundle/ruby/1.9.1/bin +export BRANCH=develop -rvm use 1.9.3 -gem install bundler -bundle install - +cd /home/mapbox/osrm-backend +gem install --user-install bundler +bundle install --path vendor/bundle +[ -d build ] && rm -rf build mkdir -p build cd build cmake .. $CMAKEOPTIONS -DBUILD_TOOLS=1 @@ -21,4 +23,4 @@ make tests -j`nproc` ./datastructure-tests ./algorithm-tests cd .. -cucumber -p verify +bundle exec cucumber -p verify From 00b0ff50f3b92e0ecf2543bd513b3f1d88b275bf Mon Sep 17 00:00:00 2001 From: Patrick Niklaus Date: Mon, 16 Mar 2015 12:44:49 +0100 Subject: [PATCH 017/122] Add clang and README --- docker/README | 6 ++++++ docker/build-image.sh | 9 +++++++++ docker/run-clang.sh | 11 +++++++++++ docker/run-gcc.sh | 4 ---- 4 files changed, 26 insertions(+), 4 deletions(-) create mode 100644 docker/README create mode 100755 docker/build-image.sh create mode 100755 docker/run-clang.sh diff --git a/docker/README b/docker/README new file mode 100644 index 00000000000..7034364a37a --- /dev/null +++ b/docker/README @@ -0,0 +1,6 @@ +# Docker based continious integration + +Run ```./docker/build-image.sh``` to build a docker image. +The image contains all the build dependencies and the state of the local git repository. + +Run ```./docker/run-gcc.sh``` to build OSRM with g++ and run all tests. diff --git a/docker/build-image.sh b/docker/build-image.sh new file mode 100755 index 00000000000..ace73056a5d --- /dev/null +++ b/docker/build-image.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash + +set -e +set -o pipefail + +docker build \ + -t mapbox/osrm:linux \ + docker/ + diff --git a/docker/run-clang.sh b/docker/run-clang.sh new file mode 100755 index 00000000000..716beb6051b --- /dev/null +++ b/docker/run-clang.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash + +set -e +set -o pipefail + +docker run \ + -i \ + -e "CXX=clang++" \ + -v `pwd`:/home/mapbox/osrm-backend \ + -t mapbox/osrm:linux \ + osrm-backend/docker/test.sh diff --git a/docker/run-gcc.sh b/docker/run-gcc.sh index df45b42f534..313b0f7c5ff 100755 --- a/docker/run-gcc.sh +++ b/docker/run-gcc.sh @@ -3,10 +3,6 @@ set -e set -o pipefail -docker build \ - -t mapbox/osrm:linux \ - docker/ - docker run \ -i \ -e "CXX=g++" \ From 8b7b32e225d7f2b3d428b1a881cd44a5be4b6d39 Mon Sep 17 00:00:00 2001 From: Patrick Niklaus Date: Wed, 15 Apr 2015 18:12:04 +0200 Subject: [PATCH 018/122] Added ccmake to docker image --- docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index ca3028522ca..d7c2f9e71ef 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -7,7 +7,7 @@ RUN apt-get -y install gcc-4.8 g++-4.8 libboost1.55-all-dev llvm-3.4 RUN apt-get -y install protobuf-compiler libprotoc-dev libprotobuf-dev RUN apt-get -y install libbz2-dev libstxxl-dev libstxxl1 libxml2-dev RUN apt-get -y install libzip-dev lua5.1 liblua5.1-0-dev libtbb-dev libgdal-dev ruby1.9 -RUN apt-get -y install curl cmake +RUN apt-get -y install curl cmake cmake-curses-gui RUN pip install awscli From 43b881d0cd5259e47e789f9ae92c90f4ea536b70 Mon Sep 17 00:00:00 2001 From: Patrick Niklaus Date: Sun, 2 Aug 2015 14:45:20 +0200 Subject: [PATCH 019/122] Simplify test.sh --- docker/test.sh | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/docker/test.sh b/docker/test.sh index 8bbb7b20a5b..0cefe5dfa72 100755 --- a/docker/test.sh +++ b/docker/test.sh @@ -3,12 +3,8 @@ set -e set -o pipefail -export TRAVIS_OS_NAME=linux -export CMAKEOPTIONS="-DCMAKE_BUILD_TYPE=Release -DCMAKE_CXX_COMPILER=g++-4.8" -export OSRM_PORT=5000 -export OSRM_TIMEOUT=60 +export CMAKEOPTIONS="-DCMAKE_BUILD_TYPE=Release" export PATH=$PATH:/home/mapbox/.gem/ruby/1.9.1/bin:/home/mapbox/osrm-backend/vendor/bundle/ruby/1.9.1/bin -export BRANCH=develop cd /home/mapbox/osrm-backend gem install --user-install bundler From 3c055642d5e3e10942b9d65e9d1b8486df23c6c1 Mon Sep 17 00:00:00 2001 From: Patrick Niklaus Date: Wed, 1 Jul 2015 19:00:37 +0200 Subject: [PATCH 020/122] Remove reference to restrictions and bollard nodes because it does not work --- algorithms/tiny_components.hpp | 50 +++++-------------------- contractor/edge_based_graph_factory.cpp | 3 +- tools/components.cpp | 23 ++---------- 3 files changed, 14 insertions(+), 62 deletions(-) diff --git a/algorithms/tiny_components.hpp b/algorithms/tiny_components.hpp index 8abc1cbc689..67ccb6ef214 100644 --- a/algorithms/tiny_components.hpp +++ b/algorithms/tiny_components.hpp @@ -33,9 +33,6 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "../data_structures/import_edge.hpp" #include "../data_structures/query_node.hpp" #include "../data_structures/percent.hpp" -#include "../data_structures/restriction.hpp" -#include "../data_structures/restriction_map.hpp" -#include "../data_structures/turn_instructions.hpp" #include "../util/integer_range.hpp" #include "../util/simple_logger.hpp" @@ -75,27 +72,21 @@ template class TarjanSCC std::vector components_index; std::vector component_size_vector; - std::shared_ptr m_node_based_graph; - std::unordered_set barrier_node_set; - RestrictionMap m_restriction_map; + std::shared_ptr m_graph; std::size_t size_one_counter; public: - template - TarjanSCC(std::shared_ptr graph, - const RestrictionMap &restrictions, - const ContainerT &barrier_node_list) - : components_index(graph->GetNumberOfNodes(), SPECIAL_NODEID), m_node_based_graph(graph), - m_restriction_map(restrictions), size_one_counter(0) + TarjanSCC(std::shared_ptr graph) + : components_index(graph->GetNumberOfNodes(), SPECIAL_NODEID), m_graph(graph), + size_one_counter(0) { - barrier_node_set.insert(std::begin(barrier_node_list), std::end(barrier_node_list)); - BOOST_ASSERT(m_node_based_graph->GetNumberOfNodes() > 0); + BOOST_ASSERT(m_graph->GetNumberOfNodes() > 0); } void run() { TIMER_START(SCC_RUN); - const NodeID max_node_id = m_node_based_graph->GetNumberOfNodes(); + const NodeID max_node_id = m_graph->GetNumberOfNodes(); // The following is a hack to distinguish between stuff that happens // before the recursive call and stuff that happens after @@ -140,30 +131,9 @@ template class TarjanSCC tarjan_node_list[v].on_stack = true; ++index; - const NodeID to_node_of_only_restriction = - m_restriction_map.CheckForEmanatingIsOnlyTurn(u, v); - - for (const auto current_edge : m_node_based_graph->GetAdjacentEdgeRange(v)) + for (const auto current_edge : m_graph->GetAdjacentEdgeRange(v)) { - const auto vprime = m_node_based_graph->GetTarget(current_edge); - - // Traverse outgoing edges - if (barrier_node_set.find(v) != barrier_node_set.end() && u != vprime) - { - continue; - } - - if (to_node_of_only_restriction != std::numeric_limits::max() && - vprime == to_node_of_only_restriction) - { - // At an only_-restriction but not at the right turn - // continue; - } - - if (m_restriction_map.CheckIfTurnIsRestricted(u, v, vprime)) - { - // continue; - } + const auto vprime = m_graph->GetTarget(current_edge); if (SPECIAL_NODEID == tarjan_node_list[vprime].index) { @@ -182,9 +152,7 @@ template class TarjanSCC else { processing_node_before_recursion[v] = true; - tarjan_node_list[currentFrame.parent].low_link = - std::min(tarjan_node_list[currentFrame.parent].low_link, - tarjan_node_list[v].low_link); + tarjan_node_list[u].low_link = std::min(tarjan_node_list[u].low_link, tarjan_node_list[v].low_link); // after recursion, lets do cycle checking // Check if we found a cycle. This is the bottom part of the recursion if (tarjan_node_list[v].low_link == tarjan_node_list[v].index) diff --git a/contractor/edge_based_graph_factory.cpp b/contractor/edge_based_graph_factory.cpp index 2926944e6f7..81662485252 100644 --- a/contractor/edge_based_graph_factory.cpp +++ b/contractor/edge_based_graph_factory.cpp @@ -282,8 +282,7 @@ void EdgeBasedGraphFactory::GenerateEdgeExpandedNodes() SimpleLogger().Write() << "Identifying components of the (compressed) road network"; // Run a BFS on the undirected graph and identify small components - TarjanSCC component_explorer(m_node_based_graph, *m_restriction_map, - m_barrier_nodes); + TarjanSCC component_explorer(m_node_based_graph); component_explorer.run(); diff --git a/tools/components.cpp b/tools/components.cpp index 660d9225e4b..e5e76377da7 100644 --- a/tools/components.cpp +++ b/tools/components.cpp @@ -76,19 +76,8 @@ void DeleteFileIfExists(const std::string &file_name) } } -void LoadRestrictions(const char* path, std::vector& restriction_list) -{ - std::ifstream input_stream(path, std::ios::binary); - if (!input_stream.is_open()) - { - throw osrm::exception("Cannot open restriction file"); - } - loadRestrictionsFromFile(input_stream, restriction_list); -} - std::size_t LoadGraph(const char* path, std::vector& coordinate_list, - std::vector& barrier_node_list, std::vector& graph_edge_list) { std::ifstream input_stream(path, std::ifstream::in | std::ifstream::binary); @@ -100,6 +89,7 @@ std::size_t LoadGraph(const char* path, // load graph data std::vector edge_list; std::vector traffic_light_node_list; + std::vector barrier_node_list; auto number_of_nodes = loadNodesFromFile(input_stream, barrier_node_list, traffic_light_node_list, @@ -136,8 +126,6 @@ std::size_t LoadGraph(const char* path, int main(int argc, char *argv[]) { std::vector coordinate_list; - std::vector restriction_list; - std::vector barrier_node_list; LogPolicy::GetInstance().Unmute(); try @@ -146,15 +134,14 @@ int main(int argc, char *argv[]) if (argc < 3) { SimpleLogger().Write(logWARNING) << "usage:\n" << argv[0] - << " "; + << " "; return -1; } SimpleLogger().Write() << "Using restrictions from file: " << argv[2]; std::vector graph_edge_list; - auto number_of_nodes = LoadGraph(argv[1], coordinate_list, barrier_node_list, graph_edge_list); - LoadRestrictions(argv[2], restriction_list); + auto number_of_nodes = LoadGraph(argv[1], coordinate_list, graph_edge_list); tbb::parallel_sort(graph_edge_list.begin(), graph_edge_list.end()); const auto graph = std::make_shared(number_of_nodes, graph_edge_list); @@ -163,9 +150,7 @@ int main(int argc, char *argv[]) SimpleLogger().Write() << "Starting SCC graph traversal"; - RestrictionMap restriction_map(restriction_list); - auto tarjan = osrm::make_unique>(graph, restriction_map, - barrier_node_list); + auto tarjan = osrm::make_unique>(graph); tarjan->run(); SimpleLogger().Write() << "identified: " << tarjan->get_number_of_components() << " many components"; From 2621f4a2fa1fe74dc0aa38997f4e5305d1989df1 Mon Sep 17 00:00:00 2001 From: Patrick Niklaus Date: Fri, 3 Jul 2015 21:26:33 +0200 Subject: [PATCH 021/122] Allow any input format for StaticGraph and check if edge list is sorted --- data_structures/static_graph.hpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/data_structures/static_graph.hpp b/data_structures/static_graph.hpp index 7434b56de66..3b8f9273824 100644 --- a/data_structures/static_graph.hpp +++ b/data_structures/static_graph.hpp @@ -87,8 +87,11 @@ template class StaticGraph return osrm::irange(BeginEdges(node), EndEdges(node)); } - StaticGraph(const int nodes, std::vector &graph) + template + StaticGraph(const int nodes, const ContainerT &graph) { + BOOST_ASSERT(std::is_sorted(const_cast(graph).begin(), const_cast(graph).end())); + number_of_nodes = nodes; number_of_edges = static_cast(graph.size()); node_array.resize(number_of_nodes + 1); From c2f0e4f683531d377d1f8e5cc60f9c8f5f4b20fc Mon Sep 17 00:00:00 2001 From: Patrick Niklaus Date: Sat, 4 Jul 2015 14:29:30 +0200 Subject: [PATCH 022/122] Implement correct const iterator for DeallocatingVector --- data_structures/deallocating_vector.hpp | 77 ++++++++++++++++++++++++- 1 file changed, 76 insertions(+), 1 deletion(-) diff --git a/data_structures/deallocating_vector.hpp b/data_structures/deallocating_vector.hpp index de5a24d85f8..51bebb957c1 100644 --- a/data_structures/deallocating_vector.hpp +++ b/data_structures/deallocating_vector.hpp @@ -36,6 +36,32 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include #include +template struct ConstDeallocatingVectorIteratorState +{ + ConstDeallocatingVectorIteratorState() + : index(std::numeric_limits::max()), bucket_list(nullptr) + { + } + explicit ConstDeallocatingVectorIteratorState(const ConstDeallocatingVectorIteratorState &r) + : index(r.index), bucket_list(r.bucket_list) + { + } + explicit ConstDeallocatingVectorIteratorState(const std::size_t idx, + const std::vector *input_list) + : index(idx), bucket_list(input_list) + { + } + std::size_t index; + const std::vector *bucket_list; + + ConstDeallocatingVectorIteratorState &operator=(const ConstDeallocatingVectorIteratorState &other) + { + index = other.index; + bucket_list = other.bucket_list; + return *this; + } +}; + template struct DeallocatingVectorIteratorState { DeallocatingVectorIteratorState() @@ -62,6 +88,55 @@ template struct DeallocatingVectorIteratorState } }; +template +class ConstDeallocatingVectorIterator + : public boost::iterator_facade, + ElementT, + std::random_access_iterator_tag> +{ + ConstDeallocatingVectorIteratorState current_state; + + public: + ConstDeallocatingVectorIterator() {} + ConstDeallocatingVectorIterator(std::size_t idx, const std::vector *input_list) + : current_state(idx, input_list) + { + } + + friend class boost::iterator_core_access; + + void advance(std::size_t n) { current_state.index += n; } + + void increment() { advance(1); } + + void decrement() { advance(-1); } + + bool equal(ConstDeallocatingVectorIterator const &other) const + { + return current_state.index == other.current_state.index; + } + + std::ptrdiff_t distance_to(ConstDeallocatingVectorIterator const &other) const + { + // it is important to implement it 'other minus this'. otherwise sorting breaks + return other.current_state.index - current_state.index; + } + + ElementT &dereference() const + { + const std::size_t current_bucket = current_state.index / ELEMENTS_PER_BLOCK; + const std::size_t current_index = current_state.index % ELEMENTS_PER_BLOCK; + return (current_state.bucket_list->at(current_bucket)[current_index]); + } + + ElementT &operator[](const std::size_t index) const + { + const std::size_t current_bucket = (index + current_state.index) / ELEMENTS_PER_BLOCK; + const std::size_t current_index = (index + current_state.index) % ELEMENTS_PER_BLOCK; + return (current_state.bucket_list->at(current_bucket)[current_index]); + } +}; + template class DeallocatingVectorIterator : public boost::iterator_facade, @@ -170,7 +245,7 @@ class DeallocatingVector public: using iterator = DeallocatingVectorIterator; - using const_iterator = DeallocatingVectorIterator; + using const_iterator = ConstDeallocatingVectorIterator; // this forward-only iterator deallocates all buckets that have been visited using deallocation_iterator = DeallocatingVectorRemoveIterator; From c80c2233c5a537696e194c55b6f4f9cf162b4875 Mon Sep 17 00:00:00 2001 From: Patrick Niklaus Date: Sat, 4 Jul 2015 17:37:24 +0200 Subject: [PATCH 023/122] Find components on edge-expanded graph --- algorithms/tiny_components.hpp | 4 +- contractor/edge_based_graph_factory.cpp | 49 +++------------------- contractor/edge_based_graph_factory.hpp | 2 +- contractor/processing_chain.cpp | 56 ++++++++++++++++++++++++- contractor/processing_chain.hpp | 1 + typedefs.h | 1 + 6 files changed, 66 insertions(+), 47 deletions(-) diff --git a/algorithms/tiny_components.hpp b/algorithms/tiny_components.hpp index 67ccb6ef214..5e19ac3dda6 100644 --- a/algorithms/tiny_components.hpp +++ b/algorithms/tiny_components.hpp @@ -72,11 +72,11 @@ template class TarjanSCC std::vector components_index; std::vector component_size_vector; - std::shared_ptr m_graph; + std::shared_ptr m_graph; std::size_t size_one_counter; public: - TarjanSCC(std::shared_ptr graph) + TarjanSCC(std::shared_ptr graph) : components_index(graph->GetNumberOfNodes(), SPECIAL_NODEID), m_graph(graph), size_one_counter(0) { diff --git a/contractor/edge_based_graph_factory.cpp b/contractor/edge_based_graph_factory.cpp index 81662485252..e79a8767017 100644 --- a/contractor/edge_based_graph_factory.cpp +++ b/contractor/edge_based_graph_factory.cpp @@ -26,7 +26,6 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #include "edge_based_graph_factory.hpp" -#include "../algorithms/tiny_components.hpp" #include "../data_structures/percent.hpp" #include "../util/compute_angle.hpp" #include "../util/integer_range.hpp" @@ -83,8 +82,7 @@ unsigned EdgeBasedGraphFactory::GetHighestEdgeID() } void EdgeBasedGraphFactory::InsertEdgeBasedNode(const NodeID node_u, - const NodeID node_v, - const unsigned component_id) + const NodeID node_v) { // merge edges together into one EdgeBasedNode BOOST_ASSERT(node_u != SPECIAL_NODEID); @@ -164,7 +162,7 @@ void EdgeBasedGraphFactory::InsertEdgeBasedNode(const NodeID node_u, forward_data.name_id, forward_geometry[i].second, reverse_geometry[geometry_size - 1 - i].second, forward_dist_prefix_sum[i], reverse_dist_prefix_sum[i], m_compressed_edge_container.GetPositionForID(edge_id_1), - component_id, i, forward_data.travel_mode, reverse_data.travel_mode); + INVALID_COMPONENTID, i, forward_data.travel_mode, reverse_data.travel_mode); current_edge_source_coordinate_id = current_edge_target_coordinate_id; BOOST_ASSERT(m_edge_based_node_list.back().IsCompressed()); @@ -207,7 +205,7 @@ void EdgeBasedGraphFactory::InsertEdgeBasedNode(const NodeID node_u, m_edge_based_node_list.emplace_back( forward_data.edge_id, reverse_data.edge_id, node_u, node_v, forward_data.name_id, forward_data.distance, reverse_data.distance, 0, 0, SPECIAL_EDGEID, - component_id, 0, forward_data.travel_mode, reverse_data.travel_mode); + INVALID_COMPONENTID, 0, forward_data.travel_mode, reverse_data.travel_mode); BOOST_ASSERT(!m_edge_based_node_list.back().IsCompressed()); } } @@ -279,21 +277,6 @@ unsigned EdgeBasedGraphFactory::RenumberEdges() /// Creates the nodes in the edge expanded graph from edges in the node-based graph. void EdgeBasedGraphFactory::GenerateEdgeExpandedNodes() { - SimpleLogger().Write() << "Identifying components of the (compressed) road network"; - - // Run a BFS on the undirected graph and identify small components - TarjanSCC component_explorer(m_node_based_graph); - - component_explorer.run(); - - SimpleLogger().Write() << "identified: " - << component_explorer.get_number_of_components() - << " (compressed) components"; - SimpleLogger().Write() << "identified " - << component_explorer.get_size_one_count() - << " (compressed) SCCs of size 1"; - SimpleLogger().Write() << "generating edge-expanded nodes"; - Percent progress(m_node_based_graph->GetNumberOfNodes()); // loop over all edges and generate new set of nodes @@ -318,34 +301,14 @@ void EdgeBasedGraphFactory::GenerateEdgeExpandedNodes() BOOST_ASSERT(node_u < node_v); - // Note: edges that end on barrier nodes or on a turn restriction - // may actually be in two distinct components. We choose the smallest - const unsigned size_of_component = - std::min(component_explorer.get_component_size(node_u), - component_explorer.get_component_size(node_v)); - - const unsigned id_of_smaller_component = [node_u, node_v, &component_explorer] - { - if (component_explorer.get_component_size(node_u) < - component_explorer.get_component_size(node_v)) - { - return component_explorer.get_component_id(node_u); - } - return component_explorer.get_component_id(node_v); - }(); - - const bool component_is_tiny = size_of_component < 1000; - - // we only set edge_id for forward edges + // if we found a non-forward edge reverse and try again if (edge_data.edge_id == SPECIAL_NODEID) { - InsertEdgeBasedNode(node_v, node_u, - (component_is_tiny ? id_of_smaller_component + 1 : 0)); + InsertEdgeBasedNode(node_v, node_u); } else { - InsertEdgeBasedNode(node_u, node_v, - (component_is_tiny ? id_of_smaller_component + 1 : 0)); + InsertEdgeBasedNode(node_u, node_v); } } } diff --git a/contractor/edge_based_graph_factory.hpp b/contractor/edge_based_graph_factory.hpp index 29f8bf77c81..ceb6a88c135 100644 --- a/contractor/edge_based_graph_factory.hpp +++ b/contractor/edge_based_graph_factory.hpp @@ -103,7 +103,7 @@ class EdgeBasedGraphFactory void GenerateEdgeExpandedEdges(const std::string &original_edge_data_filename, lua_State *lua_state); - void InsertEdgeBasedNode(const NodeID u, const NodeID v, const unsigned component_id); + void InsertEdgeBasedNode(const NodeID u, const NodeID v); void FlushVectorToStream(std::ofstream &edge_data_file, std::vector &original_edge_data_vector) const; diff --git a/contractor/processing_chain.cpp b/contractor/processing_chain.cpp index 39005462d6a..418aa11e75e 100644 --- a/contractor/processing_chain.cpp +++ b/contractor/processing_chain.cpp @@ -29,7 +29,7 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "contractor.hpp" #include "../algorithms/graph_compressor.hpp" - +#include "../algorithms/tiny_components.hpp" #include "../algorithms/crc32_processor.hpp" #include "../data_structures/compressed_edge_container.hpp" #include "../data_structures/deallocating_vector.hpp" @@ -94,6 +94,8 @@ int Prepare::Run() SimpleLogger().Write() << "building r-tree ..."; TIMER_START(rtree); + FindComponents(max_edge_id, edge_based_edge_list, *node_based_edge_list); + BuildRTree(*node_based_edge_list, *internal_to_external_node_map); TIMER_STOP(rtree); @@ -131,6 +133,58 @@ int Prepare::Run() return 0; } +void Prepare::FindComponents(unsigned max_edge_id, const DeallocatingVector& input_edge_list, + std::vector& input_nodes) const +{ + struct UncontractedEdgeData { }; + using InputEdge = StaticGraph::InputEdge; + using UncontractedGraph = StaticGraph; + std::vector edges; + edges.reserve(input_edge_list.size() * 2); + + for (const auto& edge : input_edge_list) + { + BOOST_ASSERT_MSG(static_cast(std::max(edge.weight, 1)) > 0, + "edge distance < 1"); + if (edge.forward) + { + edges.emplace_back(edge.source, edge.target); + } + + if (edge.backward) + { + edges.emplace_back(edge.target, edge.source); + } + } + + // connect forward and backward nodes of each edge + for (const auto& node : input_nodes) + { + if (node.reverse_edge_based_node_id != SPECIAL_NODEID) + { + edges.emplace_back(node.forward_edge_based_node_id, node.reverse_edge_based_node_id); + edges.emplace_back(node.reverse_edge_based_node_id, node.forward_edge_based_node_id); + } + } + + tbb::parallel_sort(edges.begin(), edges.end()); + auto uncontractor_graph = std::make_shared(max_edge_id+1, edges); + + TarjanSCC component_search(std::const_pointer_cast(uncontractor_graph)); + component_search.run(); + + for (auto& node : input_nodes) + { + auto forward_component = component_search.get_component_id(node.forward_edge_based_node_id); + BOOST_ASSERT(node.reverse_edge_based_node_id == SPECIAL_EDGEID || + forward_component == component_search.get_component_id(node.reverse_edge_based_node_id)); + + const unsigned component_size = component_search.get_component_size(forward_component); + const bool is_tiny_component = component_size < 1000; + node.component_id = is_tiny_component ? (1 + forward_component) : 0; + } +} + std::size_t Prepare::WriteContractedGraph(unsigned max_node_id, std::unique_ptr> node_based_edge_list, std::unique_ptr> contracted_edge_list) diff --git a/contractor/processing_chain.hpp b/contractor/processing_chain.hpp index d6ef7ec0fd8..09093bf0dde 100644 --- a/contractor/processing_chain.hpp +++ b/contractor/processing_chain.hpp @@ -78,6 +78,7 @@ class Prepare std::vector &node_based_edge_list, DeallocatingVector &edge_based_edge_list); void WriteNodeMapping(std::unique_ptr> internal_to_external_node_map); + void FindComponents(unsigned max_edge_id, const DeallocatingVector& edges, std::vector& nodes) const; void BuildRTree(const std::vector &node_based_edge_list, const std::vector &internal_to_external_node_map); private: diff --git a/typedefs.h b/typedefs.h index d1b0a53a841..a8b75aa74a4 100644 --- a/typedefs.h +++ b/typedefs.h @@ -45,6 +45,7 @@ using EdgeWeight = int; static const NodeID SPECIAL_NODEID = std::numeric_limits::max(); static const EdgeID SPECIAL_EDGEID = std::numeric_limits::max(); static const unsigned INVALID_NAMEID = std::numeric_limits::max(); +static const unsigned INVALID_COMPONENTID = std::numeric_limits::max(); static const EdgeWeight INVALID_EDGE_WEIGHT = std::numeric_limits::max(); #endif /* TYPEDEFS_H */ From 35542e58239094e19c7a455814daf8f98d4dabab Mon Sep 17 00:00:00 2001 From: Patrick Niklaus Date: Tue, 7 Jul 2015 18:24:18 +0200 Subject: [PATCH 024/122] Change interface of Tarjan get_component_size to take component id --- algorithms/tiny_components.hpp | 4 ++-- contractor/processing_chain.cpp | 27 +++++++++++++++++++++------ tools/components.cpp | 2 +- 3 files changed, 24 insertions(+), 9 deletions(-) diff --git a/algorithms/tiny_components.hpp b/algorithms/tiny_components.hpp index 5e19ac3dda6..0d8743543bb 100644 --- a/algorithms/tiny_components.hpp +++ b/algorithms/tiny_components.hpp @@ -196,9 +196,9 @@ template class TarjanSCC std::size_t get_size_one_count() const { return size_one_counter; } - unsigned get_component_size(const NodeID node) const + unsigned get_component_size(const unsigned component_id) const { - return component_size_vector[components_index[node]]; + return component_size_vector[component_id]; } unsigned get_component_id(const NodeID node) const { return components_index[node]; } diff --git a/contractor/processing_chain.cpp b/contractor/processing_chain.cpp index 418aa11e75e..513dbd7fb08 100644 --- a/contractor/processing_chain.cpp +++ b/contractor/processing_chain.cpp @@ -137,7 +137,11 @@ void Prepare::FindComponents(unsigned max_edge_id, const DeallocatingVector& input_nodes) const { struct UncontractedEdgeData { }; - using InputEdge = StaticGraph::InputEdge; + struct InputEdge { + unsigned source; + unsigned target; + UncontractedEdgeData data; + }; using UncontractedGraph = StaticGraph; std::vector edges; edges.reserve(input_edge_list.size() * 2); @@ -148,12 +152,12 @@ void Prepare::FindComponents(unsigned max_edge_id, const DeallocatingVector(max_edge_id+1, edges); TarjanSCC component_search(std::const_pointer_cast(uncontractor_graph)); diff --git a/tools/components.cpp b/tools/components.cpp index e5e76377da7..822ea3addd5 100644 --- a/tools/components.cpp +++ b/tools/components.cpp @@ -217,7 +217,7 @@ int main(int argc, char *argv[]) BOOST_ASSERT(target != SPECIAL_NODEID); const unsigned size_of_containing_component = std::min( - tarjan->get_component_size(source), tarjan->get_component_size(target)); + tarjan->get_component_size(tarjan->get_component_id(source)), tarjan->get_component_size(tarjan->get_component_id(target))); // edges that end on bollard nodes may actually be in two distinct components if (size_of_containing_component < 1000) From d4356b045384f1d21f208d5d6a9da17aa29a849a Mon Sep 17 00:00:00 2001 From: Patrick Niklaus Date: Wed, 15 Jul 2015 21:37:55 +0200 Subject: [PATCH 025/122] Move comparators to struct --- contractor/processing_chain.cpp | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/contractor/processing_chain.cpp b/contractor/processing_chain.cpp index 513dbd7fb08..e7831446a27 100644 --- a/contractor/processing_chain.cpp +++ b/contractor/processing_chain.cpp @@ -141,6 +141,16 @@ void Prepare::FindComponents(unsigned max_edge_id, const DeallocatingVector; std::vector edges; @@ -171,16 +181,8 @@ void Prepare::FindComponents(unsigned max_edge_id, const DeallocatingVector(max_edge_id+1, edges); From 1cc75ca636350ed3c1cbcd86d3a9a9ae2bb0e8a6 Mon Sep 17 00:00:00 2001 From: Patrick Niklaus Date: Thu, 6 Aug 2015 13:20:29 +0200 Subject: [PATCH 026/122] Only swap nodes if it contains a big component --- plugins/viaroute.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/viaroute.hpp b/plugins/viaroute.hpp index ceaeebf2851..f705fb5996c 100644 --- a/plugins/viaroute.hpp +++ b/plugins/viaroute.hpp @@ -110,7 +110,7 @@ template class ViaRoutePlugin final : public BasePlugin auto check_component_id_is_tiny = [](const phantom_node_pair &phantom_pair) { - return phantom_pair.first.component_id != 0; + return phantom_pair.first.is_in_tiny_component(); }; const bool every_phantom_is_in_tiny_cc = @@ -131,7 +131,7 @@ template class ViaRoutePlugin final : public BasePlugin auto swap_phantom_from_big_cc_into_front = [](phantom_node_pair &phantom_pair) { - if (0 != phantom_pair.first.component_id) + if (0 != phantom_pair.first.component_id && 0 == phantom_pair.second.component_id) { using namespace std; swap(phantom_pair.first, phantom_pair.second); From f838f3427b6d19b2ea94f301fe12c15288ee29a6 Mon Sep 17 00:00:00 2001 From: Patrick Niklaus Date: Thu, 6 Aug 2015 15:09:28 +0200 Subject: [PATCH 027/122] Fix static graph test --- unit_tests/data_structures/static_graph.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/unit_tests/data_structures/static_graph.cpp b/unit_tests/data_structures/static_graph.cpp index ed303b79199..65704c65218 100644 --- a/unit_tests/data_structures/static_graph.cpp +++ b/unit_tests/data_structures/static_graph.cpp @@ -131,9 +131,9 @@ BOOST_AUTO_TEST_CASE(find_test) std::vector input_edges = { TestInputEdge{0, 1, TestData{1}}, TestInputEdge{3, 0, TestData{2}}, + TestInputEdge{3, 0, TestData{5}}, TestInputEdge{3, 4, TestData{3}}, - TestInputEdge{4, 3, TestData{4}}, - TestInputEdge{3, 0, TestData{5}} + TestInputEdge{4, 3, TestData{4}} }; TestStaticGraph simple_graph(5, input_edges); From a7eabeb73fba55668b219fa5bfc65036cf27c5a1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20Gru=C3=9F?= Date: Tue, 11 Aug 2015 11:06:11 +0200 Subject: [PATCH 028/122] gps_precision and matching_beta can be used as a float value --- server/api_grammar.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/api_grammar.hpp b/server/api_grammar.hpp index 470954f7404..325f4c356c1 100644 --- a/server/api_grammar.hpp +++ b/server/api_grammar.hpp @@ -78,9 +78,9 @@ template struct APIGrammar : qi::grammar> qi::lit("num_results") >> '=' >> qi::short_[boost::bind(&HandlerT::setNumberOfResults, handler, ::_1)]; matching_beta = (-qi::lit('&')) >> qi::lit("matching_beta") >> '=' >> - qi::short_[boost::bind(&HandlerT::setMatchingBeta, handler, ::_1)]; + qi::float_[boost::bind(&HandlerT::setMatchingBeta, handler, ::_1)]; gps_precision = (-qi::lit('&')) >> qi::lit("gps_precision") >> '=' >> - qi::short_[boost::bind(&HandlerT::setGPSPrecision, handler, ::_1)]; + qi::float_[boost::bind(&HandlerT::setGPSPrecision, handler, ::_1)]; classify = (-qi::lit('&')) >> qi::lit("classify") >> '=' >> qi::bool_[boost::bind(&HandlerT::setClassify, handler, ::_1)]; locs = (-qi::lit('&')) >> qi::lit("locs") >> '=' >> From 49adf2192a0d2af1835bf49293f5a19918c4e7cc Mon Sep 17 00:00:00 2001 From: Patrick Niklaus Date: Thu, 6 Aug 2015 15:48:29 +0200 Subject: [PATCH 029/122] Move calculate_coordinate to algorithms/ Fixes #1367 --- CMakeLists.txt | 2 +- .../coordinate_calculation.cpp | 31 +++++++------ .../coordinate_calculation.hpp | 46 +++++++++---------- data_structures/coordinate.cpp | 2 +- data_structures/rectangle.hpp | 2 +- descriptors/description_factory.cpp | 2 +- descriptors/descriptor_base.hpp | 2 +- extractor/extraction_containers.cpp | 2 +- routing_algorithms/map_matching.hpp | 2 +- routing_algorithms/routing_base.hpp | 2 +- tools/components.cpp | 2 +- unit_tests/algorithms/geometry_string.cpp | 2 +- unit_tests/data_structures/coordinate.cpp | 2 +- unit_tests/data_structures/static_rtree.cpp | 2 +- 14 files changed, 53 insertions(+), 48 deletions(-) rename {data_structures => algorithms}/coordinate_calculation.cpp (89%) rename {data_structures => algorithms}/coordinate_calculation.hpp (56%) diff --git a/CMakeLists.txt b/CMakeLists.txt index 66b3fe829dc..77799a31cef 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -73,7 +73,7 @@ add_executable(osrm-prepare ${PrepareSources} $ $(buffer, value); } -float coordinate_calculation::deg_to_rad(const float degree) +float deg_to_rad(const float degree) { return degree * (static_cast(M_PI) / 180.f); } -float coordinate_calculation::rad_to_deg(const float radian) +float rad_to_deg(const float radian) { return radian * (180.f * static_cast(M_1_PI)); } -float coordinate_calculation::bearing(const FixedPointCoordinate &first_coordinate, +float bearing(const FixedPointCoordinate &first_coordinate, const FixedPointCoordinate &second_coordinate) { const float lon_diff = @@ -266,3 +269,5 @@ float coordinate_calculation::bearing(const FixedPointCoordinate &first_coordina } return result; } + +} diff --git a/data_structures/coordinate_calculation.hpp b/algorithms/coordinate_calculation.hpp similarity index 56% rename from data_structures/coordinate_calculation.hpp rename to algorithms/coordinate_calculation.hpp index 73183dfa4d6..309982a6acc 100644 --- a/data_structures/coordinate_calculation.hpp +++ b/algorithms/coordinate_calculation.hpp @@ -33,38 +33,38 @@ struct FixedPointCoordinate; #include #include -struct coordinate_calculation +namespace coordinate_calculation { - static double + double great_circle_distance(const int lat1, const int lon1, const int lat2, const int lon2); - static double great_circle_distance(const FixedPointCoordinate &first_coordinate, - const FixedPointCoordinate &second_coordinate); + double great_circle_distance(const FixedPointCoordinate &first_coordinate, + const FixedPointCoordinate &second_coordinate); - static float euclidean_distance(const FixedPointCoordinate &first_coordinate, - const FixedPointCoordinate &second_coordinate); + float euclidean_distance(const FixedPointCoordinate &first_coordinate, + const FixedPointCoordinate &second_coordinate); - static float euclidean_distance(const int lat1, const int lon1, const int lat2, const int lon2); + float euclidean_distance(const int lat1, const int lon1, const int lat2, const int lon2); - static void lat_or_lon_to_string(const int value, std::string &output); + void lat_or_lon_to_string(const int value, std::string &output); - static float perpendicular_distance(const FixedPointCoordinate &segment_source, - const FixedPointCoordinate &segment_target, - const FixedPointCoordinate &query_location); + float perpendicular_distance(const FixedPointCoordinate &segment_source, + const FixedPointCoordinate &segment_target, + const FixedPointCoordinate &query_location); - static float perpendicular_distance(const FixedPointCoordinate &segment_source, - const FixedPointCoordinate &segment_target, - const FixedPointCoordinate &query_location, - FixedPointCoordinate &nearest_location, - float &ratio); + float perpendicular_distance(const FixedPointCoordinate &segment_source, + const FixedPointCoordinate &segment_target, + const FixedPointCoordinate &query_location, + FixedPointCoordinate &nearest_location, + float &ratio); - static float perpendicular_distance_from_projected_coordinate( + float perpendicular_distance_from_projected_coordinate( const FixedPointCoordinate &segment_source, const FixedPointCoordinate &segment_target, const FixedPointCoordinate &query_location, const std::pair &projected_coordinate); - static float perpendicular_distance_from_projected_coordinate( + float perpendicular_distance_from_projected_coordinate( const FixedPointCoordinate &segment_source, const FixedPointCoordinate &segment_target, const FixedPointCoordinate &query_location, @@ -72,11 +72,11 @@ struct coordinate_calculation FixedPointCoordinate &nearest_location, float &ratio); - static float deg_to_rad(const float degree); - static float rad_to_deg(const float radian); + float deg_to_rad(const float degree); + float rad_to_deg(const float radian); - static float bearing(const FixedPointCoordinate &first_coordinate, - const FixedPointCoordinate &second_coordinate); -}; + float bearing(const FixedPointCoordinate &first_coordinate, + const FixedPointCoordinate &second_coordinate); +} #endif // COORDINATE_CALCULATION diff --git a/data_structures/coordinate.cpp b/data_structures/coordinate.cpp index df3abe44ae9..208f3eb0766 100644 --- a/data_structures/coordinate.cpp +++ b/data_structures/coordinate.cpp @@ -25,7 +25,7 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -#include "coordinate_calculation.hpp" +#include "../algorithms/coordinate_calculation.hpp" #ifndef NDEBUG #include "../util/simple_logger.hpp" diff --git a/data_structures/rectangle.hpp b/data_structures/rectangle.hpp index 55720a37daa..9066efc8b0a 100644 --- a/data_structures/rectangle.hpp +++ b/data_structures/rectangle.hpp @@ -28,7 +28,7 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #ifndef RECTANGLE_HPP #define RECTANGLE_HPP -#include "coordinate_calculation.hpp" +#include "../algorithms/coordinate_calculation.hpp" #include diff --git a/descriptors/description_factory.cpp b/descriptors/description_factory.cpp index 71dd7951349..3609bc3dff5 100644 --- a/descriptors/description_factory.cpp +++ b/descriptors/description_factory.cpp @@ -28,7 +28,7 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "description_factory.hpp" #include "../algorithms/polyline_formatter.hpp" -#include "../data_structures/coordinate_calculation.hpp" +#include "../algorithms/coordinate_calculation.hpp" #include "../data_structures/internal_route_result.hpp" #include "../data_structures/turn_instructions.hpp" #include "../util/container.hpp" diff --git a/descriptors/descriptor_base.hpp b/descriptors/descriptor_base.hpp index 497c2161d51..e85ea902f0b 100644 --- a/descriptors/descriptor_base.hpp +++ b/descriptors/descriptor_base.hpp @@ -28,7 +28,7 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #ifndef DESCRIPTOR_BASE_HPP #define DESCRIPTOR_BASE_HPP -#include "../data_structures/coordinate_calculation.hpp" +#include "../algorithms/coordinate_calculation.hpp" #include "../data_structures/internal_route_result.hpp" #include "../data_structures/phantom_node.hpp" #include "../typedefs.h" diff --git a/extractor/extraction_containers.cpp b/extractor/extraction_containers.cpp index c661ba2c8fd..584b8cc7643 100644 --- a/extractor/extraction_containers.cpp +++ b/extractor/extraction_containers.cpp @@ -28,7 +28,7 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "extraction_containers.hpp" #include "extraction_way.hpp" -#include "../data_structures/coordinate_calculation.hpp" +#include "../algorithms/coordinate_calculation.hpp" #include "../data_structures/node_id.hpp" #include "../data_structures/range_table.hpp" diff --git a/routing_algorithms/map_matching.hpp b/routing_algorithms/map_matching.hpp index 737494b64de..dfb0af0a8ac 100644 --- a/routing_algorithms/map_matching.hpp +++ b/routing_algorithms/map_matching.hpp @@ -30,7 +30,7 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "routing_base.hpp" -#include "../data_structures/coordinate_calculation.hpp" +#include "../algorithms/coordinate_calculation.hpp" #include "../data_structures/hidden_markov_model.hpp" #include "../util/json_logger.hpp" #include "../util/matching_debug_info.hpp" diff --git a/routing_algorithms/routing_base.hpp b/routing_algorithms/routing_base.hpp index 52c16a77eeb..d41250a7959 100644 --- a/routing_algorithms/routing_base.hpp +++ b/routing_algorithms/routing_base.hpp @@ -28,7 +28,7 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #ifndef ROUTING_BASE_HPP #define ROUTING_BASE_HPP -#include "../data_structures/coordinate_calculation.hpp" +#include "../algorithms/coordinate_calculation.hpp" #include "../data_structures/internal_route_result.hpp" #include "../data_structures/search_engine_data.hpp" #include "../data_structures/turn_instructions.hpp" diff --git a/tools/components.cpp b/tools/components.cpp index 822ea3addd5..eed57974651 100644 --- a/tools/components.cpp +++ b/tools/components.cpp @@ -27,7 +27,7 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "../typedefs.h" #include "../algorithms/tiny_components.hpp" -#include "../data_structures/coordinate_calculation.hpp" +#include "../algorithms/coordinate_calculation.hpp" #include "../data_structures/dynamic_graph.hpp" #include "../data_structures/static_graph.hpp" #include "../util/fingerprint.hpp" diff --git a/unit_tests/algorithms/geometry_string.cpp b/unit_tests/algorithms/geometry_string.cpp index 035b3217c80..ab9f357557e 100644 --- a/unit_tests/algorithms/geometry_string.cpp +++ b/unit_tests/algorithms/geometry_string.cpp @@ -29,7 +29,7 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include #include "../../algorithms/polyline_compressor.hpp" -#include "../../data_structures/coordinate_calculation.hpp" +#include "../../algorithms/coordinate_calculation.hpp" #include diff --git a/unit_tests/data_structures/coordinate.cpp b/unit_tests/data_structures/coordinate.cpp index 6e891563a44..e8209763e23 100644 --- a/unit_tests/data_structures/coordinate.cpp +++ b/unit_tests/data_structures/coordinate.cpp @@ -27,7 +27,7 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include -#include "../../data_structures/coordinate_calculation.hpp" +#include "../../algorithms/coordinate_calculation.hpp" #include diff --git a/unit_tests/data_structures/static_rtree.cpp b/unit_tests/data_structures/static_rtree.cpp index 42bc7189859..57636473a58 100644 --- a/unit_tests/data_structures/static_rtree.cpp +++ b/unit_tests/data_structures/static_rtree.cpp @@ -25,7 +25,7 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -#include "../../data_structures/coordinate_calculation.hpp" +#include "../../algorithms/coordinate_calculation.hpp" #include "../../data_structures/static_rtree.hpp" #include "../../data_structures/query_node.hpp" #include "../../data_structures/edge_based_node.hpp" From 4b4bc0dde2bbe33547593d393e4fbba94686aa39 Mon Sep 17 00:00:00 2001 From: Patrick Niklaus Date: Fri, 7 Aug 2015 12:02:43 +0200 Subject: [PATCH 030/122] Fix postgis lua example Fixes #1573. --- profiles/examples/postgis.lua | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/profiles/examples/postgis.lua b/profiles/examples/postgis.lua index 6e32e4c7f04..2ff72fdd9a9 100644 --- a/profiles/examples/postgis.lua +++ b/profiles/examples/postgis.lua @@ -45,7 +45,7 @@ function node_function(node) end -- ways processing, called from OSRM -function way_function (way) +function way_function (way, result) -- only route on ways with highway=* local highway = way.tags:Find("highway") if (not highway or highway=='') then @@ -65,18 +65,18 @@ function way_function (way) local cursor = assert( sql_con:execute(sql_query) ) -- execute querty local row = cursor:fetch( {}, "a" ) -- fetch first (and only) row - way.forward_speed = 20.0 -- default speed + result.forward_speed = 20.0 -- default speed + result.forward_mode = 1 if row then - local val = tonumber(row.val) -- read 'val' from row + local val = tonumber(row.val) -- read 'val' from row if val > 10 then - way.forward_speed = way.forward_speed / math.log10( val ) -- reduce speed by amount of industry close by + -- reduce speed by amount of industry close by + result.forward_speed = way.forward_speed / math.log10( val ) end end cursor:close() -- done with this query -- set other required info for this way - way.name = way.tags:Find("name") - way.direction = Way.bidirectional - way.type = 1 + result.name = way.get_value_by_key("name") return 1 end From c43a2513a8bb4171bc2341f6a35b78ee5bcadbdf Mon Sep 17 00:00:00 2001 From: Patrick Niklaus Date: Fri, 7 Aug 2015 12:18:02 +0200 Subject: [PATCH 031/122] Rename tiny_components.hpp to tarjan_scc.hpp Fixes #1561 --- algorithms/{tiny_components.hpp => tarjan_scc.hpp} | 0 contractor/processing_chain.cpp | 2 +- tools/components.cpp | 2 +- 3 files changed, 2 insertions(+), 2 deletions(-) rename algorithms/{tiny_components.hpp => tarjan_scc.hpp} (100%) diff --git a/algorithms/tiny_components.hpp b/algorithms/tarjan_scc.hpp similarity index 100% rename from algorithms/tiny_components.hpp rename to algorithms/tarjan_scc.hpp diff --git a/contractor/processing_chain.cpp b/contractor/processing_chain.cpp index e7831446a27..77e13b08904 100644 --- a/contractor/processing_chain.cpp +++ b/contractor/processing_chain.cpp @@ -29,7 +29,7 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "contractor.hpp" #include "../algorithms/graph_compressor.hpp" -#include "../algorithms/tiny_components.hpp" +#include "../algorithms/tarjan_scc.hpp" #include "../algorithms/crc32_processor.hpp" #include "../data_structures/compressed_edge_container.hpp" #include "../data_structures/deallocating_vector.hpp" diff --git a/tools/components.cpp b/tools/components.cpp index eed57974651..d00713abc9a 100644 --- a/tools/components.cpp +++ b/tools/components.cpp @@ -26,7 +26,7 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #include "../typedefs.h" -#include "../algorithms/tiny_components.hpp" +#include "../algorithms/tarjan_scc.hpp" #include "../algorithms/coordinate_calculation.hpp" #include "../data_structures/dynamic_graph.hpp" #include "../data_structures/static_graph.hpp" From e30f0e8e11b5e4498c15c4962ac44d777eb59b05 Mon Sep 17 00:00:00 2001 From: Patrick Niklaus Date: Fri, 7 Aug 2015 12:57:41 +0200 Subject: [PATCH 032/122] Always announce a turn on mode change Fixes #1558 --- contractor/edge_based_graph_factory.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contractor/edge_based_graph_factory.cpp b/contractor/edge_based_graph_factory.cpp index e79a8767017..93a07a37665 100644 --- a/contractor/edge_based_graph_factory.cpp +++ b/contractor/edge_based_graph_factory.cpp @@ -549,7 +549,7 @@ TurnInstruction EdgeBasedGraphFactory::AnalyzeTurn(const NodeID node_u, // If street names stay the same and if we are certain that it is not a // a segment of a roundabout, we skip it. - if (data1.name_id == data2.name_id) + if (data1.name_id == data2.name_id && data1.travel_mode == data2.travel_mode) { // TODO: Here we should also do a small graph exploration to check for // more complex situations From bd37c48596f88cf904855e7501bb5bced0627c1f Mon Sep 17 00:00:00 2001 From: Patrick Niklaus Date: Fri, 7 Aug 2015 13:03:35 +0200 Subject: [PATCH 033/122] Add test for mode change --- features/testbot/mode.feature | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/features/testbot/mode.feature b/features/testbot/mode.feature index 11fe83a4279..ef8cd257e04 100644 --- a/features/testbot/mode.feature +++ b/features/testbot/mode.feature @@ -11,7 +11,21 @@ Feature: Testbot - Travel mode Background: Given the profile "testbot" - + + Scenario: Testbot - Always announce mode change + Given the node map + | a | b | c | d | + + And the ways + | nodes | highway | name | + | ab | residential | foo | + | bc | river | foo | + | cd | residential | foo | + + When I route I should get + | from | to | route | modes | + | a | d | foo,foo,foo | 1,3,1 | + Scenario: Testbot - Modes in each direction, different forward/backward speeds Given the node map | | 0 | 1 | | From 84e72ede72b42f154bb1be50cef967942e5452e8 Mon Sep 17 00:00:00 2001 From: Patrick Niklaus Date: Fri, 14 Aug 2015 23:57:01 +0200 Subject: [PATCH 034/122] Warn if an edge references a missing node --- extractor/extraction_containers.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/extractor/extraction_containers.cpp b/extractor/extraction_containers.cpp index 584b8cc7643..071ea97e6d3 100644 --- a/extractor/extraction_containers.cpp +++ b/extractor/extraction_containers.cpp @@ -186,6 +186,7 @@ void ExtractionContainers::PrepareEdges() { if (edge_iterator->result.source < node_iterator->node_id) { + SimpleLogger().Write(LogLevel::logWARNING) << "Found invalid node reference " << edge_iterator->result.source; edge_iterator->result.source = SPECIAL_NODEID; ++edge_iterator; continue; @@ -243,6 +244,7 @@ void ExtractionContainers::PrepareEdges() if (edge_iterator->result.target < node_iterator->node_id) { + SimpleLogger().Write(LogLevel::logWARNING) << "Found invalid node reference " << edge_iterator->result.target; edge_iterator->result.target = SPECIAL_NODEID; ++edge_iterator; continue; From 62b20769eea3cc0fc37e70634d806a4b3a6ad789 Mon Sep 17 00:00:00 2001 From: "Daniel J. Hofmann" Date: Tue, 18 Aug 2015 12:56:34 +0200 Subject: [PATCH 035/122] Modernize the code base to C++11 standards and beyond. Apply `clang-modernize` (based on Clang 3.6) transformations to the codebase while making sure to support Clang>=3.4 and GCC>=4.8. We apply the transformations in parallel to speed up the quite time consuming process, and use our `clang-format` style file to automatically format the code respecting our coding conventions. We use the following self-explanatory transformations: * AddOverride * LoopConvert * PassByValue * ReplaceAutoPtr * UseAuto * UseNullptr This required a `compile_commands.json` compilation database, e.g. ccmake .. -DCMAKE_BUILD_TYPE=Release -DCMAKE_EXPORT_COMPILE_COMMANDS=1 for CMake or check Bear for a Makefile based solution (or even Ninja). git ls-files -x '*.cpp|*.h' | \ xargs -I{} -P $(nproc) clang-modernize -p build -final-syntax-check -format -style=file -summary -for-compilers=clang-3.4,gcc-4.8 -include . -exclude third_party {} Boom! References: * http://clang.llvm.org/extra/clang-modernize.html * http://clang.llvm.org/extra/ModernizerUsage.html --- algorithms/bayes_classifier.hpp | 9 ++--- algorithms/douglas_peucker.cpp | 4 +-- algorithms/graph_compressor.cpp | 4 +-- algorithms/graph_compressor.hpp | 2 +- contractor/edge_based_graph_factory.cpp | 26 +++++++-------- contractor/edge_based_graph_factory.hpp | 9 +++-- contractor/processing_chain.hpp | 3 +- data_structures/binary_heap.hpp | 6 ++-- data_structures/query_edge.hpp | 2 +- data_structures/segment_information.hpp | 15 +++++---- data_structures/shared_memory_factory.hpp | 2 +- data_structures/static_rtree.hpp | 6 ++-- extractor/extractor.hpp | 3 +- extractor/internal_extractor_edge.hpp | 33 ++++++++++++------- include/osrm/json_container.hpp | 2 +- include/osrm/libosrm_config.hpp | 7 ++-- plugins/match.hpp | 4 +-- routed.cpp | 4 +-- .../data_structures/internal_datafacade.hpp | 2 +- server/http/header.hpp | 2 +- util/datastore_options.hpp | 2 +- util/matching_debug_info.hpp | 10 +++--- util/osrm_exception.hpp | 5 +-- util/routed_options.hpp | 2 +- 24 files changed, 88 insertions(+), 76 deletions(-) diff --git a/algorithms/bayes_classifier.hpp b/algorithms/bayes_classifier.hpp index 3358144c750..ea300c1a999 100644 --- a/algorithms/bayes_classifier.hpp +++ b/algorithms/bayes_classifier.hpp @@ -31,6 +31,7 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include #include +#include struct NormalDistribution { @@ -80,11 +81,11 @@ class BayesClassifier }; using ClassificationT = std::pair; - BayesClassifier(const PositiveDistributionT &positive_distribution, - const NegativeDistributionT &negative_distribution, + BayesClassifier(PositiveDistributionT positive_distribution, + NegativeDistributionT negative_distribution, const double positive_apriori_probability) - : positive_distribution(positive_distribution), - negative_distribution(negative_distribution), + : positive_distribution(std::move(positive_distribution)), + negative_distribution(std::move(negative_distribution)), positive_apriori_probability(positive_apriori_probability), negative_apriori_probability(1. - positive_apriori_probability) { diff --git a/algorithms/douglas_peucker.cpp b/algorithms/douglas_peucker.cpp index fa7d7826b85..280c90fa876 100644 --- a/algorithms/douglas_peucker.cpp +++ b/algorithms/douglas_peucker.cpp @@ -99,8 +99,8 @@ void DouglasPeucker::Run(RandomAccessIt begin, RandomAccessIt end, const unsigne { BOOST_ASSERT_MSG(zoom_level < DOUGLAS_PEUCKER_THRESHOLDS.size(), "unsupported zoom level"); - RandomAccessIt left_border = begin; - RandomAccessIt right_border = std::next(begin); + auto left_border = begin; + auto right_border = std::next(begin); // Sweep over array and identify those ranges that need to be checked do { diff --git a/algorithms/graph_compressor.cpp b/algorithms/graph_compressor.cpp index d375a588bde..f3b5f8dd957 100644 --- a/algorithms/graph_compressor.cpp +++ b/algorithms/graph_compressor.cpp @@ -8,8 +8,8 @@ #include "../util/simple_logger.hpp" -GraphCompressor::GraphCompressor(const SpeedProfileProperties& speed_profile) - : speed_profile(speed_profile) +GraphCompressor::GraphCompressor(SpeedProfileProperties speed_profile) + : speed_profile(std::move(speed_profile)) { } diff --git a/algorithms/graph_compressor.hpp b/algorithms/graph_compressor.hpp index c1cca153e6c..75405c0c940 100644 --- a/algorithms/graph_compressor.hpp +++ b/algorithms/graph_compressor.hpp @@ -43,7 +43,7 @@ class GraphCompressor using EdgeData = NodeBasedDynamicGraph::EdgeData; public: - GraphCompressor(const SpeedProfileProperties& speed_profile); + GraphCompressor(SpeedProfileProperties speed_profile); void Compress(const std::unordered_set& barrier_nodes, const std::unordered_set& traffic_lights, diff --git a/contractor/edge_based_graph_factory.cpp b/contractor/edge_based_graph_factory.cpp index 93a07a37665..b2ca9778fd2 100644 --- a/contractor/edge_based_graph_factory.cpp +++ b/contractor/edge_based_graph_factory.cpp @@ -39,20 +39,18 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include #include -EdgeBasedGraphFactory::EdgeBasedGraphFactory(std::shared_ptr node_based_graph, - const CompressedEdgeContainer& compressed_edge_container, - const std::unordered_set& barrier_nodes, - const std::unordered_set& traffic_lights, - std::shared_ptr restriction_map, - const std::vector &node_info_list, - const SpeedProfileProperties &speed_profile) - : m_node_info_list(node_info_list), - m_node_based_graph(node_based_graph), - m_restriction_map(restriction_map), - m_barrier_nodes(barrier_nodes), - m_traffic_lights(traffic_lights), - m_compressed_edge_container(compressed_edge_container), - speed_profile(speed_profile) +EdgeBasedGraphFactory::EdgeBasedGraphFactory( + std::shared_ptr node_based_graph, + const CompressedEdgeContainer &compressed_edge_container, + const std::unordered_set &barrier_nodes, + const std::unordered_set &traffic_lights, + std::shared_ptr restriction_map, + const std::vector &node_info_list, + SpeedProfileProperties speed_profile) + : m_node_info_list(node_info_list), m_node_based_graph(std::move(node_based_graph)), + m_restriction_map(std::move(restriction_map)), m_barrier_nodes(barrier_nodes), + m_traffic_lights(traffic_lights), m_compressed_edge_container(compressed_edge_container), + speed_profile(std::move(speed_profile)) { } diff --git a/contractor/edge_based_graph_factory.hpp b/contractor/edge_based_graph_factory.hpp index ceb6a88c135..a9ef8ea4dfa 100644 --- a/contractor/edge_based_graph_factory.hpp +++ b/contractor/edge_based_graph_factory.hpp @@ -58,14 +58,13 @@ class EdgeBasedGraphFactory EdgeBasedGraphFactory() = delete; EdgeBasedGraphFactory(const EdgeBasedGraphFactory &) = delete; - explicit EdgeBasedGraphFactory(std::shared_ptr node_based_graph, - const CompressedEdgeContainer& compressed_edge_container, - const std::unordered_set& barrier_nodes, - const std::unordered_set& traffic_lights, + const CompressedEdgeContainer &compressed_edge_container, + const std::unordered_set &barrier_nodes, + const std::unordered_set &traffic_lights, std::shared_ptr restriction_map, const std::vector &node_info_list, - const SpeedProfileProperties &speed_profile); + SpeedProfileProperties speed_profile); void Run(const std::string &original_edge_data_filename, lua_State *lua_state); diff --git a/contractor/processing_chain.hpp b/contractor/processing_chain.hpp index 09093bf0dde..33dbe917d8a 100644 --- a/contractor/processing_chain.hpp +++ b/contractor/processing_chain.hpp @@ -51,8 +51,7 @@ class Prepare using InputEdge = DynamicGraph::InputEdge; using StaticEdge = StaticGraph::InputEdge; - explicit Prepare(const ContractorConfig& contractor_config) - : config(contractor_config) {} + explicit Prepare(ContractorConfig contractor_config) : config(std::move(contractor_config)) {} Prepare(const Prepare &) = delete; ~Prepare(); diff --git a/data_structures/binary_heap.hpp b/data_structures/binary_heap.hpp index b237486aa7a..85703f4a6b5 100644 --- a/data_structures/binary_heap.hpp +++ b/data_structures/binary_heap.hpp @@ -212,7 +212,7 @@ class BinaryHeap void DeleteAll() { auto iend = heap.end(); - for (typename std::vector::iterator i = heap.begin() + 1; i != iend; ++i) + for (auto i = heap.begin() + 1; i != iend; ++i) { inserted_nodes[i->index].key = 0; } @@ -237,7 +237,9 @@ class BinaryHeap class HeapNode { public: - HeapNode(NodeID n, Key k, Weight w, Data d) : node(n), key(k), weight(w), data(d) {} + HeapNode(NodeID n, Key k, Weight w, Data d) : node(n), key(k), weight(w), data(std::move(d)) + { + } NodeID node; Key key; diff --git a/data_structures/query_edge.hpp b/data_structures/query_edge.hpp index 417bb4a13f9..1c4af03d9c4 100644 --- a/data_structures/query_edge.hpp +++ b/data_structures/query_edge.hpp @@ -58,7 +58,7 @@ struct QueryEdge QueryEdge() : source(SPECIAL_NODEID), target(SPECIAL_NODEID) {} QueryEdge(NodeID source, NodeID target, EdgeData data) - : source(source), target(target), data(data) + : source(source), target(target), data(std::move(data)) { } diff --git a/data_structures/segment_information.hpp b/data_structures/segment_information.hpp index 7118a320e85..b22254f02b8 100644 --- a/data_structures/segment_information.hpp +++ b/data_structures/segment_information.hpp @@ -34,6 +34,7 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "../typedefs.h" #include +#include // Struct fits everything in one cache line struct SegmentInformation @@ -48,7 +49,7 @@ struct SegmentInformation bool necessary; bool is_via_location; - explicit SegmentInformation(const FixedPointCoordinate &location, + explicit SegmentInformation(FixedPointCoordinate location, const NodeID name_id, const EdgeWeight duration, const float length, @@ -56,20 +57,20 @@ struct SegmentInformation const bool necessary, const bool is_via_location, const TravelMode travel_mode) - : location(location), name_id(name_id), duration(duration), length(length), bearing(0), - turn_instruction(turn_instruction), travel_mode(travel_mode), necessary(necessary), - is_via_location(is_via_location) + : location(std::move(location)), name_id(name_id), duration(duration), length(length), + bearing(0), turn_instruction(turn_instruction), travel_mode(travel_mode), + necessary(necessary), is_via_location(is_via_location) { } - explicit SegmentInformation(const FixedPointCoordinate &location, + explicit SegmentInformation(FixedPointCoordinate location, const NodeID name_id, const EdgeWeight duration, const float length, const TurnInstruction turn_instruction, const TravelMode travel_mode) - : location(location), name_id(name_id), duration(duration), length(length), bearing(0), - turn_instruction(turn_instruction), travel_mode(travel_mode), + : location(std::move(location)), name_id(name_id), duration(duration), length(length), + bearing(0), turn_instruction(turn_instruction), travel_mode(travel_mode), necessary(turn_instruction != TurnInstruction::NoTurn), is_via_location(false) { } diff --git a/data_structures/shared_memory_factory.hpp b/data_structures/shared_memory_factory.hpp index 58fed9bb3f5..7d92eaaa596 100644 --- a/data_structures/shared_memory_factory.hpp +++ b/data_structures/shared_memory_factory.hpp @@ -126,7 +126,7 @@ class SharedMemory shm = boost::interprocess::xsi_shared_memory(boost::interprocess::open_or_create, key, size); #ifdef __linux__ - if (-1 == shmctl(shm.get_shmid(), SHM_LOCK, 0)) + if (-1 == shmctl(shm.get_shmid(), SHM_LOCK, nullptr)) { if (ENOMEM == errno) { diff --git a/data_structures/static_rtree.hpp b/data_structures/static_rtree.hpp index 8c05cf52183..e5b7be92cbf 100644 --- a/data_structures/static_rtree.hpp +++ b/data_structures/static_rtree.hpp @@ -316,8 +316,8 @@ class StaticRTree using IncrementalQueryNodeType = mapbox::util::variant; struct IncrementalQueryCandidate { - explicit IncrementalQueryCandidate(const float dist, const IncrementalQueryNodeType &node) - : min_dist(dist), node(node) + explicit IncrementalQueryCandidate(const float dist, IncrementalQueryNodeType node) + : min_dist(dist), node(std::move(node)) { } @@ -553,7 +553,7 @@ class StaticRTree const boost::filesystem::path &leaf_file, std::shared_ptr coordinate_list) : m_search_tree(tree_node_ptr, number_of_nodes), m_leaf_node_filename(leaf_file.string()), - m_coordinate_list(coordinate_list) + m_coordinate_list(std::move(coordinate_list)) { // open leaf node file and store thread specific pointer if (!boost::filesystem::exists(leaf_file)) diff --git a/extractor/extractor.hpp b/extractor/extractor.hpp index 72defca05f9..bc822413d9d 100644 --- a/extractor/extractor.hpp +++ b/extractor/extractor.hpp @@ -33,8 +33,7 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. class extractor { public: - extractor(const ExtractorConfig &extractor_config) - : config(extractor_config) {} + extractor(ExtractorConfig extractor_config) : config(std::move(extractor_config)) {} int run(); private: ExtractorConfig config; diff --git a/extractor/internal_extractor_edge.hpp b/extractor/internal_extractor_edge.hpp index 462bd3531d8..2dc3c3488ee 100644 --- a/extractor/internal_extractor_edge.hpp +++ b/extractor/internal_extractor_edge.hpp @@ -35,6 +35,7 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include #include +#include struct InternalExtractorEdge { @@ -68,18 +69,26 @@ struct InternalExtractorEdge } explicit InternalExtractorEdge(NodeID source, - NodeID target, - NodeID name_id, - const WeightData& weight_data, - bool forward, - bool backward, - bool roundabout, - bool access_restricted, - TravelMode travel_mode, - bool is_split) - : result(source, target, name_id, 0, forward, backward, roundabout, - access_restricted, travel_mode, is_split), - weight_data(weight_data) + NodeID target, + NodeID name_id, + WeightData weight_data, + bool forward, + bool backward, + bool roundabout, + bool access_restricted, + TravelMode travel_mode, + bool is_split) + : result(source, + target, + name_id, + 0, + forward, + backward, + roundabout, + access_restricted, + travel_mode, + is_split), + weight_data(std::move(weight_data)) { } diff --git a/include/osrm/json_container.hpp b/include/osrm/json_container.hpp index 40f39b82570..63accb81c0c 100644 --- a/include/osrm/json_container.hpp +++ b/include/osrm/json_container.hpp @@ -50,7 +50,7 @@ struct String { String() {} String(const char *value) : value(value) {} - String(const std::string &value) : value(value) {} + String(std::string value) : value(std::move(value)) {} std::string value; }; diff --git a/include/osrm/libosrm_config.hpp b/include/osrm/libosrm_config.hpp index 67caa399737..83772d0a5dd 100644 --- a/include/osrm/libosrm_config.hpp +++ b/include/osrm/libosrm_config.hpp @@ -39,8 +39,11 @@ struct libosrm_config { } - libosrm_config(const ServerPaths &paths, const bool sharedmemory_flag, const int max_table, const int max_matching) - : server_paths(paths), max_locations_distance_table(max_table), + libosrm_config(ServerPaths paths, + const bool sharedmemory_flag, + const int max_table, + const int max_matching) + : server_paths(std::move(paths)), max_locations_distance_table(max_table), max_locations_map_matching(max_matching), use_shared_memory(sharedmemory_flag) { } diff --git a/plugins/match.hpp b/plugins/match.hpp index a866947b4bc..b0138247c7e 100644 --- a/plugins/match.hpp +++ b/plugins/match.hpp @@ -73,7 +73,7 @@ template class MapMatchingPlugin : public BasePlugin virtual ~MapMatchingPlugin() {} - const std::string GetDescriptor() const final { return descriptor_string; } + const std::string GetDescriptor() const final override { return descriptor_string; } TraceClassification classify(const float trace_length, const float matched_length, const int removed_points) const @@ -211,7 +211,7 @@ template class MapMatchingPlugin : public BasePlugin } int HandleRequest(const RouteParameters &route_parameters, - osrm::json::Object &json_result) final + osrm::json::Object &json_result) final override { // check number of parameters if (!check_all_coordinates(route_parameters.coordinates)) diff --git a/routed.cpp b/routed.cpp index 294efa8f180..c8186f7c93e 100644 --- a/routed.cpp +++ b/routed.cpp @@ -136,12 +136,12 @@ int main(int argc, const char *argv[]) #ifndef _WIN32 sigset_t wait_mask; - pthread_sigmask(SIG_SETMASK, &old_mask, 0); + pthread_sigmask(SIG_SETMASK, &old_mask, nullptr); sigemptyset(&wait_mask); sigaddset(&wait_mask, SIGINT); sigaddset(&wait_mask, SIGQUIT); sigaddset(&wait_mask, SIGTERM); - pthread_sigmask(SIG_BLOCK, &wait_mask, 0); + pthread_sigmask(SIG_BLOCK, &wait_mask, nullptr); SimpleLogger().Write() << "running and waiting for requests"; sigwait(&wait_mask, &sig); #else diff --git a/server/data_structures/internal_datafacade.hpp b/server/data_structures/internal_datafacade.hpp index 823ac3329b8..7439cb8318a 100644 --- a/server/data_structures/internal_datafacade.hpp +++ b/server/data_structures/internal_datafacade.hpp @@ -264,7 +264,7 @@ template class InternalDataFacade final : public BaseDataFacad throw osrm::exception("no names file given in ini file"); } - ServerPaths::const_iterator paths_iterator = server_paths.find("hsgrdata"); + auto paths_iterator = server_paths.find("hsgrdata"); BOOST_ASSERT(server_paths.end() != paths_iterator); const boost::filesystem::path &hsgr_path = paths_iterator->second; paths_iterator = server_paths.find("timestamp"); diff --git a/server/http/header.hpp b/server/http/header.hpp index 08d2476b188..f2598ba1ed7 100644 --- a/server/http/header.hpp +++ b/server/http/header.hpp @@ -37,7 +37,7 @@ struct header { // explicitly use default copy c'tor as adding move c'tor header &operator=(const header &other) = default; - header(const std::string &name, const std::string &value) : name(name), value(value) {} + header(std::string name, std::string value) : name(std::move(name)), value(std::move(value)) {} header(header &&other) : name(std::move(other.name)), value(std::move(other.value)) {} void clear() diff --git a/util/datastore_options.hpp b/util/datastore_options.hpp index 76edc782799..9f16c5757ee 100644 --- a/util/datastore_options.hpp +++ b/util/datastore_options.hpp @@ -146,7 +146,7 @@ bool GenerateDataStoreOptions(const int argc, const char *argv[], ServerPaths &p } // parse config file - ServerPaths::iterator path_iterator = paths.find("config"); + auto path_iterator = paths.find("config"); if (path_iterator != paths.end() && boost::filesystem::is_regular_file(path_iterator->second) && !option_variables.count("base")) { diff --git a/util/matching_debug_info.hpp b/util/matching_debug_info.hpp index faf878e3f9e..de48a84cfbb 100644 --- a/util/matching_debug_info.hpp +++ b/util/matching_debug_info.hpp @@ -54,16 +54,16 @@ struct MatchingDebugInfo } osrm::json::Array states; - for (unsigned t = 0; t < candidates_list.size(); t++) + for (auto &elem : candidates_list) { osrm::json::Array timestamps; - for (unsigned s = 0; s < candidates_list[t].size(); s++) + for (unsigned s = 0; s < elem.size(); s++) { osrm::json::Object state; state.values["transitions"] = osrm::json::Array(); - state.values["coordinate"] = osrm::json::make_array( - candidates_list[t][s].first.location.lat / COORDINATE_PRECISION, - candidates_list[t][s].first.location.lon / COORDINATE_PRECISION); + state.values["coordinate"] = + osrm::json::make_array(elem[s].first.location.lat / COORDINATE_PRECISION, + elem[s].first.location.lon / COORDINATE_PRECISION); state.values["viterbi"] = osrm::json::clamp_float(osrm::matching::IMPOSSIBLE_LOG_PROB); state.values["pruned"] = 0u; diff --git a/util/osrm_exception.hpp b/util/osrm_exception.hpp index e9a01381ee9..369433f3437 100644 --- a/util/osrm_exception.hpp +++ b/util/osrm_exception.hpp @@ -30,6 +30,7 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include #include +#include namespace osrm { @@ -37,14 +38,14 @@ class exception final : public std::exception { public: explicit exception(const char *message) : message(message) {} - explicit exception(const std::string &message) : message(message) {} + explicit exception(std::string message) : message(std::move(message)) {} private: // This function exists to 'anchor' the class, and stop the compiler from // copying vtable and RTTI info into every object file that includes // this header. (Caught by -Wweak-vtables under Clang.) virtual void anchor() const; - const char *what() const noexcept { return message.c_str(); } + const char *what() const noexcept override { return message.c_str(); } const std::string message; }; } diff --git a/util/routed_options.hpp b/util/routed_options.hpp index 046ab71b318..106ea23f2ef 100644 --- a/util/routed_options.hpp +++ b/util/routed_options.hpp @@ -243,7 +243,7 @@ inline unsigned GenerateServerProgramOptions(const int argc, boost::program_options::notify(option_variables); // parse config file - ServerPaths::iterator path_iterator = paths.find("config"); + auto path_iterator = paths.find("config"); if (path_iterator != paths.end() && boost::filesystem::is_regular_file(path_iterator->second) && !option_variables.count("base")) { From 338ac5d4a359e2a2bd67687c41db4095b8faef41 Mon Sep 17 00:00:00 2001 From: Patrick Niklaus Date: Fri, 7 Aug 2015 18:50:21 +0200 Subject: [PATCH 036/122] Rename map to describe what it actually does --- contractor/contractor.hpp | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/contractor/contractor.hpp b/contractor/contractor.hpp index cd5f8eecb0f..dfdc0c2beec 100644 --- a/contractor/contractor.hpp +++ b/contractor/contractor.hpp @@ -349,7 +349,7 @@ class Contractor std::vector new_node_priority(remaining_nodes.size()); // this map gives the old IDs from the new ones, necessary to get a consistent graph // at the end of contraction - orig_node_id_to_new_id_map.resize(remaining_nodes.size()); + orig_node_id_from_new_node_id_map.resize(remaining_nodes.size()); // this map gives the new IDs from the old ones, necessary to remap targets from the // remaining graph std::vector new_node_id_from_orig_id_map(number_of_nodes, UINT_MAX); @@ -359,7 +359,7 @@ class Contractor for (const auto new_node_id : osrm::irange(0, remaining_nodes.size())) { // create renumbering maps in both directions - orig_node_id_to_new_id_map[new_node_id] = remaining_nodes[new_node_id].id; + orig_node_id_from_new_node_id_map[new_node_id] = remaining_nodes[new_node_id].id; new_node_id_from_orig_id_map[remaining_nodes[new_node_id].id] = new_node_id; new_node_priority[new_node_id] = node_priorities[remaining_nodes[new_node_id].id]; @@ -566,10 +566,10 @@ class Contractor { const NodeID target = contractor_graph->GetTarget(edge); const ContractorGraph::EdgeData &data = contractor_graph->GetEdgeData(edge); - if (!orig_node_id_to_new_id_map.empty()) + if (!orig_node_id_from_new_node_id_map.empty()) { - new_edge.source = orig_node_id_to_new_id_map[node]; - new_edge.target = orig_node_id_to_new_id_map[target]; + new_edge.source = orig_node_id_from_new_node_id_map[node]; + new_edge.target = orig_node_id_from_new_node_id_map[target]; } else { @@ -580,9 +580,10 @@ class Contractor BOOST_ASSERT_MSG(UINT_MAX != new_edge.target, "Target id invalid"); new_edge.data.distance = data.distance; new_edge.data.shortcut = data.shortcut; - if (!data.is_original_via_node_ID && !orig_node_id_to_new_id_map.empty()) + if (!data.is_original_via_node_ID && !orig_node_id_from_new_node_id_map.empty()) { - new_edge.data.id = orig_node_id_to_new_id_map[data.id]; + // tranlate the _node id_ of the shortcutted node + new_edge.data.id = orig_node_id_from_new_node_id_map[data.id]; } else { @@ -597,10 +598,10 @@ class Contractor } } contractor_graph.reset(); - orig_node_id_to_new_id_map.clear(); - orig_node_id_to_new_id_map.shrink_to_fit(); + orig_node_id_from_new_node_id_map.clear(); + orig_node_id_from_new_node_id_map.shrink_to_fit(); - BOOST_ASSERT(0 == orig_node_id_to_new_id_map.capacity()); + BOOST_ASSERT(0 == orig_node_id_from_new_node_id_map.capacity()); edges.append(external_edge_list.begin(), external_edge_list.end()); external_edge_list.clear(); @@ -958,7 +959,7 @@ class Contractor std::shared_ptr contractor_graph; stxxl::vector external_edge_list; - std::vector orig_node_id_to_new_id_map; + std::vector orig_node_id_from_new_node_id_map; XORFastHash fast_hash; }; From ddff9b612f32136d231cfe33647fd9ab43b36d21 Mon Sep 17 00:00:00 2001 From: Patrick Niklaus Date: Sat, 8 Aug 2015 15:28:05 +0200 Subject: [PATCH 037/122] Serialize out .core file containing core node markers --- contractor/contractor.hpp | 16 ++++++++++++++++ contractor/contractor_options.cpp | 1 + contractor/contractor_options.hpp | 1 + contractor/processing_chain.cpp | 21 +++++++++++++++++++-- contractor/processing_chain.hpp | 4 +++- 5 files changed, 40 insertions(+), 3 deletions(-) diff --git a/contractor/contractor.hpp b/contractor/contractor.hpp index dfdc0c2beec..a2d7941ee1c 100644 --- a/contractor/contractor.hpp +++ b/contractor/contractor.hpp @@ -306,6 +306,7 @@ class Contractor std::vector remaining_nodes(number_of_nodes); std::vector node_priorities(number_of_nodes); std::vector node_data(number_of_nodes); + is_core_node.resize(number_of_nodes, false); // initialize priorities in parallel tbb::parallel_for(tbb::blocked_range(0, number_of_nodes, InitGrainSize), @@ -546,11 +547,25 @@ class Contractor p.printStatus(number_of_contracted_nodes); } + if (remaining_nodes.size() > 2) + { + for (const auto& node : remaining_nodes) + { + auto orig_id = orig_node_id_from_new_node_id_map[node.id]; + is_core_node[orig_id] = true; + } + } + SimpleLogger().Write() << "[core] " << remaining_nodes.size() << " nodes " << contractor_graph->GetNumberOfEdges() << " edges." << std::endl; thread_data_list.data.clear(); } + inline void GetCoreMarker(std::vector &out_is_core_node) + { + out_is_core_node.swap(is_core_node); + } + template inline void GetEdges(DeallocatingVector &edges) { Percent p(contractor_graph->GetNumberOfNodes()); @@ -960,6 +975,7 @@ class Contractor std::shared_ptr contractor_graph; stxxl::vector external_edge_list; std::vector orig_node_id_from_new_node_id_map; + std::vector is_core_node; XORFastHash fast_hash; }; diff --git a/contractor/contractor_options.cpp b/contractor/contractor_options.cpp index cd304bc3c77..f0d434a9162 100644 --- a/contractor/contractor_options.cpp +++ b/contractor/contractor_options.cpp @@ -131,6 +131,7 @@ ContractorOptions::ParseArguments(int argc, char *argv[], ContractorConfig &cont void ContractorOptions::GenerateOutputFilesNames(ContractorConfig &contractor_config) { contractor_config.node_output_path = contractor_config.osrm_input_path.string() + ".nodes"; + contractor_config.core_output_path = contractor_config.osrm_input_path.string() + ".core"; contractor_config.edge_output_path = contractor_config.osrm_input_path.string() + ".edges"; contractor_config.geometry_output_path = contractor_config.osrm_input_path.string() + ".geometry"; contractor_config.graph_output_path = contractor_config.osrm_input_path.string() + ".hsgr"; diff --git a/contractor/contractor_options.hpp b/contractor/contractor_options.hpp index 248659527c8..3836627d53c 100644 --- a/contractor/contractor_options.hpp +++ b/contractor/contractor_options.hpp @@ -49,6 +49,7 @@ struct ContractorConfig boost::filesystem::path profile_path; std::string node_output_path; + std::string core_output_path; std::string edge_output_path; std::string geometry_output_path; std::string graph_output_path; diff --git a/contractor/processing_chain.cpp b/contractor/processing_chain.cpp index 77e13b08904..c205ac79361 100644 --- a/contractor/processing_chain.cpp +++ b/contractor/processing_chain.cpp @@ -106,8 +106,9 @@ int Prepare::Run() // Contracting the edge-expanded graph TIMER_START(contraction); + std::vector is_core_node; auto contracted_edge_list = osrm::make_unique>(); - ContractGraph(max_edge_id, edge_based_edge_list, *contracted_edge_list); + ContractGraph(max_edge_id, edge_based_edge_list, *contracted_edge_list, is_core_node); TIMER_STOP(contraction); SimpleLogger().Write() << "Contraction took " << TIMER_SEC(contraction) << " sec"; @@ -115,6 +116,7 @@ int Prepare::Run() std::size_t number_of_used_edges = WriteContractedGraph(max_edge_id, std::move(node_based_edge_list), std::move(contracted_edge_list)); + WriteCoreNodeMarker(std::move(is_core_node)); TIMER_STOP(preparing); @@ -202,6 +204,19 @@ void Prepare::FindComponents(unsigned max_edge_id, const DeallocatingVector&& in_is_core_node) const +{ + std::vector is_core_node(in_is_core_node); + std::vector unpacked_bool_flags(is_core_node.size()); + for (auto i = 0u; i < is_core_node.size(); ++i) + { + unpacked_bool_flags[i] = is_core_node[i] ? 1 : 0; + } + + boost::filesystem::ofstream core_marker_output_stream(config.core_output_path, std::ios::binary); + core_marker_output_stream.write((char *)unpacked_bool_flags.data(), sizeof(char)*unpacked_bool_flags.size()); +} + std::size_t Prepare::WriteContractedGraph(unsigned max_node_id, std::unique_ptr> node_based_edge_list, std::unique_ptr> contracted_edge_list) @@ -482,11 +497,13 @@ Prepare::BuildEdgeExpandedGraph(std::vector &internal_to_external_nod */ void Prepare::ContractGraph(const unsigned max_edge_id, DeallocatingVector& edge_based_edge_list, - DeallocatingVector& contracted_edge_list) + DeallocatingVector& contracted_edge_list, + std::vector& is_core_node) { Contractor contractor(max_edge_id + 1, edge_based_edge_list); contractor.Run(config.core_factor); contractor.GetEdges(contracted_edge_list); + contractor.GetCoreMarker(is_core_node); } /** diff --git a/contractor/processing_chain.hpp b/contractor/processing_chain.hpp index 33dbe917d8a..63560ebf9ea 100644 --- a/contractor/processing_chain.hpp +++ b/contractor/processing_chain.hpp @@ -63,7 +63,9 @@ class Prepare unsigned CalculateEdgeChecksum(std::unique_ptr> node_based_edge_list); void ContractGraph(const unsigned max_edge_id, DeallocatingVector& edge_based_edge_list, - DeallocatingVector& contracted_edge_list); + DeallocatingVector& contracted_edge_list, + std::vector& is_core_node); + void WriteCoreNodeMarker(std::vector&& is_core_node) const; std::size_t WriteContractedGraph(unsigned number_of_edge_based_nodes, std::unique_ptr> node_based_edge_list, std::unique_ptr> contracted_edge_list); From 707dd700b0e3fe40fc80e2df03146a20c4683bf7 Mon Sep 17 00:00:00 2001 From: Patrick Niklaus Date: Sun, 9 Aug 2015 18:30:04 +0200 Subject: [PATCH 038/122] Write number of markers to .core file --- contractor/processing_chain.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/contractor/processing_chain.cpp b/contractor/processing_chain.cpp index c205ac79361..15c4e00778b 100644 --- a/contractor/processing_chain.cpp +++ b/contractor/processing_chain.cpp @@ -214,6 +214,8 @@ void Prepare::WriteCoreNodeMarker(std::vector&& in_is_core_node) const } boost::filesystem::ofstream core_marker_output_stream(config.core_output_path, std::ios::binary); + unsigned size = unpacked_bool_flags.size(); + core_marker_output_stream.write((char *)&size, sizeof(unsigned)); core_marker_output_stream.write((char *)unpacked_bool_flags.data(), sizeof(char)*unpacked_bool_flags.size()); } From 9387f583fa5de0c6490214822bcc33181cc0a9b5 Mon Sep 17 00:00:00 2001 From: Patrick Niklaus Date: Sun, 9 Aug 2015 18:31:11 +0200 Subject: [PATCH 039/122] Add loading of .core file to InternalDataFacade --- server/data_structures/datafacade_base.hpp | 2 ++ .../data_structures/internal_datafacade.hpp | 33 +++++++++++++++++++ server/data_structures/shared_datafacade.hpp | 7 ++++ util/routed_options.hpp | 2 ++ 4 files changed, 44 insertions(+) diff --git a/server/data_structures/datafacade_base.hpp b/server/data_structures/datafacade_base.hpp index 20d04305768..fc4519da2fc 100644 --- a/server/data_structures/datafacade_base.hpp +++ b/server/data_structures/datafacade_base.hpp @@ -113,6 +113,8 @@ template class BaseDataFacade virtual unsigned GetCheckSum() const = 0; + virtual bool IsCoreNode(const NodeID id) const = 0; + virtual unsigned GetNameIndexFromEdgeID(const unsigned id) const = 0; virtual std::string get_name_for_id(const unsigned name_id) const = 0; diff --git a/server/data_structures/internal_datafacade.hpp b/server/data_structures/internal_datafacade.hpp index 7439cb8318a..3493ebdab53 100644 --- a/server/data_structures/internal_datafacade.hpp +++ b/server/data_structures/internal_datafacade.hpp @@ -73,6 +73,7 @@ template class InternalDataFacade final : public BaseDataFacad ShM::vector m_edge_is_compressed; ShM::vector m_geometry_indices; ShM::vector m_geometry_list; + ShM::vector m_is_core_node; boost::thread_specific_ptr< StaticRTree::vector, false>> m_static_rtree; @@ -171,6 +172,23 @@ template class InternalDataFacade final : public BaseDataFacad edges_input_stream.close(); } + void LoadCoreInformation(const boost::filesystem::path &core_data_file) + { + std::ifstream core_stream(core_data_file.string().c_str(), std::ios::binary); + unsigned number_of_markers; + core_stream.read((char *)&number_of_markers, sizeof(unsigned)); + + std::vector unpacked_core_markers(number_of_markers); + core_stream.read((char *)unpacked_core_markers.data(), sizeof(char)*number_of_markers); + + m_is_core_node.resize(number_of_markers); + for (auto i = 0u; i < number_of_markers; ++i) + { + BOOST_ASSERT(unpacked_core_markers[i] == 0 || unpacked_core_markers[i] == 1); + m_is_core_node[i] = unpacked_core_markers[i] == 1; + } + } + void LoadGeometries(const boost::filesystem::path &geometry_file) { std::ifstream geometry_stream(geometry_file.string().c_str(), std::ios::binary); @@ -255,6 +273,10 @@ template class InternalDataFacade final : public BaseDataFacad { throw osrm::exception("no nodes file given in ini file"); } + if (server_paths.find("coredata") == server_paths.end()) + { + throw osrm::exception("no core file given in ini file"); + } if (server_paths.find("edgesdata") == server_paths.end()) { throw osrm::exception("no edges file given in ini file"); @@ -288,6 +310,9 @@ template class InternalDataFacade final : public BaseDataFacad paths_iterator = server_paths.find("geometries"); BOOST_ASSERT(server_paths.end() != paths_iterator); const boost::filesystem::path &geometries_path = paths_iterator->second; + paths_iterator = server_paths.find("coredata"); + BOOST_ASSERT(server_paths.end() != paths_iterator); + const boost::filesystem::path &core_data_path = paths_iterator->second; // load data SimpleLogger().Write() << "loading graph data"; @@ -297,6 +322,9 @@ template class InternalDataFacade final : public BaseDataFacad AssertPathExists(nodes_data_path); AssertPathExists(edges_data_path); LoadNodeAndEdgeInformation(nodes_data_path, edges_data_path); + SimpleLogger().Write() << "loading core information"; + AssertPathExists(core_data_path); + LoadCoreInformation(core_data_path); SimpleLogger().Write() << "loading geometries"; AssertPathExists(geometries_path); LoadGeometries(geometries_path); @@ -464,6 +492,11 @@ template class InternalDataFacade final : public BaseDataFacad return m_via_node_list.at(id); } + virtual bool IsCoreNode(const NodeID id) const override final + { + return m_is_core_node[id]; + } + virtual void GetUncompressedGeometry(const unsigned id, std::vector &result_nodes) const override final { diff --git a/server/data_structures/shared_datafacade.hpp b/server/data_structures/shared_datafacade.hpp index b1564231606..373da7fff8b 100644 --- a/server/data_structures/shared_datafacade.hpp +++ b/server/data_structures/shared_datafacade.hpp @@ -84,6 +84,7 @@ template class SharedDataFacade final : public BaseDataFacade< ShM::vector m_edge_is_compressed; ShM::vector m_geometry_indices; ShM::vector m_geometry_list; + ShM::vector m_is_core_node; boost::thread_specific_ptr>> m_static_rtree; boost::filesystem::path file_index_path; @@ -447,6 +448,12 @@ template class SharedDataFacade final : public BaseDataFacade< return result; } + bool IsCoreNode(const NodeID id) const override final + { + //return m_is_core_node[id]; + return false; + } + std::string GetTimestamp() const override final { return m_timestamp; } }; diff --git a/util/routed_options.hpp b/util/routed_options.hpp index 106ea23f2ef..5701514affb 100644 --- a/util/routed_options.hpp +++ b/util/routed_options.hpp @@ -59,6 +59,8 @@ inline void populate_base_path(ServerPaths &server_paths) BOOST_ASSERT(server_paths.find("hsgrdata") != server_paths.end()); server_paths["nodesdata"] = base_string + ".nodes"; BOOST_ASSERT(server_paths.find("nodesdata") != server_paths.end()); + server_paths["coredata"] = base_string + ".core"; + BOOST_ASSERT(server_paths.find("coredata") != server_paths.end()); server_paths["edgesdata"] = base_string + ".edges"; BOOST_ASSERT(server_paths.find("edgesdata") != server_paths.end()); server_paths["geometries"] = base_string + ".geometry"; From 7cc875b8db27bddc9247f7da5ffdd78b3d9d3d26 Mon Sep 17 00:00:00 2001 From: Patrick Niklaus Date: Sun, 9 Aug 2015 18:31:46 +0200 Subject: [PATCH 040/122] Initial version of core based search --- routing_algorithms/direct_shortest_path.hpp | 110 +++++++++++++++++--- 1 file changed, 96 insertions(+), 14 deletions(-) diff --git a/routing_algorithms/direct_shortest_path.hpp b/routing_algorithms/direct_shortest_path.hpp index 45515f2616a..80a9c5d1c11 100644 --- a/routing_algorithms/direct_shortest_path.hpp +++ b/routing_algorithms/direct_shortest_path.hpp @@ -64,10 +64,15 @@ class DirectShortestPathRouting final { engine_working_data.InitializeOrClearFirstThreadLocalStorage( super::facade->GetNumberOfNodes()); + engine_working_data.InitializeOrClearSecondThreadLocalStorage( + super::facade->GetNumberOfNodes()); QueryHeap &forward_heap = *(engine_working_data.forward_heap_1); QueryHeap &reverse_heap = *(engine_working_data.reverse_heap_1); + QueryHeap &forward_core_heap = *(engine_working_data.forward_heap_2); + QueryHeap &reverse_core_heap = *(engine_working_data.reverse_heap_2); + // Get distance to next pair of target nodes. BOOST_ASSERT_MSG(1 == phantom_nodes_vector.size(), "Direct Shortest Path Query only accepts a single source and target pair. Multiple ones have been specified."); @@ -114,23 +119,89 @@ class DirectShortestPathRouting final phantom_node_pair.target_phantom.reverse_node_id); } + std::vector> forward_entry_points; + std::vector> reverse_entry_points; + // run two-Target Dijkstra routing step. while (0 < (forward_heap.Size() + reverse_heap.Size()) ) { if (!forward_heap.Empty()) { - super::RoutingStep(forward_heap, reverse_heap, &middle, &distance, - min_edge_offset, true); + if (super::facade->IsCoreNode(forward_heap.Min())) + { + const NodeID node = forward_heap.DeleteMin(); + const int key = forward_heap.GetKey(node); + forward_entry_points.emplace_back(node, key); + } + else + { + super::RoutingStep(forward_heap, reverse_heap, &middle, &distance, + min_edge_offset, true); + } } if (!reverse_heap.Empty()) { - super::RoutingStep(reverse_heap, forward_heap, &middle, &distance, + if (super::facade->IsCoreNode(reverse_heap.Min())) + { + const NodeID node = reverse_heap.DeleteMin(); + const int key = reverse_heap.GetKey(node); + reverse_entry_points.emplace_back(node, key); + } + else + { + super::RoutingStep(reverse_heap, forward_heap, &middle, &distance, + min_edge_offset, false); + } + } + } + + // TODO check if unordered_set might be faster + // sort by id and increasing by distance + auto entry_point_comparator = [](const std::pair& lhs, const std::pair& rhs) + { + return lhs.first < rhs.first || (lhs.first == rhs.first && lhs.second < rhs.second); + }; + std::sort(forward_entry_points.begin(), forward_entry_points.end(), entry_point_comparator); + std::sort(reverse_entry_points.begin(), reverse_entry_points.end(), entry_point_comparator); + + NodeID last_id = SPECIAL_NODEID; + for (const auto p : forward_entry_points) + { + if (p.first == last_id) + { + continue; + } + forward_core_heap.Insert(p.first, p.second, p.first); + last_id = p.first; + } + last_id = SPECIAL_NODEID; + for (const auto p : reverse_entry_points) + { + if (p.first == last_id) + { + continue; + } + reverse_core_heap.Insert(p.first, p.second, p.first); + last_id = p.first; + } + + // run two-target Dijkstra routing step on core with termination criterion + while (distance > (forward_core_heap.MinKey() + reverse_core_heap.MinKey()) ) + { + if (!forward_core_heap.Empty()) + { + super::RoutingStep(forward_core_heap, reverse_core_heap, &middle, &distance, + min_edge_offset, true); + } + if (!reverse_core_heap.Empty()) + { + super::RoutingStep(reverse_core_heap, forward_core_heap, &middle, &distance, min_edge_offset, false); } } // No path found for both target nodes? - if ((INVALID_EDGE_WEIGHT == distance)) + if (INVALID_EDGE_WEIGHT == distance) { raw_route_data.shortest_path_length = INVALID_EDGE_WEIGHT; raw_route_data.alternative_path_length = INVALID_EDGE_WEIGHT; @@ -141,22 +212,33 @@ class DirectShortestPathRouting final BOOST_ASSERT_MSG((SPECIAL_NODEID == middle || INVALID_EDGE_WEIGHT != distance), "no path found"); - // Unpack paths if they exist std::vector packed_leg; - if (INVALID_EDGE_WEIGHT != distance) + // we need to unpack sub path from core heaps + if(super::facade->IsCoreNode(middle)) + { + std::vector packed_core_leg; + super::RetrievePackedPathFromHeap(forward_core_heap, reverse_core_heap, middle, packed_core_leg); + BOOST_ASSERT(packed_core_leg.size() > 0); + super::RetrievePackedPathFromSingleHeap(forward_heap, packed_core_leg.front(), packed_leg); + std::reverse(packed_leg.begin(), packed_leg.end()); + packed_leg.insert(packed_leg.end(), packed_core_leg.begin(), packed_core_leg.end()); + super::RetrievePackedPathFromSingleHeap(reverse_heap, packed_core_leg.back(), packed_leg); + } + else { super::RetrievePackedPathFromHeap(forward_heap, reverse_heap, middle, packed_leg); + } - BOOST_ASSERT_MSG(!packed_leg.empty(), "packed path empty"); - raw_route_data.unpacked_path_segments.resize(1); - raw_route_data.source_traversed_in_reverse.push_back( - (packed_leg.front() != phantom_node_pair.source_phantom.forward_node_id)); - raw_route_data.target_traversed_in_reverse.push_back( - (packed_leg.back() != phantom_node_pair.target_phantom.forward_node_id)); + BOOST_ASSERT_MSG(!packed_leg.empty(), "packed path empty"); - super::UnpackPath(packed_leg, phantom_node_pair, raw_route_data.unpacked_path_segments.front()); - } + raw_route_data.unpacked_path_segments.resize(1); + raw_route_data.source_traversed_in_reverse.push_back( + (packed_leg.front() != phantom_node_pair.source_phantom.forward_node_id)); + raw_route_data.target_traversed_in_reverse.push_back( + (packed_leg.back() != phantom_node_pair.target_phantom.forward_node_id)); + + super::UnpackPath(packed_leg, phantom_node_pair, raw_route_data.unpacked_path_segments.front()); raw_route_data.shortest_path_length = distance; } From 2ff2ce460c662d555ab28a4044999a4846e17cea Mon Sep 17 00:00:00 2001 From: Patrick Niklaus Date: Mon, 10 Aug 2015 00:50:51 +0200 Subject: [PATCH 041/122] Add .core to cucumber renaming --- features/support/data.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/features/support/data.rb b/features/support/data.rb index 1ff186181ff..0c70bb6a595 100644 --- a/features/support/data.rb +++ b/features/support/data.rb @@ -296,7 +296,7 @@ def prepare_data raise PrepareError.new $?.exitstatus, "osrm-prepare exited with code #{$?.exitstatus}." end begin - ["osrm.hsgr","osrm.fileIndex","osrm.geometry","osrm.nodes","osrm.ramIndex"].each do |file| + ["osrm.hsgr","osrm.fileIndex","osrm.geometry","osrm.nodes","osrm.ramIndex","osrm.core"].each do |file| File.rename "#{extracted_file}.#{file}", "#{prepared_file}.#{file}" end rescue Exception => e From 48d1a5ec5d63b962d7fde02c75a529adfa88b2fa Mon Sep 17 00:00:00 2001 From: Patrick Niklaus Date: Mon, 10 Aug 2015 00:51:05 +0200 Subject: [PATCH 042/122] Make sure to terminate when the core heaps are empty --- routing_algorithms/direct_shortest_path.hpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/routing_algorithms/direct_shortest_path.hpp b/routing_algorithms/direct_shortest_path.hpp index 80a9c5d1c11..2abee9874d4 100644 --- a/routing_algorithms/direct_shortest_path.hpp +++ b/routing_algorithms/direct_shortest_path.hpp @@ -186,7 +186,8 @@ class DirectShortestPathRouting final } // run two-target Dijkstra routing step on core with termination criterion - while (distance > (forward_core_heap.MinKey() + reverse_core_heap.MinKey()) ) + while (0 < (forward_core_heap.Size() + reverse_core_heap.Size()) && + distance > (forward_core_heap.MinKey() + reverse_core_heap.MinKey())) { if (!forward_core_heap.Empty()) { From 92956f2b455febba157758e707e4a0bb1aa9e9c9 Mon Sep 17 00:00:00 2001 From: Patrick Niklaus Date: Wed, 19 Aug 2015 11:44:49 +0200 Subject: [PATCH 043/122] Also support loading core information into shared memory --- contractor/contractor.hpp | 7 +++ datastore.cpp | 44 ++++++++++++++++++ .../data_structures/internal_datafacade.hpp | 15 ++++++- server/data_structures/shared_datafacade.hpp | 22 ++++++++- server/data_structures/shared_datatype.hpp | 45 +++---------------- util/datastore_options.hpp | 10 +++++ 6 files changed, 103 insertions(+), 40 deletions(-) diff --git a/contractor/contractor.hpp b/contractor/contractor.hpp index a2d7941ee1c..f4f2c95e8e6 100644 --- a/contractor/contractor.hpp +++ b/contractor/contractor.hpp @@ -549,12 +549,19 @@ class Contractor if (remaining_nodes.size() > 2) { + // TODO: for small cores a sorted array of core ids might also work good for (const auto& node : remaining_nodes) { auto orig_id = orig_node_id_from_new_node_id_map[node.id]; is_core_node[orig_id] = true; } } + else + { + // in this case we don't need core markers since we fully contracted + // the graph + is_core_node.clear(); + } SimpleLogger().Write() << "[core] " << remaining_nodes.size() << " nodes " << contractor_graph->GetNumberOfEdges() << " edges." << std::endl; diff --git a/datastore.cpp b/datastore.cpp index 89efefcb2e5..0d194992c37 100644 --- a/datastore.cpp +++ b/datastore.cpp @@ -164,6 +164,10 @@ int main(const int argc, const char *argv[]) { throw osrm::exception("no geometry file found"); } + if (server_paths.find("core") == server_paths.end()) + { + throw osrm::exception("no core file found"); + } ServerPaths::const_iterator paths_iterator = server_paths.find("hsgrdata"); BOOST_ASSERT(server_paths.end() != paths_iterator); @@ -199,6 +203,10 @@ int main(const int argc, const char *argv[]) BOOST_ASSERT(server_paths.end() != paths_iterator); BOOST_ASSERT(!paths_iterator->second.empty()); const boost::filesystem::path &geometries_data_path = paths_iterator->second; + paths_iterator = server_paths.find("core"); + BOOST_ASSERT(server_paths.end() != paths_iterator); + BOOST_ASSERT(!paths_iterator->second.empty()); + const boost::filesystem::path &core_marker_path = paths_iterator->second; // determine segment to use bool segment2_in_use = SharedMemory::RegionExists(LAYOUT_2); @@ -329,6 +337,13 @@ int main(const int argc, const char *argv[]) } shared_layout_ptr->SetBlockSize(SharedDataLayout::TIMESTAMP, m_timestamp.length()); + // load core marker size + boost::filesystem::ifstream core_marker_file(core_marker_path, std::ios::binary); + + uint32_t number_of_core_markers = 0; + core_marker_file.read((char *)&number_of_core_markers, sizeof(uint32_t)); + shared_layout_ptr->SetBlockSize(SharedDataLayout::CORE_MARKER, number_of_core_markers); + // load coordinate size boost::filesystem::ifstream nodes_input_stream(nodes_data_path, std::ios::binary); unsigned coordinate_list_size = 0; @@ -509,6 +524,35 @@ int main(const int argc, const char *argv[]) } tree_node_file.close(); + // load core markers + std::vector unpacked_core_markers(number_of_core_markers); + core_marker_file.read((char *)unpacked_core_markers.data(), sizeof(char)*number_of_core_markers); + + unsigned *core_marker_ptr = shared_layout_ptr->GetBlockPtr( + shared_memory_ptr, SharedDataLayout::CORE_MARKER); + + for (auto i = 0u; i < number_of_core_markers; ++i) + { + BOOST_ASSERT(unpacked_core_markers[i] == 0 || unpacked_core_markers[i] == 1); + + if (unpacked_core_markers[i] == 1) + { + const unsigned bucket = i / 32; + const unsigned offset = i % 32; + const unsigned value = [&] + { + unsigned return_value = 0; + if (0 != offset) + { + return_value = core_marker_ptr[bucket]; + } + return return_value; + }(); + + core_marker_ptr[bucket] = (value | (1 << offset)); + } + } + // load the nodes of the search graph QueryGraph::NodeArrayEntry *graph_node_list_ptr = shared_layout_ptr->GetBlockPtr( diff --git a/server/data_structures/internal_datafacade.hpp b/server/data_structures/internal_datafacade.hpp index 3493ebdab53..0678b09370d 100644 --- a/server/data_structures/internal_datafacade.hpp +++ b/server/data_structures/internal_datafacade.hpp @@ -181,6 +181,12 @@ template class InternalDataFacade final : public BaseDataFacad std::vector unpacked_core_markers(number_of_markers); core_stream.read((char *)unpacked_core_markers.data(), sizeof(char)*number_of_markers); + // in this case we have nothing to do + if (number_of_markers <= 0) + { + return; + } + m_is_core_node.resize(number_of_markers); for (auto i = 0u; i < number_of_markers; ++i) { @@ -494,7 +500,14 @@ template class InternalDataFacade final : public BaseDataFacad virtual bool IsCoreNode(const NodeID id) const override final { - return m_is_core_node[id]; + if (m_is_core_node.size() > 0) + { + return m_is_core_node[id]; + } + else + { + return false; + } } virtual void GetUncompressedGeometry(const unsigned id, diff --git a/server/data_structures/shared_datafacade.hpp b/server/data_structures/shared_datafacade.hpp index 373da7fff8b..d9d907a0abc 100644 --- a/server/data_structures/shared_datafacade.hpp +++ b/server/data_structures/shared_datafacade.hpp @@ -194,6 +194,21 @@ template class SharedDataFacade final : public BaseDataFacade< m_names_char_list.swap(names_char_list); } + void LoadCoreInformation() + { + if (data_layout->num_entries[SharedDataLayout::CORE_MARKER] <= 0) + { + return; + } + + unsigned *core_marker_ptr = data_layout->GetBlockPtr( + shared_memory, SharedDataLayout::CORE_MARKER); + typename ShM::vector is_core_node( + core_marker_ptr, + data_layout->num_entries[SharedDataLayout::CORE_MARKER]); + m_is_core_node.swap(is_core_node); + } + void LoadGeometries() { unsigned *geometries_compressed_ptr = data_layout->GetBlockPtr( @@ -269,6 +284,7 @@ template class SharedDataFacade final : public BaseDataFacade< LoadTimestamp(); LoadViaNodeList(); LoadNames(); + LoadCoreInformation(); data_layout->PrintInformation(); @@ -450,7 +466,11 @@ template class SharedDataFacade final : public BaseDataFacade< bool IsCoreNode(const NodeID id) const override final { - //return m_is_core_node[id]; + if (m_is_core_node.size() > 0) + { + return m_is_core_node.at(id); + } + return false; } diff --git a/server/data_structures/shared_datatype.hpp b/server/data_structures/shared_datatype.hpp index 326157389bd..c0ad7d45be9 100644 --- a/server/data_structures/shared_datatype.hpp +++ b/server/data_structures/shared_datatype.hpp @@ -62,6 +62,7 @@ struct SharedDataLayout HSGR_CHECKSUM, TIMESTAMP, FILE_INDEX_PATH, + CORE_MARKER, NUM_BLOCKS }; @@ -72,40 +73,6 @@ struct SharedDataLayout void PrintInformation() const { - SimpleLogger().Write(logDEBUG) << "-"; - SimpleLogger().Write(logDEBUG) - << "name_offsets_size: " << num_entries[NAME_OFFSETS]; - SimpleLogger().Write(logDEBUG) - << "name_blocks_size: " << num_entries[NAME_BLOCKS]; - SimpleLogger().Write(logDEBUG) - << "name_char_list_size: " << num_entries[NAME_CHAR_LIST]; - SimpleLogger().Write(logDEBUG) - << "name_id_list_size: " << num_entries[NAME_ID_LIST]; - SimpleLogger().Write(logDEBUG) - << "via_node_list_size: " << num_entries[VIA_NODE_LIST]; - SimpleLogger().Write(logDEBUG) - << "graph_node_list_size: " << num_entries[GRAPH_NODE_LIST]; - SimpleLogger().Write(logDEBUG) - << "graph_edge_list_size: " << num_entries[GRAPH_EDGE_LIST]; - SimpleLogger().Write(logDEBUG) << "timestamp_length: " << num_entries[TIMESTAMP]; - SimpleLogger().Write(logDEBUG) - << "coordinate_list_size: " << num_entries[COORDINATE_LIST]; - SimpleLogger().Write(logDEBUG) - << "turn_instruction_list_size: " << num_entries[TURN_INSTRUCTION]; - SimpleLogger().Write(logDEBUG) - << "travel_mode_list_size: " << num_entries[TRAVEL_MODE]; - SimpleLogger().Write(logDEBUG) - << "r_search_tree_size: " << num_entries[R_SEARCH_TREE]; - SimpleLogger().Write(logDEBUG) - << "geometries_indicators: " << num_entries[GEOMETRIES_INDICATORS] << "/" - << ((num_entries[GEOMETRIES_INDICATORS] / 8) + 1); - SimpleLogger().Write(logDEBUG) - << "geometries_index_list_size: " << num_entries[GEOMETRIES_INDEX]; - SimpleLogger().Write(logDEBUG) - << "geometries_list_size: " << num_entries[GEOMETRIES_LIST]; - SimpleLogger().Write(logDEBUG) - << "sizeof(checksum): " << entry_size[HSGR_CHECKSUM]; - SimpleLogger().Write(logDEBUG) << "NAME_OFFSETS " << ": " << GetBlockSize(NAME_OFFSETS); SimpleLogger().Write(logDEBUG) << "NAME_BLOCKS " @@ -140,6 +107,8 @@ struct SharedDataLayout << ": " << GetBlockSize(TIMESTAMP); SimpleLogger().Write(logDEBUG) << "FILE_INDEX_PATH " << ": " << GetBlockSize(FILE_INDEX_PATH); + SimpleLogger().Write(logDEBUG) << "CORE_MARKER " + << ": " << GetBlockSize(CORE_MARKER); } template inline void SetBlockSize(BlockID bid, uint64_t entries) @@ -150,11 +119,11 @@ struct SharedDataLayout inline uint64_t GetBlockSize(BlockID bid) const { - // special encoding - if (bid == GEOMETRIES_INDICATORS) + // special bit encoding + if (bid == GEOMETRIES_INDICATORS || bid == CORE_MARKER) { - return (num_entries[GEOMETRIES_INDICATORS] / 32 + 1) * - entry_size[GEOMETRIES_INDICATORS]; + return (num_entries[bid] / 32 + 1) * + entry_size[bid]; } return num_entries[bid] * entry_size[bid]; diff --git a/util/datastore_options.hpp b/util/datastore_options.hpp index 9f16c5757ee..4cff3d3e118 100644 --- a/util/datastore_options.hpp +++ b/util/datastore_options.hpp @@ -69,6 +69,8 @@ bool GenerateDataStoreOptions(const int argc, const char *argv[], ServerPaths &p ".ramIndex file")( "fileindex", boost::program_options::value(&paths["fileindex"]), ".fileIndex file")( + "core", boost::program_options::value(&paths["core"]), + ".core file")( "namesdata", boost::program_options::value(&paths["namesdata"]), ".names file")("timestamp", boost::program_options::value(&paths["timestamp"]), @@ -130,6 +132,8 @@ bool GenerateDataStoreOptions(const int argc, const char *argv[], ServerPaths &p !paths.find("ramindex")->second.string().empty()) || (paths.find("fileindex") != paths.end() && !paths.find("fileindex")->second.string().empty()) || + (paths.find("core") != paths.end() && + !paths.find("core")->second.string().empty()) || (paths.find("timestamp") != paths.end() && !paths.find("timestamp")->second.string().empty()); @@ -199,6 +203,12 @@ bool GenerateDataStoreOptions(const int argc, const char *argv[], ServerPaths &p path_iterator->second = base_string + ".fileIndex"; } + path_iterator = paths.find("core"); + if (path_iterator != paths.end()) + { + path_iterator->second = base_string + ".core"; + } + path_iterator = paths.find("namesdata"); if (path_iterator != paths.end()) { From bbd0239ece1e837616cfd321c181cc8d47b637db Mon Sep 17 00:00:00 2001 From: Patrick Niklaus Date: Thu, 20 Aug 2015 12:28:14 +0200 Subject: [PATCH 044/122] Fix Coverity warning in EBGF --- contractor/edge_based_graph_factory.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contractor/edge_based_graph_factory.cpp b/contractor/edge_based_graph_factory.cpp index b2ca9778fd2..92f1da9ff85 100644 --- a/contractor/edge_based_graph_factory.cpp +++ b/contractor/edge_based_graph_factory.cpp @@ -47,7 +47,7 @@ EdgeBasedGraphFactory::EdgeBasedGraphFactory( std::shared_ptr restriction_map, const std::vector &node_info_list, SpeedProfileProperties speed_profile) - : m_node_info_list(node_info_list), m_node_based_graph(std::move(node_based_graph)), + : m_max_edge_id(0), m_node_info_list(node_info_list), m_node_based_graph(std::move(node_based_graph)), m_restriction_map(std::move(restriction_map)), m_barrier_nodes(barrier_nodes), m_traffic_lights(traffic_lights), m_compressed_edge_container(compressed_edge_container), speed_profile(std::move(speed_profile)) From 9a0877379c60ca5956c4f90d25287bba05325354 Mon Sep 17 00:00:00 2001 From: "Daniel J. Hofmann" Date: Thu, 20 Aug 2015 16:15:20 +0200 Subject: [PATCH 045/122] Remove dead code. --- contractor/processing_chain.cpp | 3 +-- datastore.cpp | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/contractor/processing_chain.cpp b/contractor/processing_chain.cpp index 15c4e00778b..376c8ac89ac 100644 --- a/contractor/processing_chain.cpp +++ b/contractor/processing_chain.cpp @@ -257,7 +257,7 @@ std::size_t Prepare::WriteContractedGraph(unsigned max_node_id, SimpleLogger().Write() << "Building node array"; StaticGraph::EdgeIterator edge = 0; StaticGraph::EdgeIterator position = 0; - StaticGraph::EdgeIterator last_edge = edge; + StaticGraph::EdgeIterator last_edge; // initializing 'first_edge'-field of nodes: for (const auto node : osrm::irange(0u, max_used_node_id+1)) @@ -295,7 +295,6 @@ std::size_t Prepare::WriteContractedGraph(unsigned max_node_id, // serialize all edges SimpleLogger().Write() << "Building edge array"; - edge = 0; int number_of_used_edges = 0; StaticGraph::EdgeArrayEntry current_edge; diff --git a/datastore.cpp b/datastore.cpp index 0d194992c37..60c2754c305 100644 --- a/datastore.cpp +++ b/datastore.cpp @@ -230,8 +230,7 @@ int main(const int argc, const char *argv[]) // Allocate a memory layout in shared memory, deallocate previous SharedMemory *layout_memory = SharedMemoryFactory::Get(layout_region, sizeof(SharedDataLayout)); - SharedDataLayout *shared_layout_ptr = static_cast(layout_memory->Ptr()); - shared_layout_ptr = new (layout_memory->Ptr()) SharedDataLayout(); + SharedDataLayout *shared_layout_ptr = new (layout_memory->Ptr()) SharedDataLayout(); shared_layout_ptr->SetBlockSize(SharedDataLayout::FILE_INDEX_PATH, file_index_path.length() + 1); From beb2ab9ad560be87d718397b3eca34fd7a55a5f0 Mon Sep 17 00:00:00 2001 From: "Daniel J. Hofmann" Date: Wed, 19 Aug 2015 19:26:07 +0200 Subject: [PATCH 046/122] Add script to update subtree-ed third party dependencies more easily. Note: this updates the subtrees immediately. Discussion: would it make sense to do something along the lines of: $ http --body https://api.github.com/repos/mapbox/variant/releases/latest | jq ".tag_name" "v1.0" And warn the user if the latest release tag is not the tag the update script was called with. Or at least ask for confirmation? --- update_depdendencies.sh | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100755 update_depdendencies.sh diff --git a/update_depdendencies.sh b/update_depdendencies.sh new file mode 100755 index 00000000000..13ecb719103 --- /dev/null +++ b/update_depdendencies.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash + +OSMIUM_REPO=https://github.com/osmcode/libosmium.git +OSMIUM_TAG=v2.3.0 + +VARIANT_REPO=https://github.com/mapbox/variant.git +VARIANT_TAG=v1.0 + + +git subtree pull -P third_party/libosmium/ $OSMIUM_REPO $OSMIUM_TAG --squash +git subtree pull -P third_party/variant/ $VARIANT_REPO $VARIANT_TAG --squash From 3d84dbc73fe808327db40f6afb9e51cc532db7e4 Mon Sep 17 00:00:00 2001 From: "Daniel J. Hofmann" Date: Wed, 19 Aug 2015 20:20:43 +0200 Subject: [PATCH 047/122] Check for releases and request user confirmation before updating subtrees --- update_depdendencies.sh | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/update_depdendencies.sh b/update_depdendencies.sh index 13ecb719103..0783208292f 100755 --- a/update_depdendencies.sh +++ b/update_depdendencies.sh @@ -6,6 +6,16 @@ OSMIUM_TAG=v2.3.0 VARIANT_REPO=https://github.com/mapbox/variant.git VARIANT_TAG=v1.0 +VARIANT_LATEST=$(http --body https://api.github.com/repos/mapbox/variant/releases/latest | jq ".tag_name") +OSMIUM_LATEST=$(http --body https://api.github.com/repos/osmcode/libosmium/releases/latest | jq ".tag_name") -git subtree pull -P third_party/libosmium/ $OSMIUM_REPO $OSMIUM_TAG --squash -git subtree pull -P third_party/variant/ $VARIANT_REPO $VARIANT_TAG --squash +echo "Latest osmium release is $OSMIUM_LATEST, pulling in \"$OSMIUM_TAG\"" +echo "Latest variant release is $VARIANT_LATEST, pulling in \"$VARIANT_TAG\"" + +read -p "Looks good? (Y/n) " ok + +if [[ $ok =~ [yY] ]] +then + echo git subtree pull -P third_party/libosmium/ $OSMIUM_REPO $OSMIUM_TAG --squash + echo git subtree pull -P third_party/variant/ $VARIANT_REPO $VARIANT_TAG --squash +fi From cb4e7614eedfa900c4dc78adf3247d802f2faddd Mon Sep 17 00:00:00 2001 From: "Daniel J. Hofmann" Date: Wed, 19 Aug 2015 20:22:57 +0200 Subject: [PATCH 048/122] Actually do the subtree pull instead of just notifying the user --- update_depdendencies.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/update_depdendencies.sh b/update_depdendencies.sh index 0783208292f..3891d7d8af3 100755 --- a/update_depdendencies.sh +++ b/update_depdendencies.sh @@ -16,6 +16,6 @@ read -p "Looks good? (Y/n) " ok if [[ $ok =~ [yY] ]] then - echo git subtree pull -P third_party/libosmium/ $OSMIUM_REPO $OSMIUM_TAG --squash - echo git subtree pull -P third_party/variant/ $VARIANT_REPO $VARIANT_TAG --squash + git subtree pull -P third_party/libosmium/ $OSMIUM_REPO $OSMIUM_TAG --squash + git subtree pull -P third_party/variant/ $VARIANT_REPO $VARIANT_TAG --squash fi From 2b5aa142fb56cdd037c8efb8157cc3b0ae041fdf Mon Sep 17 00:00:00 2001 From: Wilhelm Berg Date: Mon, 24 Aug 2015 21:23:12 +0200 Subject: [PATCH 049/122] appveyor.yml update url to binary deps --- appveyor-build.bat | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/appveyor-build.bat b/appveyor-build.bat index 41f15c3fd53..9e49f1ae8ad 100644 --- a/appveyor-build.bat +++ b/appveyor-build.bat @@ -13,7 +13,7 @@ IF EXIST %DEPSPKG% DEL %DEPSPKG% IF %ERRORLEVEL% NEQ 0 GOTO ERROR ECHO downloading %DEPSPKG% -powershell Invoke-WebRequest https://mapbox.s3.amazonaws.com/windows-builds/windows-deps/$env:DEPSPKG -OutFile C:\projects\osrm\$env:DEPSPKG +powershell Invoke-WebRequest https://mapbox.s3.amazonaws.com/windows-builds/windows-build-deps/$env:DEPSPKG -OutFile C:\projects\osrm\$env:DEPSPKG IF %ERRORLEVEL% NEQ 0 GOTO ERROR 7z -y x %DEPSPKG% | %windir%\system32\FIND "ing archive" From 0a53dccd4cdf080d9ce86cfd5e4bab87d11ea468 Mon Sep 17 00:00:00 2001 From: Lauren Budorick Date: Fri, 21 Aug 2015 19:01:24 -0700 Subject: [PATCH 050/122] Use .round instead of .to_i for cucumber speeds --- features/bicycle/bridge.feature | 2 +- features/car/bridge.feature | 4 ++-- features/car/maxspeed.feature | 7 +++---- features/step_definitions/routing.rb | 2 +- features/testbot/compression.feature | 4 ++-- 5 files changed, 9 insertions(+), 10 deletions(-) diff --git a/features/bicycle/bridge.feature b/features/bicycle/bridge.feature index 232581647e6..8c26ee99db0 100644 --- a/features/bicycle/bridge.feature +++ b/features/bicycle/bridge.feature @@ -42,6 +42,6 @@ Feature: Bicycle - Handle movable bridge When I route I should get | from | to | route | modes | speed | | a | g | abc,cde,efg | 1,5,1 | 5 km/h | - | b | f | abc,cde,efg | 1,5,1 | 3 km/h | + | b | f | abc,cde,efg | 1,5,1 | 4 km/h | | c | e | cde | 5 | 2 km/h | | e | c | cde | 5 | 2 km/h | diff --git a/features/car/bridge.feature b/features/car/bridge.feature index 41dc10bdb6a..d3e470e412e 100644 --- a/features/car/bridge.feature +++ b/features/car/bridge.feature @@ -41,7 +41,7 @@ Feature: Car - Handle movable bridge When I route I should get | from | to | route | modes | speed | - | a | g | abc,cde,efg | 1,3,1 | 6 km/h | - | b | f | abc,cde,efg | 1,3,1 | 4 km/h | + | a | g | abc,cde,efg | 1,3,1 | 7 km/h | + | b | f | abc,cde,efg | 1,3,1 | 5 km/h | | c | e | cde | 3 | 2 km/h | | e | c | cde | 3 | 2 km/h | diff --git a/features/car/maxspeed.feature b/features/car/maxspeed.feature index 764de728a94..e2d26e0e0b5 100644 --- a/features/car/maxspeed.feature +++ b/features/car/maxspeed.feature @@ -23,10 +23,10 @@ OSRM will use 4/5 of the projected free-flow speed. | from | to | route | speed | | a | b | ab | 78 km/h | | b | c | bc | 59 km/h +- 1 | - | c | d | cd | 50 km/h | + | c | d | cd | 51 km/h | | d | e | de | 75 km/h | | e | f | ef | 90 km/h | - | f | g | fg | 105 km/h | + | f | g | fg | 106 km/h | Scenario: Car - Do not ignore maxspeed when higher than way speed Given the node map @@ -42,7 +42,7 @@ OSRM will use 4/5 of the projected free-flow speed. | from | to | route | speed | | a | b | ab | 31 km/h | | b | c | bc | 83 km/h +- 1 | - | c | d | cd | 50 km/h | + | c | d | cd | 51 km/h | Scenario: Car - Forward/backward maxspeed Given a grid size of 100 meters @@ -119,4 +119,3 @@ OSRM will use 4/5 of the projected free-flow speed. | primary | 30 | 1 | -1 | | 34 km/h | | primary | 30 | 1 | | 15 km/h | 15 km/h | | primary | 30 | 2 | | 34 km/h | 34 km/h | - diff --git a/features/step_definitions/routing.rb b/features/step_definitions/routing.rb index 597ea65d644..6a73d860460 100644 --- a/features/step_definitions/routing.rb +++ b/features/step_definitions/routing.rb @@ -97,7 +97,7 @@ raise "*** Speed must be specied in km/h. (ex: 50 km/h)" unless row['speed'] =~ /\d+ km\/h/ time = json['route_summary']['total_time'] distance = json['route_summary']['total_distance'] - speed = time>0 ? (3.6*distance/time).to_i : nil + speed = time>0 ? (3.6*distance/time).round : nil got['speed'] = "#{speed} km/h" else got['speed'] = '' diff --git a/features/testbot/compression.feature b/features/testbot/compression.feature index 146ad63df7c..4c0121d6f96 100644 --- a/features/testbot/compression.feature +++ b/features/testbot/compression.feature @@ -18,5 +18,5 @@ Feature: Geometry Compression When I route I should get | from | to | route | distance | speed | - | b | e | abcdef | 589m | 35 km/h | - | e | b | abcdef | 589m | 35 km/h | + | b | e | abcdef | 589m | 36 km/h | + | e | b | abcdef | 589m | 36 km/h | From 3e8ef5e462c368135df3dfccad70e2fb1cdf5f8a Mon Sep 17 00:00:00 2001 From: "Daniel J. Hofmann" Date: Tue, 25 Aug 2015 14:24:43 +0200 Subject: [PATCH 051/122] Remove unused `obey_bollards` from profiles, already handled via `barrier_whitelist`. --- profiles/bicycle.lua | 1 - profiles/car.lua | 1 - profiles/examples/postgis.lua | 1 - 3 files changed, 3 deletions(-) diff --git a/profiles/bicycle.lua b/profiles/bicycle.lua index f2360639027..88709ec3c1d 100644 --- a/profiles/bicycle.lua +++ b/profiles/bicycle.lua @@ -94,7 +94,6 @@ traffic_signal_penalty = 2 use_turn_restrictions = false local obey_oneway = true -local obey_bollards = false local ignore_areas = true local u_turn_penalty = 20 local turn_penalty = 60 diff --git a/profiles/car.lua b/profiles/car.lua index 52b11fd2fd4..03d266437be 100644 --- a/profiles/car.lua +++ b/profiles/car.lua @@ -132,7 +132,6 @@ traffic_signal_penalty = 2 use_turn_restrictions = true local obey_oneway = true -local obey_bollards = true local ignore_areas = true local u_turn_penalty = 20 diff --git a/profiles/examples/postgis.lua b/profiles/examples/postgis.lua index 2ff72fdd9a9..d30f35b1a30 100644 --- a/profiles/examples/postgis.lua +++ b/profiles/examples/postgis.lua @@ -33,7 +33,6 @@ print("PostGIS connection opened") -- these settings are read directly by osrm take_minimum_of_speeds = true obey_oneway = true -obey_bollards = true use_restrictions = true ignore_areas = true -- future feature traffic_signal_penalty = 7 -- seconds From db30836b537e1ecc00cbf7a688b6c14fab5ffee9 Mon Sep 17 00:00:00 2001 From: "Daniel J. Hofmann" Date: Tue, 25 Aug 2015 14:46:05 +0200 Subject: [PATCH 052/122] Add rising bollard exception to barriers for car profile. This handles `barrier=bollard` with `bollard=rising`, by making an exception to the barrier whitelist. Barriers tagged as such do no longer require an explicit access tag. This also adds corresponding tests, check this out: cucumber --tags @barrier References: - http://wiki.openstreetmap.org/wiki/Tag:barrier%3Dbollard - http://wiki.openstreetmap.org/wiki/Key:bollard - https://github.com/Project-OSRM/osrm-backend/issues/1616 --- features/car/barrier.feature | 7 +++++++ profiles/car.lua | 6 +++++- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/features/car/barrier.feature b/features/car/barrier.feature index 7c89688c9e8..3f5bef53e36 100644 --- a/features/car/barrier.feature +++ b/features/car/barrier.feature @@ -37,3 +37,10 @@ Feature: Car - Barriers | wall | no | | | wall | private | | | wall | agricultural | | + + Scenario: Car - Rising bollard exception for barriers + Then routability should be + | node/barrier | node/bollard | bothw | + | bollard | | | + | bollard | rising | x | + | bollard | removable | | diff --git a/profiles/car.lua b/profiles/car.lua index 03d266437be..03ca924dc97 100644 --- a/profiles/car.lua +++ b/profiles/car.lua @@ -195,7 +195,11 @@ function node_function (node, result) else local barrier = node:get_value_by_key("barrier") if barrier and "" ~= barrier then - if not barrier_whitelist[barrier] then + -- make an exception for rising bollard barriers + local bollard = node:get_value_by_key("bollard") + local rising_bollard = bollard and "rising" == bollard + + if not barrier_whitelist[barrier] and not rising_bollard then result.barrier = true end end From 6143f1ff5ba78fd8dc99baac74f89f1eeac0ded7 Mon Sep 17 00:00:00 2001 From: bergwerkgis Date: Fri, 21 Aug 2015 10:43:37 +0000 Subject: [PATCH 053/122] AppVeyor: try "os:VS2015" --- appveyor.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/appveyor.yml b/appveyor.yml index cf45a3224f8..2fefebd75c8 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -7,7 +7,7 @@ environment: init: - git config --global core.autocrlf input -os: Visual Studio 2015 RC +os: Visual Studio 2015 # clone directory clone_folder: c:\projects\osrm From 8e1f70865eeacfee21fdd043775207e2ec417140 Mon Sep 17 00:00:00 2001 From: Patrick Niklaus Date: Fri, 28 Aug 2015 12:40:40 +0200 Subject: [PATCH 054/122] Use curl instead of http in update script. --- update_depdendencies.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/update_depdendencies.sh b/update_depdendencies.sh index 3891d7d8af3..aa2ede5fc10 100755 --- a/update_depdendencies.sh +++ b/update_depdendencies.sh @@ -6,8 +6,8 @@ OSMIUM_TAG=v2.3.0 VARIANT_REPO=https://github.com/mapbox/variant.git VARIANT_TAG=v1.0 -VARIANT_LATEST=$(http --body https://api.github.com/repos/mapbox/variant/releases/latest | jq ".tag_name") -OSMIUM_LATEST=$(http --body https://api.github.com/repos/osmcode/libosmium/releases/latest | jq ".tag_name") +VARIANT_LATEST=$(curl https://api.github.com/repos/mapbox/variant/releases/latest | jq ".tag_name") +OSMIUM_LATEST=$(curl https://api.github.com/repos/osmcode/libosmium/releases/latest | jq ".tag_name") echo "Latest osmium release is $OSMIUM_LATEST, pulling in \"$OSMIUM_TAG\"" echo "Latest variant release is $VARIANT_LATEST, pulling in \"$VARIANT_TAG\"" From 788bc67faa7738cf7c6b2a192ecf3e3567d1c20e Mon Sep 17 00:00:00 2001 From: Patrick Niklaus Date: Fri, 28 Aug 2015 12:42:03 +0200 Subject: [PATCH 055/122] Squashed 'third_party/libosmium/' changes from 8bcd4ea..c43f8db c43f8db Release v2.3.0 44c135f Update README to show dependencies used internally. ece54cd Add external licenses. 908cd5f Updated change log. 96dbf0e Change %-escape in OPL format. 98f6e27 Change write benchmark to interleave reading and writing. 39620ce Make writing of metadata configurable for XML and OPL output. e5a4e5e Add debug output format. 597390f Remove superfluous include and pragmas. ecc57b0 Update pbf reader/writer to use new protozero functions. 5d1e8d2 Update protozero from upstream. ef8746b Fix build on Windows. ddba46f Remove superfluous include. 098c57f Add some paranoia checks to pbf reader. 0f804c2 Try building with newer boost library on travis. 6f79d63 Use explicit return types on lambdas. 355f3b1 New PBF reader and writer based on protozero. 71d719b Add pbf writing benchmark. f014b4c Fix iwyu.sh script: Works now if build directory doesn't exist. a0ace49 Use utf8cpp header-only lib instead of boost for utf8 decoding. 796f18e Bugfix: Reading large XML files could block. 5a2bcbe Replace strcmp by std::string comparison in test. bc49e2c Bugfix: XML writer was not writing whitespace correctly. 61222f8 Fix 64bit byte swap. e56f090 Fix new CRC code on OSX and Windows. 70229aa Add low-level building blocks that allow calculating CRC of OSM data. 0968a66 Remove assert checking for unset version. 62e0261 Refactor test case. 4bfc7fc Allow instantiating osmium::geom::GEOSFactory with existing GEOS factory. e70af0c Remove calls to protobuf cleanup function im benchmarks and examples. 718518d Bugfix in OPL output. Relation member roles were not encoded. 759d5cb Rename parameter that had the same name as a type. 7054cab Provide (Typed)MemoryMapping constructors for backwards compatibility. d09f5d1 Fix typo. b4e578f Make memory mapping utility class more flexible. 633fa8e Travis build without sudo. 7ff23f1 Improved code setting file format from suffix/format argument. 90ef3b9 Remove some tests that didn't test much and failed on FreeBSD. af86273 Add some pragmas to disable warnings for GCC. efac7fd Fix some include problems found by IWYU. 79d2f4c Changed add_user() and add_role() in builders. Add add_member(). 9375d00 Add function to set tags from ptr + length. Improve TagBuilder tests. bafca20 Test helper: Use more const and have sub-builders in their own scope. f73c993 Simplify code. fee1710 Disable warning only when compiling with GCC. 74402f3 Merge pull request #98 from dforsi/master 2c4b449 Update to new upstream catch.hpp version. 1318732 Release v2.2.0 1873998 Add missing test. 2e5ea1d Do not add timestamp to html doc pages. 1b2ea89 Remove debug output. 0be9599 Improved parsing of ids, versions, uids, etc. from strings. 4308d80 Add second version of split_string utility function. f18c9e5 Move part of pbf.hpp into new pbf_type_conv.hpp. d201152 Use new DeltaEncode class in pbf writer. e205610 Add DeltaEncode/DeltaDecode utility classes. 32905d6 Bugfix: Actually throw the exception we are creating... d3e86d8 Add functions to convert item_type to zero-based index. daddf07 Bugfix: Programs writing OSM files can stall up to a second after writing. 00b0247 Add function to set the id of a relation member. f85316a Fix error message. 19bc6cc Fix name of travis install script. 719cd33 spatialite-bin package now available on travis cb03821 Shorten long test string (MSVC doesn't like it). c3440a6 Add BoolVector index class. da08073 Add min_op/max_op utility functions. 411d112 AppVeyor.yml: new links for binary deps 7d9095f add test for badly formatted timestamps a073f73 Add helper methods to DiffObject. 3b9819a Add GeoJSON factory using the RapidJSON library. 107bca5 Use a reference instead of a copy. a6943a4 Mark a few variables that are not changing as const. 51b7e53 Improved error message for geometry exceptions. 5c37a13 Some minor spelling fixes 8ae5723 Bugfix: Dense location store was written out only partially. 5994322 Add support for tiles. 2168bac Add has_map_type() method to map factory. a9634bd Add more tests for mercator projection. 3c13e4d Add functionality to create simple polygons from ways in geom factories. e8c5bb1 Use uint64_t as counter, so there can be no overflows. 07fc9b9 libsparsehash-dev now in travis package whitelist 820e112 Add coverage support to CMake config. 5e9f943 Bugfix: Use the right include to really allow any input file type. d4b48eb CMake: Make version string a cached variable. e6baccb Add (c)begin/end functions to TypedMemoryMapping. Removed get_addr(). 3e32710 Use size() from MemoryMapping in TypedMemoryMapping. 96390db Improve MemoryMapping class documentation. 60a6217 Do not round memory mapped files to page size boundaries. 4907cbe Bugfix: function name. cac01d8 Use _filelengthi64 on Windows instead of fstat(2). 6a25bdf Windows: Put invalid parameter handler into wrapper class. Re-enable test. 110df9b Add invalid parameter handler on Windows to test. 549ed5f Disable some tests (to find which one fails on appveyor). a5b8873 Use resize_file() in memory mapping test instead of ftruncate directly. 40e41d3 Use _chsize_s() instead of _chsize() on Windows. 048397e Refactoring: Use low-level util functions in DataFile. 6a033f9 Remove now unused Windows implementation of mmap. 3eccdbb Move dword_hi/lo functions into osmium::util namespace. be7351b Remove unused code. b859b18 Make dword_hi/lo functions inline. 2e3bc37 Simplify mmap_vector_base/anon/file. f819cf3 Always map full pages. Make sure files behind mapping are large enough. d0c84b6 Add some low-level helper functions for file system access. 62e8d91 Make DataFile constructor explicit. fba684c Fix memory mapping test for windows. 78a7fd5 Add constructor to DataFile to create tmp file with given size. f911893 Bugfix: typo. 1cf2739 Add AnonymousMemoryMapping class. 56eac30 Implement MemoryMapping::resize() function. 1a73262 Bugfix: Counter variables were too small. 1ade32c Fix include position. b03aec3 Fixed some bugs in new DataFile class/tests. f109534 Add DataFile utility class. 9ed3c43 Fix/cleanup some code. 4f326c9 Fix bug: Copy-and-paste error. 78a5b2f Use reinterpret_cast instead of static_cast to get HANDLE on Windows. 7baa318 Fix typo. e669069 Make huge value even huger to see if code reliable fails then. 66137ad Improved documentation of MemoryMapping and TypedMemoryMapping classes. 3121393 Add TypedMemoryMapping class. f45335e Default for get_addr() template type. 685bbaf Remove unused code from tests. ce65bd4 Fix some issue with new MemoryMapping class. e7b8e15 Added MemoryMapping wrapper class for mmap() and Windows equivalent. 6b1effe typo fixed 33d479d Refactored travis build. 4348522 Fix xml data test. 769b1e8 Bugfix: Better check for invalid locations. bba7e68 Appveyor: Disable test failing because of missing dependency. 3d40dc7 Link with /debug on MSVC, add note about LNK4099 warnings. 5ef051f Appveyor: Disable header builds, add benchmarks. ce7485e Reformat Appveyor config. c60e505 use shallow clones for faster git fetch 3b18bca Travis cleanups. b8dfac0 Cleanup travis build. 5f19838 Trying to fix travis gcc build... d4255a4 Remove -Wno-return-type from recommended options. 5f1a41b Add dump_as_array() function to maps. ff22f76 Add constructors and begin()/end() functions to VectorBasedSparseMultimap. c7e05dd Bugfix: Make REGISTER_MAP() macro work when called several time with same name parameter. abdc317 Bugfix: Mark cbegin() and cend() of mmap_vector_base as const functions. d81d439 Add close() function to mmap_vector_base class. d74cff2 Add function on Buffer to get iterator to specific offset. git-subtree-dir: third_party/libosmium git-subtree-split: c43f8db50d93912a8bec5cd9fea733f7fec05549 --- .travis.yml | 79 +- CHANGELOG.md | 86 +- CMakeLists.txt | 63 +- EXTERNAL_LICENSES.txt | 233 +++ README.md | 4 + appveyor.yml | 63 +- benchmarks/CMakeLists.txt | 1 + benchmarks/osmium_benchmark_count.cpp | 9 +- benchmarks/osmium_benchmark_count_tag.cpp | 7 +- benchmarks/osmium_benchmark_index_map.cpp | 2 - ...mium_benchmark_static_vs_dynamic_index.cpp | 1 - benchmarks/osmium_benchmark_write_pbf.cpp | 34 + benchmarks/run_benchmark_write_pbf.sh | 28 + cmake/FindOSMPBF.cmake | 50 - cmake/FindOsmium.cmake | 10 +- cmake/iwyu.sh | 4 +- doc/CMakeLists.txt | 2 - doc/Doxyfile.in | 2 +- doc/README.md | 2 +- examples/osmium_area_test.cpp | 2 - examples/osmium_convert.cpp | 1 - examples/osmium_count.cpp | 9 +- examples/osmium_create_node_cache.cpp | 5 +- examples/osmium_debug.cpp | 2 - examples/osmium_read.cpp | 2 - examples/osmium_serdump.cpp | 5 +- examples/osmium_toogr.cpp | 2 - examples/osmium_toogr2.cpp | 2 - examples/osmium_toogr2_exp.cpp | 2 - examples/osmium_use_node_cache.cpp | 5 +- include/boost_unicode_iterator.hpp | 776 ---------- include/mmap_for_windows.hpp | 103 -- include/osmium/area/assembler.hpp | 18 +- .../osmium/area/multipolygon_collector.hpp | 25 +- include/osmium/builder/builder.hpp | 24 +- include/osmium/builder/osm_object_builder.hpp | 53 +- include/osmium/geom/factory.hpp | 162 +- include/osmium/geom/geos.hpp | 84 +- include/osmium/geom/mercator_projection.hpp | 1 + include/osmium/geom/rapid_geojson.hpp | 190 +++ include/osmium/geom/tile.hpp | 101 ++ include/osmium/geom/wkb.hpp | 10 +- include/osmium/index/bool_vector.hpp | 83 + .../index/detail/create_map_with_fd.hpp | 2 - .../osmium/index/detail/mmap_vector_anon.hpp | 21 +- .../osmium/index/detail/mmap_vector_base.hpp | 84 +- .../osmium/index/detail/mmap_vector_file.hpp | 29 +- include/osmium/index/detail/typed_mmap.hpp | 229 --- include/osmium/index/detail/vector_map.hpp | 10 +- .../osmium/index/detail/vector_multimap.hpp | 34 + include/osmium/index/index.hpp | 2 +- include/osmium/index/map.hpp | 16 +- include/osmium/index/map/dense_mmap_array.hpp | 2 +- include/osmium/index/map/sparse_mem_map.hpp | 2 +- include/osmium/io/any_input.hpp | 4 +- include/osmium/io/any_output.hpp | 5 +- include/osmium/io/bzip2_compression.hpp | 5 + include/osmium/io/compression.hpp | 5 + include/osmium/io/debug_output.hpp | 39 + .../osmium/io/detail/debug_output_format.hpp | 482 ++++++ .../osmium/io/detail/opl_output_format.hpp | 75 +- include/osmium/io/detail/pbf.hpp | 54 +- include/osmium/io/detail/pbf_decoder.hpp | 760 +++++++++ include/osmium/io/detail/pbf_input_format.hpp | 93 +- .../osmium/io/detail/pbf_output_format.hpp | 1083 +++++-------- include/osmium/io/detail/pbf_parser.hpp | 455 ------ include/osmium/io/detail/pbf_stringtable.hpp | 218 --- include/osmium/io/detail/protobuf_tags.hpp | 170 +++ include/osmium/io/detail/read_write.hpp | 2 +- include/osmium/io/detail/string_table.hpp | 250 +++ include/osmium/io/detail/xml_input_format.hpp | 37 +- .../osmium/io/detail/xml_output_format.hpp | 60 +- include/osmium/io/detail/zlib.hpp | 15 +- include/osmium/io/file.hpp | 68 +- include/osmium/io/file_format.hpp | 8 +- include/osmium/io/gzip_compression.hpp | 5 + include/osmium/io/pbf_input.hpp | 1 - include/osmium/io/pbf_output.hpp | 1 - include/osmium/memory/buffer.hpp | 21 +- include/osmium/memory/collection.hpp | 1 - include/osmium/memory/item.hpp | 1 - include/osmium/osm/changeset.hpp | 2 +- include/osmium/osm/crc.hpp | 223 +++ include/osmium/osm/diff_object.hpp | 29 +- include/osmium/osm/entity.hpp | 1 + include/osmium/osm/item_type.hpp | 20 + include/osmium/osm/node_ref.hpp | 2 +- include/osmium/osm/object.hpp | 1 + include/osmium/osm/relation.hpp | 5 + include/osmium/osm/timestamp.hpp | 12 +- include/osmium/osm/types.hpp | 21 - include/osmium/osm/types_from_string.hpp | 116 ++ include/osmium/relations/collector.hpp | 2 +- include/osmium/thread/pool.hpp | 1 + include/osmium/thread/queue.hpp | 31 +- include/osmium/thread/util.hpp | 2 +- include/osmium/util/data_file.hpp | 194 +++ include/osmium/util/delta.hpp | 147 ++ include/osmium/util/endian.hpp | 45 + include/osmium/util/file.hpp | 119 ++ include/osmium/util/memory_mapping.hpp | 750 +++++++++ include/osmium/util/minmax.hpp | 120 ++ include/osmium/util/string.hpp | 42 +- include/protozero/byteswap.hpp | 49 + include/protozero/exception.hpp | 68 + include/protozero/pbf_builder.hpp | 111 ++ include/protozero/pbf_message.hpp | 50 + include/protozero/pbf_reader.hpp | 1059 +++++++++++++ include/protozero/pbf_types.hpp | 49 + include/protozero/pbf_writer.hpp | 664 ++++++++ include/protozero/varint.hpp | 136 ++ include/utf8.h | 34 + include/utf8/checked.h | 327 ++++ include/utf8/core.h | 329 ++++ include/utf8/unchecked.h | 228 +++ scripts/travis_install.sh | 20 + scripts/travis_script.sh | 29 + test/CMakeLists.txt | 17 +- test/data-tests/testdata-testcases.cpp | 2 - test/data-tests/testdata-xml.cpp | 120 +- test/include/catch.hpp | 1355 +++++++++++------ test/include/catch_orig.hpp | 1347 ++++++++++------ test/t/basic/helper.hpp | 32 +- test/t/basic/test_box.cpp | 7 + test/t/basic/test_changeset.cpp | 12 +- test/t/basic/test_crc.cpp | 49 + test/t/basic/test_node.cpp | 8 + test/t/basic/test_relation.cpp | 11 +- test/t/basic/test_timestamp.cpp | 4 + test/t/basic/test_types_from_string.cpp | 90 ++ test/t/basic/test_way.cpp | 10 +- test/t/buffer/test_buffer_purge.cpp | 14 +- test/t/geom/test_exception.cpp | 16 + test/t/geom/test_geos.cpp | 35 +- test/t/geom/test_projection.cpp | 18 + test/t/geom/test_tile.cpp | 93 ++ test/t/geom/test_tile_data.hpp | 475 ++++++ test/t/geom/test_wkt.cpp | 8 + test/t/index/test_typed_mmap.cpp | 76 - test/t/index/test_typed_mmap_grow.cpp | 34 - test/t/io/test_file_formats.cpp | 24 + test/t/io/test_string_table.cpp | 94 ++ test/t/tags/test_tag_list.cpp | 126 +- test/t/thread/test_pool.cpp | 1 - test/t/util/test_data_file.cpp | 81 + test/t/util/test_delta.cpp | 68 + test/t/util/test_file.cpp | 69 + test/t/util/test_memory_mapping.cpp | 419 +++++ test/t/util/test_minmax.cpp | 68 + test/t/util/test_string.cpp | 11 + 150 files changed, 12266 insertions(+), 4262 deletions(-) create mode 100644 EXTERNAL_LICENSES.txt create mode 100644 benchmarks/osmium_benchmark_write_pbf.cpp create mode 100755 benchmarks/run_benchmark_write_pbf.sh delete mode 100644 cmake/FindOSMPBF.cmake delete mode 100644 include/boost_unicode_iterator.hpp delete mode 100644 include/mmap_for_windows.hpp create mode 100644 include/osmium/geom/rapid_geojson.hpp create mode 100644 include/osmium/geom/tile.hpp create mode 100644 include/osmium/index/bool_vector.hpp delete mode 100644 include/osmium/index/detail/typed_mmap.hpp create mode 100644 include/osmium/io/debug_output.hpp create mode 100644 include/osmium/io/detail/debug_output_format.hpp create mode 100644 include/osmium/io/detail/pbf_decoder.hpp delete mode 100644 include/osmium/io/detail/pbf_parser.hpp delete mode 100644 include/osmium/io/detail/pbf_stringtable.hpp create mode 100644 include/osmium/io/detail/protobuf_tags.hpp create mode 100644 include/osmium/io/detail/string_table.hpp create mode 100644 include/osmium/osm/crc.hpp create mode 100644 include/osmium/osm/types_from_string.hpp create mode 100644 include/osmium/util/data_file.hpp create mode 100644 include/osmium/util/delta.hpp create mode 100644 include/osmium/util/endian.hpp create mode 100644 include/osmium/util/file.hpp create mode 100644 include/osmium/util/memory_mapping.hpp create mode 100644 include/osmium/util/minmax.hpp create mode 100644 include/protozero/byteswap.hpp create mode 100644 include/protozero/exception.hpp create mode 100644 include/protozero/pbf_builder.hpp create mode 100644 include/protozero/pbf_message.hpp create mode 100644 include/protozero/pbf_reader.hpp create mode 100644 include/protozero/pbf_types.hpp create mode 100644 include/protozero/pbf_writer.hpp create mode 100644 include/protozero/varint.hpp create mode 100644 include/utf8.h create mode 100644 include/utf8/checked.h create mode 100644 include/utf8/core.h create mode 100644 include/utf8/unchecked.h create mode 100755 scripts/travis_install.sh create mode 100755 scripts/travis_script.sh create mode 100644 test/t/basic/test_crc.cpp create mode 100644 test/t/basic/test_types_from_string.cpp create mode 100644 test/t/geom/test_exception.cpp create mode 100644 test/t/geom/test_tile.cpp create mode 100644 test/t/geom/test_tile_data.hpp delete mode 100644 test/t/index/test_typed_mmap.cpp delete mode 100644 test/t/index/test_typed_mmap_grow.cpp create mode 100644 test/t/io/test_string_table.cpp create mode 100644 test/t/util/test_data_file.cpp create mode 100644 test/t/util/test_delta.cpp create mode 100644 test/t/util/test_file.cpp create mode 100644 test/t/util/test_memory_mapping.cpp create mode 100644 test/t/util/test_minmax.cpp diff --git a/.travis.yml b/.travis.yml index 73dff72a1aa..6ebdd7167e9 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,46 +6,49 @@ language: cpp -compiler: - - gcc - - clang - -env: - - CONFIGURATION=Dev - - CONFIGURATION=Release - -before_install: - # we need at least g++-4.8 for c++11 features - - sudo add-apt-repository --yes ppa:ubuntu-toolchain-r/test - - sudo apt-get update --yes --quiet +sudo: false + +matrix: + include: + - os: linux + compiler: clang + env: BUILD_TYPE=Dev + - os: linux + compiler: clang + env: BUILD_TYPE=Release + - os: linux + compiler: gcc + env: BUILD_TYPE=Dev + - os: linux + compiler: gcc + env: BUILD_TYPE=Release + - os: osx + compiler: clang + env: BUILD_TYPE=Dev + - os: osx + compiler: clang + env: BUILD_TYPE=Release + +# http://docs.travis-ci.com/user/apt/ +addons: + apt: + sources: + - boost-latest + - ubuntu-toolchain-r-test + packages: + - g++-4.8 + - gcc-4.8 + - libboost1.55-dev + - libboost-program-options1.55-dev + - libgdal-dev + - libgeos++-dev + - libproj-dev + - libsparsehash-dev + - spatialite-bin install: - - cd .. - # upgrade compilers - - sudo apt-get install --yes gcc-4.8 g++-4.8 - # make sure 'cpp' is the just installed current one - - sudo rm /usr/bin/cpp - - sudo ln -s /usr/bin/cpp-4.8 /usr/bin/cpp - # upgrade libosmium dependencies - - sudo apt-get install --yes make libboost-dev libboost-program-options-dev libsparsehash-dev libprotobuf-dev protobuf-compiler libgeos++-dev libproj-dev libgdal1h libgdal-dev - - git clone https://github.com/osmcode/osm-testdata.git - # OSMPBF is too old, install from git - #- sudo apt-get install --yes libosmpbf-dev - - git clone https://github.com/scrosby/OSM-binary.git - - cd OSM-binary/src - - make - - sudo make install - - cd ../.. - - cd libosmium - -before_script: - - true + - scripts/travis_install.sh script: - - if [ "${CXX}" = 'g++' ]; then export CXX=g++-4.8; fi; - - mkdir build - - cd build - - cmake -LA -DCMAKE_BUILD_TYPE=${CONFIGURATION} .. - - make VERBOSE=1 - - ctest --output-on-failure + - scripts/travis_script.sh diff --git a/CHANGELOG.md b/CHANGELOG.md index 4c345a49817..22eb06aac56 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,88 @@ This project adheres to [Semantic Versioning](http://semver.org/). ## [unreleased] - +### Added + +### Changed + +### Fixed + +## [2.3.0] - 2015-08-18 + +### Added + +- Allow instantiating osmium::geom::GEOSFactory with existing GEOS factory. +- Low-level functions to support generating a architecture- and endian- + independant CRC from OSM data. This is intended to be uses with boost::crc. +- Add new debug output format. This format is not intended to be read + automatically, but for human consumption. It formats the data nicely. +- Make writing of metadata configurable for XML and OPL output (use + `add_metadata=false` as file option). + +### Changed + +- Changed `add_user()` and `add_role()` in builders to use string length + without the 0-termination. +- Improved code setting file format from suffix/format argument. +- Memory mapping utility class now supports readonly, private writable or + shared writable operation. +- Allow empty version (0) in PBF files. +- Use utf8cpp header-only lib instead of boost for utf8 decoding. The library + is included in the libosmium distribution. +- New PBF reader and writer based on the protozero. A complete rewrite of the + code for reading and writing OSM PBF files. It doesn't use the Google + protobuf library and it doesn't use the OSMPBF/OSM-Binary library any more. + Instead is uses the protozero lightweight protobuf header library which is + included in the code. Not only does the new code have less dependencies, it + is faster and more robust. https://github.com/mapbox/protozero + +### Fixed + +- Various smaller bug fixes. +- Add encoding for relation member roles in OPL format. +- Change character encoding to new format in OPL: variable length hex code + between % characters instead of a % followed by 4-digit hex code. This is + necessary because unicode characters can be longer than the 4-digit hex + code. +- XML writer: The linefeed, carriage return, and tab characters are now + escaped properly. +- Reading large XML files could block. + +## [2.2.0] - 2015-07-04 + +### Added + +- Conversion functions for some low-level types. +- BoolVector index class. +- `min_op`/`max_op` utility functions. +- More tests here and there. +- Helper methods `is_between()` and `is_visible_at()` to DiffObject. +- GeoJSON factory using the RapidJSON library. +- Support for tile calculations. +- Create simple polygons from ways in geom factories. +- `MemoryMapping` and `TypedMemoryMapping` helper classes. +- `close()` function to `mmap_vector_base` class. +- Function on `Buffer` class to get iterator to specific offset. +- Explicit cast operator from `osmium::Timestamp` to `uint32_t`. + +### Changed + +- Throw exception on illegal values in functions parsing strings to get ids, + versions, etc. +- Improved error message for geometry exceptions. + +### Fixed + +- Throw exception from `dump_as_array()` and `dump_as_list()` functions if not + implemented in an index. +- After writing OSM files, program could stall up to a second. +- Dense location store was written out only partially. +- Use `uint64_t` as counter in benchmarks, so there can be no overflows. +- Example programs now read packed XML files, too. +- Refactoring of memory mapping code. Removes leak on Windows. +- Better check for invalid locations. +- Mark `cbegin()` and `cend()` of `mmap_vector_base` as const functions. + ## [2.1.0] - 2015-03-31 ### Added @@ -26,6 +108,8 @@ This project adheres to [Semantic Versioning](http://semver.org/). Doxygen (up to version 1.8.8). This version contains a workaround to fix this. -[unreleased]: https://github.com/osmcode/libosmium/compare/v2.1.0...HEAD +[unreleased]: https://github.com/osmcode/libosmium/compare/v2.3.0...HEAD +[2.3.0]: https://github.com/osmcode/libosmium/compare/v2.3.0...v2.3.0 +[2.2.0]: https://github.com/osmcode/libosmium/compare/v2.1.0...v2.2.0 [2.1.0]: https://github.com/osmcode/libosmium/compare/v2.0.0...v2.1.0 diff --git a/CMakeLists.txt b/CMakeLists.txt index 5e70a9935aa..fba967a4ed4 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -18,7 +18,7 @@ set(CMAKE_EXPORT_COMPILE_COMMANDS ON) # #----------------------------------------------------------------------------- -set(CMAKE_CONFIGURATION_TYPES "Debug;Release;RelWithDebInfo;MinSizeRel;Dev" +set(CMAKE_CONFIGURATION_TYPES "Debug;Release;RelWithDebInfo;MinSizeRel;Dev;Coverage" CACHE STRING "List of available configuration types" FORCE) @@ -26,11 +26,13 @@ set(CMAKE_CONFIGURATION_TYPES "Debug;Release;RelWithDebInfo;MinSizeRel;Dev" project(libosmium) set(LIBOSMIUM_VERSION_MAJOR 2) -set(LIBOSMIUM_VERSION_MINOR 1) +set(LIBOSMIUM_VERSION_MINOR 3) set(LIBOSMIUM_VERSION_PATCH 0) set(LIBOSMIUM_VERSION - ${LIBOSMIUM_VERSION_MAJOR}.${LIBOSMIUM_VERSION_MINOR}.${LIBOSMIUM_VERSION_PATCH}) + "${LIBOSMIUM_VERSION_MAJOR}.${LIBOSMIUM_VERSION_MINOR}.${LIBOSMIUM_VERSION_PATCH}" + CACHE STRING + "Libosmium version") #----------------------------------------------------------------------------- @@ -55,6 +57,57 @@ option(BUILD_BENCHMARKS "compile benchmark programs" ${dev_build}) option(BUILD_DATA_TESTS "compile data tests, please run them with ctest" ${dev_build}) +#----------------------------------------------------------------------------- +# +# Coverage support +# +#----------------------------------------------------------------------------- + +include(CheckCXXCompilerFlag) +check_cxx_compiler_flag("-fkeep-inline-functions" HAS_KEEP_INLINE_FUNCTIONS) +if(HAS_KEEP_INLINE_FUNCTIONS) + set(extra_coverage_flags_ "-fkeep-inline-functions") +endif() + +set(CMAKE_CXX_FLAGS_COVERAGE + "-g -O0 -fno-inline-functions -fno-inline --coverage ${extra_coverage_flags_}" + CACHE STRING "Flags used by the compiler during coverage builds.") + +set(CMAKE_EXE_LINKER_FLAGS_COVERAGE + "--coverage" + CACHE STRING "Flags used by the linker during coverage builds.") + +if(CMAKE_BUILD_TYPE STREQUAL "Coverage") + if(BUILD_EXAMPLES OR BUILD_HEADERS OR BUILD_BENCHMARKS OR BUILD_DATA_TESTS) + message(WARNING "Coverage builds don't work for anything but the unit tests") + endif() + + if(CMAKE_CXX_COMPILER_ID STREQUAL "Clang") + string(REGEX REPLACE "^([0-9]+)\\.([0-9]+).*$" "llvm-cov-\\1.\\2" + gcov_ ${CMAKE_CXX_COMPILER_VERSION}) + else() + set(gcov_ "gcov") + endif() + + find_program(GCOV ${gcov_} DOC "Coverage tool") + find_program(GCOVR "gcovr" DOC "Coverage report tool") + + set(coverage_report_dir "${CMAKE_BINARY_DIR}/coverage") + file(MAKE_DIRECTORY ${coverage_report_dir}) + add_custom_target(coverage + ${GCOVR} + ${CMAKE_BINARY_DIR} + --root=${CMAKE_SOURCE_DIR} + --html --html-details + #--verbose + #--keep + '--filter=.*include/osmium.*' + --sort-percentage + --gcov-executable=${GCOV} + --output=${coverage_report_dir}/index.html) +endif() + + #----------------------------------------------------------------------------- # # Find external dependencies @@ -113,8 +166,10 @@ endif() #----------------------------------------------------------------------------- if(MSVC) set(USUAL_COMPILE_OPTIONS "/Ox") + set(USUAL_LINK_OPTIONS "/debug") else() set(USUAL_COMPILE_OPTIONS "-O3 -g") + set(USUAL_LINK_OPTIONS "") endif() if(WIN32) @@ -126,7 +181,7 @@ set(CMAKE_CXX_FLAGS_DEV "${USUAL_COMPILE_OPTIONS}" CACHE STRING "Flags used by the compiler during developer builds." FORCE) -set(CMAKE_EXE_LINKER_FLAGS_DEV "" +set(CMAKE_EXE_LINKER_FLAGS_DEV "${USUAL_LINK_OPTIONS}" CACHE STRING "Flags used by the linker during developer builds." FORCE) mark_as_advanced( diff --git a/EXTERNAL_LICENSES.txt b/EXTERNAL_LICENSES.txt new file mode 100644 index 00000000000..7b06fcf4110 --- /dev/null +++ b/EXTERNAL_LICENSES.txt @@ -0,0 +1,233 @@ + +==== For protozero from https://github.com/mapbox/protozero + +protozero copyright (c) Mapbox. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS +IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +==== For protozero from https://github.com/mapbox/protozero + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + +==== For utf8.h + +Copyright 2006 Nemanja Trifunovic + +Permission is hereby granted, free of charge, to any person or organization +obtaining a copy of the software and accompanying documentation covered by +this license (the "Software") to use, reproduce, display, distribute, +execute, and transmit the Software, and to prepare derivative works of the +Software, and to permit third-parties to whom the Software is furnished to +do so, all subject to the following: + +The copyright notices in the Software and this entire statement, including +the above license grant, this restriction and the following disclaimer, +must be included in all copies of the Software, in whole or in part, and +all derivative works of the Software, unless such copies or derivative +works are solely in the form of machine-executable object code generated by +a source language processor. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT +SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE +FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, +ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. + diff --git a/README.md b/README.md index 503440e8ed8..9676d80b74b 100644 --- a/README.md +++ b/README.md @@ -27,6 +27,10 @@ you need for your programs. For details see the [list of dependencies](https://github.com/osmcode/libosmium/wiki/Libosmium-dependencies). +The [protozero](https://github.com/mapbox/protozero) and +[utf8-cpp](http://utfcpp.sourceforge.net/) header-only libraries are included +in the libosmium repository. + ## Directories diff --git a/appveyor.yml b/appveyor.yml index 06c8e697496..a05c396cc3b 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -15,6 +15,8 @@ branches: only: - master +shallow_clone: true + # Operating system (build VM template) os: Visual Studio 2014 CTP4 @@ -27,7 +29,7 @@ clone_folder: c:\projects\libosmium platform: x64 install: - # show all availble env vars + # show all available env vars - set - echo cmake on AppVeyor - cmake -version @@ -50,8 +52,8 @@ install: #cmake cannot find it otherwise - set LIBBZIP2=%LODEPSDIR%\bzip2\lib\libbz2.lib - set LIBBZIP2=%LIBBZIP2:\=/% - - ps: Start-FileDownload https://mapnik.s3.amazonaws.com/deps/cmake-3.1.0-win32-x86.7z -FileName cm.7z - - ps: Start-FileDownload https://mapnik.s3.amazonaws.com/dist/dev/libosmium-deps-win-14.0-x64.7z -FileName lodeps.7z + - ps: Start-FileDownload https://mapbox.s3.amazonaws.com/windows-builds/windows-build-deps/cmake-3.1.0-win32-x86.7z -FileName cm.7z + - ps: Start-FileDownload https://mapbox.s3.amazonaws.com/windows-builds/windows-build-deps/libosmium-deps-win-14.0-x64.7z -FileName lodeps.7z - 7z x cm.7z | %windir%\system32\find "ing archive" - 7z x lodeps.7z | %windir%\system32\find "ing archive" - echo %LODEPSDIR% @@ -59,19 +61,64 @@ install: - echo our own cmake - cmake -version - cd c:\projects - - git clone https://github.com/osmcode/osm-testdata.git + - git clone --depth 1 https://github.com/osmcode/osm-testdata.git build_script: - cd c:\projects\libosmium - mkdir build - cd build - echo %config% - - cmake .. -LA -G "Visual Studio 14 Win64" -DOsmium_DEBUG=TRUE -DCMAKE_BUILD_TYPE=%config% -DBUILD_BENCHMARKS=OFF -DBOOST_ROOT=%LODEPSDIR%\boost -DBoost_PROGRAM_OPTIONS_LIBRARY=%LODEPSDIR%\boost\lib\libboost_program_options-vc140-mt-1_57.lib -DOSMPBF_LIBRARY=%LODEPSDIR%\osmpbf\lib\osmpbf.lib -DOSMPBF_INCLUDE_DIR=%LODEPSDIR%\osmpbf\include -DPROTOBUF_LIBRARY=%LODEPSDIR%\protobuf\lib\libprotobuf.lib -DPROTOBUF_LITE_LIBRARY=%LODEPSDIR%\protobuf\lib\libprotobuf-lite.lib -DPROTOBUF_INCLUDE_DIR=%LODEPSDIR%\protobuf\include -DZLIB_LIBRARY=%LODEPSDIR%\zlib\lib\zlibwapi.lib -DZLIB_INCLUDE_DIR=%LODEPSDIR%\zlib\include -DEXPAT_LIBRARY=%LODEPSDIR%\expat\lib\libexpat.lib -DEXPAT_INCLUDE_DIR=%LODEPSDIR%\expat\include -DBZIP2_LIBRARIES=%LIBBZIP2% -DBZIP2_INCLUDE_DIR=%LODEPSDIR%\bzip2\include -DGDAL_LIBRARY=%LODEPSDIR%\gdal\lib\gdal_i.lib -DGDAL_INCLUDE_DIR=%LODEPSDIR%\gdal\include -DGEOS_LIBRARY=%LODEPSDIR%\geos\lib\geos.lib -DGEOS_INCLUDE_DIR=%LODEPSDIR%\geos\include -DPROJ_LIBRARY=%LODEPSDIR%\proj\lib\proj.lib -DPROJ_INCLUDE_DIR=%LODEPSDIR%\proj\include -DSPARSEHASH_INCLUDE_DIR=%LODEPSDIR%\sparsehash\include -DGETOPT_LIBRARY=%LODEPSDIR%\wingetopt\lib\wingetopt.lib -DGETOPT_INCLUDE_DIR=%LODEPSDIR%\wingetopt\include + # This will produce lots of LNK4099 warnings which can be ignored. + # Unfortunately they can't be disabled, see + # http://stackoverflow.com/questions/661606/visual-c-how-to-disable-specific-linker-warnings + - cmake .. -LA -G "Visual Studio 14 Win64" + -DOsmium_DEBUG=TRUE + -DCMAKE_BUILD_TYPE=%config% + -DBUILD_HEADERS=OFF + -DBOOST_ROOT=%LODEPSDIR%\boost + -DBoost_PROGRAM_OPTIONS_LIBRARY=%LODEPSDIR%\boost\lib\libboost_program_options-vc140-mt-1_57.lib + -DZLIB_LIBRARY=%LODEPSDIR%\zlib\lib\zlibwapi.lib + -DZLIB_INCLUDE_DIR=%LODEPSDIR%\zlib\include + -DEXPAT_LIBRARY=%LODEPSDIR%\expat\lib\libexpat.lib + -DEXPAT_INCLUDE_DIR=%LODEPSDIR%\expat\include + -DBZIP2_LIBRARIES=%LIBBZIP2% + -DBZIP2_INCLUDE_DIR=%LODEPSDIR%\bzip2\include + -DGDAL_LIBRARY=%LODEPSDIR%\gdal\lib\gdal_i.lib + -DGDAL_INCLUDE_DIR=%LODEPSDIR%\gdal\include + -DGEOS_LIBRARY=%LODEPSDIR%\geos\lib\geos.lib + -DGEOS_INCLUDE_DIR=%LODEPSDIR%\geos\include + -DPROJ_LIBRARY=%LODEPSDIR%\proj\lib\proj.lib + -DPROJ_INCLUDE_DIR=%LODEPSDIR%\proj\include + -DSPARSEHASH_INCLUDE_DIR=%LODEPSDIR%\sparsehash\include + -DGETOPT_LIBRARY=%LODEPSDIR%\wingetopt\lib\wingetopt.lib + -DGETOPT_INCLUDE_DIR=%LODEPSDIR%\wingetopt\include - msbuild libosmium.sln /p:Configuration=%config% /toolsversion:14.0 /p:Platform=x64 /p:PlatformToolset=v140 - #- cmake .. -LA -G "NMake Makefiles" -DOsmium_DEBUG=TRUE -DCMAKE_BUILD_TYPE=%config% -DBOOST_ROOT=%LODEPSDIR%\boost -DBoost_PROGRAM_OPTIONS_LIBRARY=%LODEPSDIR%\boost\lib\libboost_program_options-vc140-mt-1_57.lib -DOSMPBF_LIBRARY=%LODEPSDIR%\osmpbf\lib\osmpbf.lib -DOSMPBF_INCLUDE_DIR=%LODEPSDIR%\osmpbf\include -DPROTOBUF_LIBRARY=%LODEPSDIR%\protobuf\lib\libprotobuf.lib -DPROTOBUF_LITE_LIBRARY=%LODEPSDIR%\protobuf\lib\libprotobuf-lite.lib -DPROTOBUF_INCLUDE_DIR=%LODEPSDIR%\protobuf\include -DZLIB_LIBRARY=%LODEPSDIR%\zlib\lib\zlibwapi.lib -DZLIB_INCLUDE_DIR=%LODEPSDIR%\zlib\include -DEXPAT_LIBRARY=%LODEPSDIR%\expat\lib\libexpat.lib -DEXPAT_INCLUDE_DIR=%LODEPSDIR%\expat\include -DBZIP2_LIBRARIES=%LIBBZIP2% -DBZIP2_INCLUDE_DIR=%LODEPSDIR%\bzip2\include -DGDAL_LIBRARY=%LODEPSDIR%\gdal\lib\gdal_i.lib -DGDAL_INCLUDE_DIR=%LODEPSDIR%\gdal\include -DGEOS_LIBRARY=%LODEPSDIR%\geos\lib\geos.lib -DGEOS_INCLUDE_DIR=%LODEPSDIR%\geos\include -DPROJ_LIBRARY=%LODEPSDIR%\proj\lib\proj.lib -DPROJ_INCLUDE_DIR=%LODEPSDIR%\proj\include -DSPARSEHASH_INCLUDE_DIR=%LODEPSDIR%\sparsehash\include -DGETOPT_LIBRARY=%LODEPSDIR%\wingetopt\lib\wingetopt.lib -DGETOPT_INCLUDE_DIR=%LODEPSDIR%\wingetopt\include + #- cmake .. -LA -G "NMake Makefiles" + # -DOsmium_DEBUG=TRUE + # -DCMAKE_BUILD_TYPE=%config% + # -DBOOST_ROOT=%LODEPSDIR%\boost + # -DBoost_PROGRAM_OPTIONS_LIBRARY=%LODEPSDIR%\boost\lib\libboost_program_options-vc140-mt-1_57.lib + # -DZLIB_LIBRARY=%LODEPSDIR%\zlib\lib\zlibwapi.lib + # -DZLIB_INCLUDE_DIR=%LODEPSDIR%\zlib\include + # -DEXPAT_LIBRARY=%LODEPSDIR%\expat\lib\libexpat.lib + # -DEXPAT_INCLUDE_DIR=%LODEPSDIR%\expat\include + # -DBZIP2_LIBRARIES=%LIBBZIP2% + # -DBZIP2_INCLUDE_DIR=%LODEPSDIR%\bzip2\include + # -DGDAL_LIBRARY=%LODEPSDIR%\gdal\lib\gdal_i.lib + # -DGDAL_INCLUDE_DIR=%LODEPSDIR%\gdal\include + # -DGEOS_LIBRARY=%LODEPSDIR%\geos\lib\geos.lib + # -DGEOS_INCLUDE_DIR=%LODEPSDIR%\geos\include + # -DPROJ_LIBRARY=%LODEPSDIR%\proj\lib\proj.lib + # -DPROJ_INCLUDE_DIR=%LODEPSDIR%\proj\include + # -DSPARSEHASH_INCLUDE_DIR=%LODEPSDIR%\sparsehash\include + # -DGETOPT_LIBRARY=%LODEPSDIR%\wingetopt\lib\wingetopt.lib + # -DGETOPT_INCLUDE_DIR=%LODEPSDIR%\wingetopt\include #- nmake test_script: - # -LE fails_on_windows exempts tests we know will fail - - ctest --output-on-failure -C %config% -LE fails_on_windows + # "-E testdata-overview" exempts one test we know fails on Appveyor + # because we currently don't have spatialite support. + - ctest --output-on-failure + -C %config% + -E testdata-overview diff --git a/benchmarks/CMakeLists.txt b/benchmarks/CMakeLists.txt index 6a4ca162dba..e46c833496d 100644 --- a/benchmarks/CMakeLists.txt +++ b/benchmarks/CMakeLists.txt @@ -13,6 +13,7 @@ set(BENCHMARKS count_tag index_map static_vs_dynamic_index + write_pbf CACHE STRING "Benchmark programs" ) diff --git a/benchmarks/osmium_benchmark_count.cpp b/benchmarks/osmium_benchmark_count.cpp index 701d6faec8c..d50c53dc814 100644 --- a/benchmarks/osmium_benchmark_count.cpp +++ b/benchmarks/osmium_benchmark_count.cpp @@ -4,6 +4,7 @@ */ +#include #include #include @@ -12,9 +13,9 @@ struct CountHandler : public osmium::handler::Handler { - int nodes = 0; - int ways = 0; - int relations = 0; + uint64_t nodes = 0; + uint64_t ways = 0; + uint64_t relations = 0; void node(osmium::Node&) { ++nodes; @@ -48,7 +49,5 @@ int main(int argc, char* argv[]) { std::cout << "Nodes: " << handler.nodes << "\n"; std::cout << "Ways: " << handler.ways << "\n"; std::cout << "Relations: " << handler.relations << "\n"; - - google::protobuf::ShutdownProtobufLibrary(); } diff --git a/benchmarks/osmium_benchmark_count_tag.cpp b/benchmarks/osmium_benchmark_count_tag.cpp index 4a77c34529b..8fa696a4ee6 100644 --- a/benchmarks/osmium_benchmark_count_tag.cpp +++ b/benchmarks/osmium_benchmark_count_tag.cpp @@ -4,6 +4,7 @@ */ +#include #include #include @@ -12,8 +13,8 @@ struct CountHandler : public osmium::handler::Handler { - int counter = 0; - int all = 0; + uint64_t counter = 0; + uint64_t all = 0; void node(osmium::Node& node) { ++all; @@ -49,7 +50,5 @@ int main(int argc, char* argv[]) { reader.close(); std::cout << "r_all=" << handler.all << " r_counter=" << handler.counter << "\n"; - - google::protobuf::ShutdownProtobufLibrary(); } diff --git a/benchmarks/osmium_benchmark_index_map.cpp b/benchmarks/osmium_benchmark_index_map.cpp index fa75fb2b86a..02578261808 100644 --- a/benchmarks/osmium_benchmark_index_map.cpp +++ b/benchmarks/osmium_benchmark_index_map.cpp @@ -35,7 +35,5 @@ int main(int argc, char* argv[]) { osmium::apply(reader, location_handler); reader.close(); - - google::protobuf::ShutdownProtobufLibrary(); } diff --git a/benchmarks/osmium_benchmark_static_vs_dynamic_index.cpp b/benchmarks/osmium_benchmark_static_vs_dynamic_index.cpp index 9c47c84499e..66e2a0bd918 100644 --- a/benchmarks/osmium_benchmark_static_vs_dynamic_index.cpp +++ b/benchmarks/osmium_benchmark_static_vs_dynamic_index.cpp @@ -46,7 +46,6 @@ int main(int argc, char* argv[]) { std::string input_filename = argv[1]; osmium::memory::Buffer buffer = osmium::io::read_file(input_filename); - google::protobuf::ShutdownProtobufLibrary(); const auto& map_factory = osmium::index::MapFactory::instance(); diff --git a/benchmarks/osmium_benchmark_write_pbf.cpp b/benchmarks/osmium_benchmark_write_pbf.cpp new file mode 100644 index 00000000000..869f3a8f849 --- /dev/null +++ b/benchmarks/osmium_benchmark_write_pbf.cpp @@ -0,0 +1,34 @@ +/* + + The code in this file is released into the Public Domain. + +*/ + +#include +#include + +#include +#include + +int main(int argc, char* argv[]) { + if (argc != 3) { + std::cerr << "Usage: " << argv[0] << " INPUT-FILE OUTPUT-FILE\n"; + exit(1); + } + + std::string input_filename = argv[1]; + std::string output_filename = argv[2]; + + osmium::io::Reader reader(input_filename); + osmium::io::File output_file(output_filename, "pbf"); + osmium::io::Header header; + osmium::io::Writer writer(output_file, header, osmium::io::overwrite::allow); + + while (osmium::memory::Buffer buffer = reader.read()) { + writer(std::move(buffer)); + } + + writer.close(); + reader.close(); +} + diff --git a/benchmarks/run_benchmark_write_pbf.sh b/benchmarks/run_benchmark_write_pbf.sh new file mode 100755 index 00000000000..8143097702a --- /dev/null +++ b/benchmarks/run_benchmark_write_pbf.sh @@ -0,0 +1,28 @@ +#!/bin/sh +# +# run_benchmark_write_pbf.sh +# +# Will read the input file and after reading it into memory completely, +# write it to /dev/null. Because this will need the time to read *and* write +# the file, it will report the times for reading and writing. You can +# subtract the times needed for the "count" benchmark to (roughly) get the +# write times. +# + +set -e + +BENCHMARK_NAME=write_pbf + +. @CMAKE_BINARY_DIR@/benchmarks/setup.sh + +CMD=$OB_DIR/osmium_benchmark_$BENCHMARK_NAME + +echo "# file size num mem time cpu_kernel cpu_user cpu_percent cmd options" +for data in $OB_DATA_FILES; do + filename=`basename $data` + filesize=`stat --format="%s" --dereference $data` + for n in $OB_SEQ; do + $OB_TIME_CMD -f "$filename $filesize $n $OB_TIME_FORMAT" $CMD $data /dev/null 2>&1 >/dev/null | sed -e "s%$DATA_DIR/%%" | sed -e "s%$OB_DIR/%%" + done +done + diff --git a/cmake/FindOSMPBF.cmake b/cmake/FindOSMPBF.cmake deleted file mode 100644 index deeebd8b6c9..00000000000 --- a/cmake/FindOSMPBF.cmake +++ /dev/null @@ -1,50 +0,0 @@ -# -# Locate OSMPBF library -# -# This module defines -# OSMPBF_FOUND - if false, do not try to link to OSMPBF -# OSMPBF_LIBRARIES - full library path name -# OSMPBF_INCLUDE_DIRS - where to find OSMPBF.hpp -# -# Note that the expected include convention is -# #include -# and not -# #include -# - -find_path(OSMPBF_INCLUDE_DIR osmpbf/osmpbf.h - HINTS $ENV{OSMPBF_DIR} - PATH_SUFFIXES include - PATHS - ~/Library/Frameworks - /Library/Frameworks - /usr/local - /usr - /opt/local # DarwinPorts - /opt -) - -find_library(OSMPBF_LIBRARY - NAMES osmpbf - HINTS $ENV{OSMPBF_DIR} - PATH_SUFFIXES lib64 lib - PATHS - ~/Library/Frameworks - /Library/Frameworks - /usr/local - /usr - /opt/local - /opt -) - -# Handle the QUIETLY and REQUIRED arguments and set OSMPBF_FOUND to TRUE if -# all listed variables are TRUE. -include(FindPackageHandleStandardArgs) -find_package_handle_standard_args(OSMPBF DEFAULT_MSG OSMPBF_LIBRARY OSMPBF_INCLUDE_DIR) - -# Copy the results to the output variables. -if(OSMPBF_FOUND) - set(OSMPBF_INCLUDE_DIRS ${OSMPBF_INCLUDE_DIR}) - set(OSMPBF_LIBRARIES ${OSMPBF_LIBRARY}) -endif() - diff --git a/cmake/FindOsmium.cmake b/cmake/FindOsmium.cmake index 1de41a02268..bb140718bdf 100644 --- a/cmake/FindOsmium.cmake +++ b/cmake/FindOsmium.cmake @@ -110,15 +110,11 @@ endif() #---------------------------------------------------------------------- # Component 'pbf' if(Osmium_USE_PBF) - find_package(OSMPBF) - find_package(Protobuf) find_package(ZLIB) find_package(Threads) - if(OSMPBF_FOUND AND PROTOBUF_FOUND AND ZLIB_FOUND AND Threads_FOUND) + if(ZLIB_FOUND AND Threads_FOUND) list(APPEND OSMIUM_PBF_LIBRARIES - ${OSMPBF_LIBRARIES} - ${PROTOBUF_LITE_LIBRARY} ${ZLIB_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT} ) @@ -126,8 +122,6 @@ if(Osmium_USE_PBF) list(APPEND OSMIUM_PBF_LIBRARIES ws2_32) endif() list(APPEND OSMIUM_INCLUDE_DIRS - ${OSMPBF_INCLUDE_DIRS} - ${PROTOBUF_INCLUDE_DIR} ${ZLIB_INCLUDE_DIR} ) else() @@ -325,7 +319,7 @@ endif() if(MSVC) set(OSMIUM_WARNING_OPTIONS "/W3 /wd4514" CACHE STRING "Recommended warning options for libosmium") else() - set(OSMIUM_WARNING_OPTIONS "-Wall -Wextra -pedantic -Wredundant-decls -Wdisabled-optimization -Wctor-dtor-privacy -Wnon-virtual-dtor -Woverloaded-virtual -Wsign-promo -Wold-style-cast -Wno-return-type" CACHE STRING "Recommended warning options for libosmium") + set(OSMIUM_WARNING_OPTIONS "-Wall -Wextra -pedantic -Wredundant-decls -Wdisabled-optimization -Wctor-dtor-privacy -Wnon-virtual-dtor -Woverloaded-virtual -Wsign-promo -Wold-style-cast" CACHE STRING "Recommended warning options for libosmium") endif() set(OSMIUM_DRACONIC_CLANG_OPTIONS "-Wdocumentation -Wunused-exception-parameter -Wmissing-declarations -Weverything -Wno-c++98-compat -Wno-c++98-compat-pedantic -Wno-unused-macros -Wno-exit-time-destructors -Wno-global-constructors -Wno-padded -Wno-switch-enum -Wno-missing-prototypes -Wno-weak-vtables -Wno-cast-align -Wno-float-equal") diff --git a/cmake/iwyu.sh b/cmake/iwyu.sh index d2038449151..f7d8a15e809 100755 --- a/cmake/iwyu.sh +++ b/cmake/iwyu.sh @@ -10,12 +10,12 @@ cmdline="iwyu -Xiwyu --mapping_file=osmium.imp -std=c++11 -I include" log=build/iwyu.log +mkdir -p build/check_reports + echo "INCLUDE WHAT YOU USE REPORT:" >$log allok=yes -mkdir -p build/check_reports - for file in `find include/osmium -name \*.hpp`; do mkdir -p `dirname build/check_reports/$file` ifile="build/check_reports/${file%.hpp}.iwyu" diff --git a/doc/CMakeLists.txt b/doc/CMakeLists.txt index 9d69a16bd9e..5ea819b9220 100644 --- a/doc/CMakeLists.txt +++ b/doc/CMakeLists.txt @@ -21,8 +21,6 @@ if(DOXYGEN_FOUND) WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} COMMENT "Generating API documentation with Doxygen" VERBATIM ) -# install(DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/html" -# DESTINATION "share/doc/libosmium-dev") else() message(STATUS "Looking for doxygen - not found") message(STATUS " Disabled making of documentation.") diff --git a/doc/Doxyfile.in b/doc/Doxyfile.in index c03e255e9ef..d5ed13d2e80 100644 --- a/doc/Doxyfile.in +++ b/doc/Doxyfile.in @@ -1128,7 +1128,7 @@ HTML_COLORSTYLE_GAMMA = 80 # The default value is: YES. # This tag requires that the tag GENERATE_HTML is set to YES. -HTML_TIMESTAMP = YES +HTML_TIMESTAMP = NO # If the HTML_DYNAMIC_SECTIONS tag is set to YES then the generated HTML # documentation will contain sections that can be hidden and shown after the diff --git a/doc/README.md b/doc/README.md index 7ca8e7cf73c..5e1cf4bd2e0 100644 --- a/doc/README.md +++ b/doc/README.md @@ -3,6 +3,6 @@ The `header.html` is created with: `doxygen -w html header.html footer.html stylesheet.css` -This might have to be rn again for newer Doxygen versions. After that add +This might have to be run again for newer Doxygen versions. After that add changes back in. diff --git a/examples/osmium_area_test.cpp b/examples/osmium_area_test.cpp index ee2ba129648..f072c5e546a 100644 --- a/examples/osmium_area_test.cpp +++ b/examples/osmium_area_test.cpp @@ -132,7 +132,5 @@ int main(int argc, char* argv[]) { } std::cerr << "\n"; } - - google::protobuf::ShutdownProtobufLibrary(); } diff --git a/examples/osmium_convert.cpp b/examples/osmium_convert.cpp index 7956e1181c1..4f2ba33185f 100644 --- a/examples/osmium_convert.cpp +++ b/examples/osmium_convert.cpp @@ -106,7 +106,6 @@ int main(int argc, char* argv[]) { exit_code = 1; } - google::protobuf::ShutdownProtobufLibrary(); return exit_code; } diff --git a/examples/osmium_count.cpp b/examples/osmium_count.cpp index dca18bfdc0a..baea153b88f 100644 --- a/examples/osmium_count.cpp +++ b/examples/osmium_count.cpp @@ -7,6 +7,7 @@ */ +#include #include #include @@ -15,9 +16,9 @@ struct CountHandler : public osmium::handler::Handler { - int nodes = 0; - int ways = 0; - int relations = 0; + uint64_t nodes = 0; + uint64_t ways = 0; + uint64_t relations = 0; void node(osmium::Node&) { ++nodes; @@ -51,7 +52,5 @@ int main(int argc, char* argv[]) { std::cout << "Nodes: " << handler.nodes << "\n"; std::cout << "Ways: " << handler.ways << "\n"; std::cout << "Relations: " << handler.relations << "\n"; - - google::protobuf::ShutdownProtobufLibrary(); } diff --git a/examples/osmium_create_node_cache.cpp b/examples/osmium_create_node_cache.cpp index 74f7596fe77..359fa1978a8 100644 --- a/examples/osmium_create_node_cache.cpp +++ b/examples/osmium_create_node_cache.cpp @@ -12,8 +12,7 @@ #include #include -#include -#include +#include #include #include @@ -51,8 +50,6 @@ int main(int argc, char* argv[]) { osmium::apply(reader, location_handler); reader.close(); - google::protobuf::ShutdownProtobufLibrary(); - return 0; } diff --git a/examples/osmium_debug.cpp b/examples/osmium_debug.cpp index 6878ed1e8d1..365fc729754 100644 --- a/examples/osmium_debug.cpp +++ b/examples/osmium_debug.cpp @@ -46,7 +46,5 @@ int main(int argc, char* argv[]) { } reader.close(); - - google::protobuf::ShutdownProtobufLibrary(); } diff --git a/examples/osmium_read.cpp b/examples/osmium_read.cpp index 1bb0299662c..653600684a1 100644 --- a/examples/osmium_read.cpp +++ b/examples/osmium_read.cpp @@ -26,7 +26,5 @@ int main(int argc, char* argv[]) { } reader.close(); - - google::protobuf::ShutdownProtobufLibrary(); } diff --git a/examples/osmium_serdump.cpp b/examples/osmium_serdump.cpp index a774a8dbd38..9ab26e4eeee 100644 --- a/examples/osmium_serdump.cpp +++ b/examples/osmium_serdump.cpp @@ -19,8 +19,7 @@ # include #endif -#include -#include +#include #include #include @@ -203,7 +202,5 @@ int main(int argc, char* argv[]) { map_relation2relation.dump_as_list(fd); close(fd); } - - google::protobuf::ShutdownProtobufLibrary(); } diff --git a/examples/osmium_toogr.cpp b/examples/osmium_toogr.cpp index 6d8ab8d940d..7c5a965c503 100644 --- a/examples/osmium_toogr.cpp +++ b/examples/osmium_toogr.cpp @@ -234,8 +234,6 @@ int main(int argc, char* argv[]) { osmium::apply(reader, location_handler, ogr_handler); reader.close(); - google::protobuf::ShutdownProtobufLibrary(); - int locations_fd = open("locations.dump", O_WRONLY | O_CREAT, 0644); if (locations_fd < 0) { throw std::system_error(errno, std::system_category(), "Open failed"); diff --git a/examples/osmium_toogr2.cpp b/examples/osmium_toogr2.cpp index a966c5edcca..e1b505688bd 100644 --- a/examples/osmium_toogr2.cpp +++ b/examples/osmium_toogr2.cpp @@ -327,7 +327,5 @@ int main(int argc, char* argv[]) { } std::cerr << "\n"; } - - google::protobuf::ShutdownProtobufLibrary(); } diff --git a/examples/osmium_toogr2_exp.cpp b/examples/osmium_toogr2_exp.cpp index 474da96d013..db8d5cf4ec3 100644 --- a/examples/osmium_toogr2_exp.cpp +++ b/examples/osmium_toogr2_exp.cpp @@ -301,7 +301,5 @@ int main(int argc, char* argv[]) { } std::cerr << "\n"; } - - google::protobuf::ShutdownProtobufLibrary(); } diff --git a/examples/osmium_use_node_cache.cpp b/examples/osmium_use_node_cache.cpp index 6b8f964abaa..cfee6df4683 100644 --- a/examples/osmium_use_node_cache.cpp +++ b/examples/osmium_use_node_cache.cpp @@ -12,8 +12,7 @@ #include #include -#include -#include +#include #include #include @@ -64,8 +63,6 @@ int main(int argc, char* argv[]) { osmium::apply(reader, location_handler, handler); reader.close(); - google::protobuf::ShutdownProtobufLibrary(); - return 0; } diff --git a/include/boost_unicode_iterator.hpp b/include/boost_unicode_iterator.hpp deleted file mode 100644 index 3a7b68fbe7a..00000000000 --- a/include/boost_unicode_iterator.hpp +++ /dev/null @@ -1,776 +0,0 @@ -/* - * - * Copyright (c) 2004 - * John Maddock - * - * Use, modification and distribution are subject to the - * Boost Software License, Version 1.0. (See accompanying file - * LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) - * - */ - - /* - * LOCATION: see http://www.boost.org for most recent version. - * FILE unicode_iterator.hpp - * VERSION see - * DESCRIPTION: Iterator adapters for converting between different Unicode encodings. - */ - -/**************************************************************************** - -Contents: -~~~~~~~~~ - -1) Read Only, Input Adapters: -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -template -class u32_to_u8_iterator; - -Adapts sequence of UTF-32 code points to "look like" a sequence of UTF-8. - -template -class u8_to_u32_iterator; - -Adapts sequence of UTF-8 code points to "look like" a sequence of UTF-32. - -template -class u32_to_u16_iterator; - -Adapts sequence of UTF-32 code points to "look like" a sequence of UTF-16. - -template -class u16_to_u32_iterator; - -Adapts sequence of UTF-16 code points to "look like" a sequence of UTF-32. - -2) Single pass output iterator adapters: - -template -class utf8_output_iterator; - -Accepts UTF-32 code points and forwards them on as UTF-8 code points. - -template -class utf16_output_iterator; - -Accepts UTF-32 code points and forwards them on as UTF-16 code points. - -****************************************************************************/ - -#ifndef BOOST_REGEX_UNICODE_ITERATOR_HPP -#define BOOST_REGEX_UNICODE_ITERATOR_HPP -#include -#include -#include -#include -#include -#include -#ifndef BOOST_NO_STD_LOCALE -#include -#include -#endif -#include // CHAR_BIT - -namespace boost{ - -namespace detail{ - -static const ::boost::uint16_t high_surrogate_base = 0xD7C0u; -static const ::boost::uint16_t low_surrogate_base = 0xDC00u; -static const ::boost::uint32_t ten_bit_mask = 0x3FFu; - -inline bool is_high_surrogate(::boost::uint16_t v) -{ - return (v & 0xFFFFFC00u) == 0xd800u; -} -inline bool is_low_surrogate(::boost::uint16_t v) -{ - return (v & 0xFFFFFC00u) == 0xdc00u; -} -template -inline bool is_surrogate(T v) -{ - return (v & 0xFFFFF800u) == 0xd800; -} - -inline unsigned utf8_byte_count(boost::uint8_t c) -{ - // if the most significant bit with a zero in it is in position - // 8-N then there are N bytes in this UTF-8 sequence: - boost::uint8_t mask = 0x80u; - unsigned result = 0; - while(c & mask) - { - ++result; - mask >>= 1; - } - return (result == 0) ? 1 : ((result > 4) ? 4 : result); -} - -inline unsigned utf8_trailing_byte_count(boost::uint8_t c) -{ - return utf8_byte_count(c) - 1; -} - -#ifdef BOOST_MSVC -#pragma warning(push) -#pragma warning(disable:4100) -#endif -inline void invalid_utf32_code_point(::boost::uint32_t val) -{ -#ifndef BOOST_NO_STD_LOCALE - std::stringstream ss; - ss << "Invalid UTF-32 code point U+" << std::showbase << std::hex << val << " encountered while trying to encode UTF-16 sequence"; - std::out_of_range e(ss.str()); -#else - std::out_of_range e("Invalid UTF-32 code point encountered while trying to encode UTF-16 sequence"); -#endif - boost::throw_exception(e); -} -#ifdef BOOST_MSVC -#pragma warning(pop) -#endif - - -} // namespace detail - -template -class u32_to_u16_iterator - : public boost::iterator_facade, U16Type, std::bidirectional_iterator_tag, const U16Type> -{ - typedef boost::iterator_facade, U16Type, std::bidirectional_iterator_tag, const U16Type> base_type; - -#if !defined(BOOST_NO_STD_ITERATOR_TRAITS) && !defined(BOOST_NO_TEMPLATE_PARTIAL_SPECIALIZATION) - typedef typename std::iterator_traits::value_type base_value_type; - - BOOST_STATIC_ASSERT(sizeof(base_value_type)*CHAR_BIT == 32); - BOOST_STATIC_ASSERT(sizeof(U16Type)*CHAR_BIT == 16); -#endif - -public: - typename base_type::reference - dereference()const - { - if(m_current == 2) - extract_current(); - return m_values[m_current]; - } - bool equal(const u32_to_u16_iterator& that)const - { - if(m_position == that.m_position) - { - // Both m_currents must be equal, or both even - // this is the same as saying their sum must be even: - return (m_current + that.m_current) & 1u ? false : true; - } - return false; - } - void increment() - { - // if we have a pending read then read now, so that we know whether - // to skip a position, or move to a low-surrogate: - if(m_current == 2) - { - // pending read: - extract_current(); - } - // move to the next surrogate position: - ++m_current; - // if we've reached the end skip a position: - if(m_values[m_current] == 0) - { - m_current = 2; - ++m_position; - } - } - void decrement() - { - if(m_current != 1) - { - // decrementing an iterator always leads to a valid position: - --m_position; - extract_current(); - m_current = m_values[1] ? 1 : 0; - } - else - { - m_current = 0; - } - } - BaseIterator base()const - { - return m_position; - } - // construct: - u32_to_u16_iterator() : m_position(), m_current(0) - { - m_values[0] = 0; - m_values[1] = 0; - m_values[2] = 0; - } - u32_to_u16_iterator(BaseIterator b) : m_position(b), m_current(2) - { - m_values[0] = 0; - m_values[1] = 0; - m_values[2] = 0; - } -private: - - void extract_current()const - { - // begin by checking for a code point out of range: - ::boost::uint32_t v = *m_position; - if(v >= 0x10000u) - { - if(v > 0x10FFFFu) - detail::invalid_utf32_code_point(*m_position); - // split into two surrogates: - m_values[0] = static_cast(v >> 10) + detail::high_surrogate_base; - m_values[1] = static_cast(v & detail::ten_bit_mask) + detail::low_surrogate_base; - m_current = 0; - BOOST_ASSERT(detail::is_high_surrogate(m_values[0])); - BOOST_ASSERT(detail::is_low_surrogate(m_values[1])); - } - else - { - // 16-bit code point: - m_values[0] = static_cast(*m_position); - m_values[1] = 0; - m_current = 0; - // value must not be a surrogate: - if(detail::is_surrogate(m_values[0])) - detail::invalid_utf32_code_point(*m_position); - } - } - BaseIterator m_position; - mutable U16Type m_values[3]; - mutable unsigned m_current; -}; - -template -class u16_to_u32_iterator - : public boost::iterator_facade, U32Type, std::bidirectional_iterator_tag, const U32Type> -{ - typedef boost::iterator_facade, U32Type, std::bidirectional_iterator_tag, const U32Type> base_type; - // special values for pending iterator reads: - BOOST_STATIC_CONSTANT(U32Type, pending_read = 0xffffffffu); - -#if !defined(BOOST_NO_STD_ITERATOR_TRAITS) && !defined(BOOST_NO_TEMPLATE_PARTIAL_SPECIALIZATION) - typedef typename std::iterator_traits::value_type base_value_type; - - BOOST_STATIC_ASSERT(sizeof(base_value_type)*CHAR_BIT == 16); - BOOST_STATIC_ASSERT(sizeof(U32Type)*CHAR_BIT == 32); -#endif - -public: - typename base_type::reference - dereference()const - { - if(m_value == pending_read) - extract_current(); - return m_value; - } - bool equal(const u16_to_u32_iterator& that)const - { - return m_position == that.m_position; - } - void increment() - { - // skip high surrogate first if there is one: - if(detail::is_high_surrogate(*m_position)) ++m_position; - ++m_position; - m_value = pending_read; - } - void decrement() - { - --m_position; - // if we have a low surrogate then go back one more: - if(detail::is_low_surrogate(*m_position)) - --m_position; - m_value = pending_read; - } - BaseIterator base()const - { - return m_position; - } - // construct: - u16_to_u32_iterator() : m_position() - { - m_value = pending_read; - } - u16_to_u32_iterator(BaseIterator b) : m_position(b) - { - m_value = pending_read; - } - // - // Range checked version: - // - u16_to_u32_iterator(BaseIterator b, BaseIterator start, BaseIterator end) : m_position(b) - { - m_value = pending_read; - // - // The range must not start with a low surrogate, or end in a high surrogate, - // otherwise we run the risk of running outside the underlying input range. - // Likewise b must not be located at a low surrogate. - // - boost::uint16_t val; - if(start != end) - { - if((b != start) && (b != end)) - { - val = *b; - if(detail::is_surrogate(val) && ((val & 0xFC00u) == 0xDC00u)) - invalid_code_point(val); - } - val = *start; - if(detail::is_surrogate(val) && ((val & 0xFC00u) == 0xDC00u)) - invalid_code_point(val); - val = *--end; - if(detail::is_high_surrogate(val)) - invalid_code_point(val); - } - } -private: - static void invalid_code_point(::boost::uint16_t val) - { -#ifndef BOOST_NO_STD_LOCALE - std::stringstream ss; - ss << "Misplaced UTF-16 surrogate U+" << std::showbase << std::hex << val << " encountered while trying to encode UTF-32 sequence"; - std::out_of_range e(ss.str()); -#else - std::out_of_range e("Misplaced UTF-16 surrogate encountered while trying to encode UTF-32 sequence"); -#endif - boost::throw_exception(e); - } - void extract_current()const - { - m_value = static_cast(static_cast< ::boost::uint16_t>(*m_position)); - // if the last value is a high surrogate then adjust m_position and m_value as needed: - if(detail::is_high_surrogate(*m_position)) - { - // precondition; next value must have be a low-surrogate: - BaseIterator next(m_position); - ::boost::uint16_t t = *++next; - if((t & 0xFC00u) != 0xDC00u) - invalid_code_point(t); - m_value = (m_value - detail::high_surrogate_base) << 10; - m_value |= (static_cast(static_cast< ::boost::uint16_t>(t)) & detail::ten_bit_mask); - } - // postcondition; result must not be a surrogate: - if(detail::is_surrogate(m_value)) - invalid_code_point(static_cast< ::boost::uint16_t>(m_value)); - } - BaseIterator m_position; - mutable U32Type m_value; -}; - -template -class u32_to_u8_iterator - : public boost::iterator_facade, U8Type, std::bidirectional_iterator_tag, const U8Type> -{ - typedef boost::iterator_facade, U8Type, std::bidirectional_iterator_tag, const U8Type> base_type; - -#if !defined(BOOST_NO_STD_ITERATOR_TRAITS) && !defined(BOOST_NO_TEMPLATE_PARTIAL_SPECIALIZATION) - typedef typename std::iterator_traits::value_type base_value_type; - - BOOST_STATIC_ASSERT(sizeof(base_value_type)*CHAR_BIT == 32); - BOOST_STATIC_ASSERT(sizeof(U8Type)*CHAR_BIT == 8); -#endif - -public: - typename base_type::reference - dereference()const - { - if(m_current == 4) - extract_current(); - return m_values[m_current]; - } - bool equal(const u32_to_u8_iterator& that)const - { - if(m_position == that.m_position) - { - // either the m_current's must be equal, or one must be 0 and - // the other 4: which means neither must have bits 1 or 2 set: - return (m_current == that.m_current) - || (((m_current | that.m_current) & 3) == 0); - } - return false; - } - void increment() - { - // if we have a pending read then read now, so that we know whether - // to skip a position, or move to a low-surrogate: - if(m_current == 4) - { - // pending read: - extract_current(); - } - // move to the next surrogate position: - ++m_current; - // if we've reached the end skip a position: - if(m_values[m_current] == 0) - { - m_current = 4; - ++m_position; - } - } - void decrement() - { - if((m_current & 3) == 0) - { - --m_position; - extract_current(); - m_current = 3; - while(m_current && (m_values[m_current] == 0)) - --m_current; - } - else - --m_current; - } - BaseIterator base()const - { - return m_position; - } - // construct: - u32_to_u8_iterator() : m_position(), m_current(0) - { - m_values[0] = 0; - m_values[1] = 0; - m_values[2] = 0; - m_values[3] = 0; - m_values[4] = 0; - } - u32_to_u8_iterator(BaseIterator b) : m_position(b), m_current(4) - { - m_values[0] = 0; - m_values[1] = 0; - m_values[2] = 0; - m_values[3] = 0; - m_values[4] = 0; - } -private: - - void extract_current()const - { - boost::uint32_t c = *m_position; - if(c > 0x10FFFFu) - detail::invalid_utf32_code_point(c); - if(c < 0x80u) - { - m_values[0] = static_cast(c); - m_values[1] = static_cast(0u); - m_values[2] = static_cast(0u); - m_values[3] = static_cast(0u); - } - else if(c < 0x800u) - { - m_values[0] = static_cast(0xC0u + (c >> 6)); - m_values[1] = static_cast(0x80u + (c & 0x3Fu)); - m_values[2] = static_cast(0u); - m_values[3] = static_cast(0u); - } - else if(c < 0x10000u) - { - m_values[0] = static_cast(0xE0u + (c >> 12)); - m_values[1] = static_cast(0x80u + ((c >> 6) & 0x3Fu)); - m_values[2] = static_cast(0x80u + (c & 0x3Fu)); - m_values[3] = static_cast(0u); - } - else - { - m_values[0] = static_cast(0xF0u + (c >> 18)); - m_values[1] = static_cast(0x80u + ((c >> 12) & 0x3Fu)); - m_values[2] = static_cast(0x80u + ((c >> 6) & 0x3Fu)); - m_values[3] = static_cast(0x80u + (c & 0x3Fu)); - } - m_current= 0; - } - BaseIterator m_position; - mutable U8Type m_values[5]; - mutable unsigned m_current; -}; - -template -class u8_to_u32_iterator - : public boost::iterator_facade, U32Type, std::bidirectional_iterator_tag, const U32Type> -{ - typedef boost::iterator_facade, U32Type, std::bidirectional_iterator_tag, const U32Type> base_type; - // special values for pending iterator reads: - BOOST_STATIC_CONSTANT(U32Type, pending_read = 0xffffffffu); - -#if !defined(BOOST_NO_STD_ITERATOR_TRAITS) && !defined(BOOST_NO_TEMPLATE_PARTIAL_SPECIALIZATION) - typedef typename std::iterator_traits::value_type base_value_type; - - BOOST_STATIC_ASSERT(sizeof(base_value_type)*CHAR_BIT == 8); - BOOST_STATIC_ASSERT(sizeof(U32Type)*CHAR_BIT == 32); -#endif - -public: - typename base_type::reference - dereference()const - { - if(m_value == pending_read) - extract_current(); - return m_value; - } - bool equal(const u8_to_u32_iterator& that)const - { - return m_position == that.m_position; - } - void increment() - { - // We must not start with a continuation character: - if((static_cast(*m_position) & 0xC0) == 0x80) - invalid_sequence(); - // skip high surrogate first if there is one: - unsigned c = detail::utf8_byte_count(*m_position); - if(m_value == pending_read) - { - // Since we haven't read in a value, we need to validate the code points: - for(unsigned i = 0; i < c; ++i) - { - ++m_position; - // We must have a continuation byte: - if((i != c - 1) && ((static_cast(*m_position) & 0xC0) != 0x80)) - invalid_sequence(); - } - } - else - { - std::advance(m_position, c); - } - m_value = pending_read; - } - void decrement() - { - // Keep backtracking until we don't have a trailing character: - unsigned count = 0; - while((*--m_position & 0xC0u) == 0x80u) ++count; - // now check that the sequence was valid: - if(count != detail::utf8_trailing_byte_count(*m_position)) - invalid_sequence(); - m_value = pending_read; - } - BaseIterator base()const - { - return m_position; - } - // construct: - u8_to_u32_iterator() : m_position() - { - m_value = pending_read; - } - u8_to_u32_iterator(BaseIterator b) : m_position(b) - { - m_value = pending_read; - } - // - // Checked constructor: - // - u8_to_u32_iterator(BaseIterator b, BaseIterator start, BaseIterator end) : m_position(b) - { - m_value = pending_read; - // - // We must not start with a continuation character, or end with a - // truncated UTF-8 sequence otherwise we run the risk of going past - // the start/end of the underlying sequence: - // - if(start != end) - { - unsigned char v = *start; - if((v & 0xC0u) == 0x80u) - invalid_sequence(); - if((b != start) && (b != end) && ((*b & 0xC0u) == 0x80u)) - invalid_sequence(); - BaseIterator pos = end; - do - { - v = *--pos; - } - while((start != pos) && ((v & 0xC0u) == 0x80u)); - std::ptrdiff_t extra = detail::utf8_byte_count(v); - if(std::distance(pos, end) < extra) - invalid_sequence(); - } - } -private: - static void invalid_sequence() - { - std::out_of_range e("Invalid UTF-8 sequence encountered while trying to encode UTF-32 character"); - boost::throw_exception(e); - } - void extract_current()const - { - m_value = static_cast(static_cast< ::boost::uint8_t>(*m_position)); - // we must not have a continuation character: - if((m_value & 0xC0u) == 0x80u) - invalid_sequence(); - // see how many extra bytes we have: - unsigned extra = detail::utf8_trailing_byte_count(*m_position); - // extract the extra bits, 6 from each extra byte: - BaseIterator next(m_position); - for(unsigned c = 0; c < extra; ++c) - { - ++next; - m_value <<= 6; - // We must have a continuation byte: - if((static_cast(*next) & 0xC0) != 0x80) - invalid_sequence(); - m_value += static_cast(*next) & 0x3Fu; - } - // we now need to remove a few of the leftmost bits, but how many depends - // upon how many extra bytes we've extracted: - static const boost::uint32_t masks[4] = - { - 0x7Fu, - 0x7FFu, - 0xFFFFu, - 0x1FFFFFu, - }; - m_value &= masks[extra]; - // check the result: - if(m_value > static_cast(0x10FFFFu)) - invalid_sequence(); - } - BaseIterator m_position; - mutable U32Type m_value; -}; - -template -class utf16_output_iterator -{ -public: - typedef void difference_type; - typedef void value_type; - typedef boost::uint32_t* pointer; - typedef boost::uint32_t& reference; - typedef std::output_iterator_tag iterator_category; - - utf16_output_iterator(const BaseIterator& b) - : m_position(b){} - utf16_output_iterator(const utf16_output_iterator& that) - : m_position(that.m_position){} - utf16_output_iterator& operator=(const utf16_output_iterator& that) - { - m_position = that.m_position; - return *this; - } - const utf16_output_iterator& operator*()const - { - return *this; - } - void operator=(boost::uint32_t val)const - { - push(val); - } - utf16_output_iterator& operator++() - { - return *this; - } - utf16_output_iterator& operator++(int) - { - return *this; - } - BaseIterator base()const - { - return m_position; - } -private: - void push(boost::uint32_t v)const - { - if(v >= 0x10000u) - { - // begin by checking for a code point out of range: - if(v > 0x10FFFFu) - detail::invalid_utf32_code_point(v); - // split into two surrogates: - *m_position++ = static_cast(v >> 10) + detail::high_surrogate_base; - *m_position++ = static_cast(v & detail::ten_bit_mask) + detail::low_surrogate_base; - } - else - { - // 16-bit code point: - // value must not be a surrogate: - if(detail::is_surrogate(v)) - detail::invalid_utf32_code_point(v); - *m_position++ = static_cast(v); - } - } - mutable BaseIterator m_position; -}; - -template -class utf8_output_iterator -{ -public: - typedef void difference_type; - typedef void value_type; - typedef boost::uint32_t* pointer; - typedef boost::uint32_t& reference; - typedef std::output_iterator_tag iterator_category; - - utf8_output_iterator(const BaseIterator& b) - : m_position(b){} - utf8_output_iterator(const utf8_output_iterator& that) - : m_position(that.m_position){} - utf8_output_iterator& operator=(const utf8_output_iterator& that) - { - m_position = that.m_position; - return *this; - } - const utf8_output_iterator& operator*()const - { - return *this; - } - void operator=(boost::uint32_t val)const - { - push(val); - } - utf8_output_iterator& operator++() - { - return *this; - } - utf8_output_iterator& operator++(int) - { - return *this; - } - BaseIterator base()const - { - return m_position; - } -private: - void push(boost::uint32_t c)const - { - if(c > 0x10FFFFu) - detail::invalid_utf32_code_point(c); - if(c < 0x80u) - { - *m_position++ = static_cast(c); - } - else if(c < 0x800u) - { - *m_position++ = static_cast(0xC0u + (c >> 6)); - *m_position++ = static_cast(0x80u + (c & 0x3Fu)); - } - else if(c < 0x10000u) - { - *m_position++ = static_cast(0xE0u + (c >> 12)); - *m_position++ = static_cast(0x80u + ((c >> 6) & 0x3Fu)); - *m_position++ = static_cast(0x80u + (c & 0x3Fu)); - } - else - { - *m_position++ = static_cast(0xF0u + (c >> 18)); - *m_position++ = static_cast(0x80u + ((c >> 12) & 0x3Fu)); - *m_position++ = static_cast(0x80u + ((c >> 6) & 0x3Fu)); - *m_position++ = static_cast(0x80u + (c & 0x3Fu)); - } - } - mutable BaseIterator m_position; -}; - -} // namespace boost - -#endif // BOOST_REGEX_UNICODE_ITERATOR_HPP - diff --git a/include/mmap_for_windows.hpp b/include/mmap_for_windows.hpp deleted file mode 100644 index abe62d64039..00000000000 --- a/include/mmap_for_windows.hpp +++ /dev/null @@ -1,103 +0,0 @@ -#ifndef MMAP_FOR_WINDOWS_HPP -#define MMAP_FOR_WINDOWS_HPP - -/* mmap() replacement for Windows - * - * Author: Mike Frysinger - * Placed into the public domain - */ - -/* References: - * CreateFileMapping: http://msdn.microsoft.com/en-us/library/aa366537(VS.85).aspx - * CloseHandle: http://msdn.microsoft.com/en-us/library/ms724211(VS.85).aspx - * MapViewOfFile: http://msdn.microsoft.com/en-us/library/aa366761(VS.85).aspx - * UnmapViewOfFile: http://msdn.microsoft.com/en-us/library/aa366882(VS.85).aspx - */ - -#include -#include -#include - -#define PROT_READ 0x1 -#define PROT_WRITE 0x2 -/* This flag is only available in WinXP+ */ -#ifdef FILE_MAP_EXECUTE -#define PROT_EXEC 0x4 -#else -#define PROT_EXEC 0x0 -#define FILE_MAP_EXECUTE 0 -#endif - -#define MAP_SHARED 0x01 -#define MAP_PRIVATE 0x02 -#define MAP_ANONYMOUS 0x20 -#define MAP_ANON MAP_ANONYMOUS -#define MAP_FAILED ((void *) -1) - -static DWORD dword_hi(uint64_t x) { - return static_cast(x >> 32); -} - -static DWORD dword_lo(uint64_t x) { - return static_cast(x & 0xffffffff); -} - -static void *mmap(void *start, size_t length, int prot, int flags, int fd, off_t offset) -{ - if (prot & ~(PROT_READ | PROT_WRITE | PROT_EXEC)) - return MAP_FAILED; - if (fd == -1) { - if (!(flags & MAP_ANON) || offset) - return MAP_FAILED; - } else if (flags & MAP_ANON) - return MAP_FAILED; - - DWORD flProtect; - if (prot & PROT_WRITE) { - if (prot & PROT_EXEC) - flProtect = PAGE_EXECUTE_READWRITE; - else - flProtect = PAGE_READWRITE; - } else if (prot & PROT_EXEC) { - if (prot & PROT_READ) - flProtect = PAGE_EXECUTE_READ; - else if (prot & PROT_EXEC) - flProtect = PAGE_EXECUTE; - } else - flProtect = PAGE_READONLY; - - uint64_t end = static_cast(length) + offset; - HANDLE mmap_fd; - if (fd == -1) - mmap_fd = INVALID_HANDLE_VALUE; - else - mmap_fd = (HANDLE)_get_osfhandle(fd); - - HANDLE h = CreateFileMapping(mmap_fd, NULL, flProtect, dword_hi(end), dword_lo(end), NULL); - if (h == NULL) - return MAP_FAILED; - - DWORD dwDesiredAccess; - if (prot & PROT_WRITE) - dwDesiredAccess = FILE_MAP_WRITE; - else - dwDesiredAccess = FILE_MAP_READ; - if (prot & PROT_EXEC) - dwDesiredAccess |= FILE_MAP_EXECUTE; - if (flags & MAP_PRIVATE) - dwDesiredAccess |= FILE_MAP_COPY; - void *ret = MapViewOfFile(h, dwDesiredAccess, dword_hi(offset), dword_lo(offset), length); - if (ret == NULL) { - CloseHandle(h); - ret = MAP_FAILED; - } - return ret; -} - -static int munmap(void *addr, size_t length) -{ - return UnmapViewOfFile(addr) ? 0 : -1; - /* ruh-ro, we leaked handle from CreateFileMapping() ... */ -} - -#endif diff --git a/include/osmium/area/assembler.hpp b/include/osmium/area/assembler.hpp index 0a5f12323e1..f1729910913 100644 --- a/include/osmium/area/assembler.hpp +++ b/include/osmium/area/assembler.hpp @@ -171,7 +171,7 @@ namespace osmium { } void add_tags_to_area(osmium::builder::AreaBuilder& builder, const osmium::Relation& relation) const { - auto count = std::count_if(relation.tags().begin(), relation.tags().end(), filter()); + const auto count = std::count_if(relation.tags().begin(), relation.tags().end(), filter()); if (debug()) { std::cerr << " found " << count << " tags on relation (without ignored ones)\n"; @@ -331,7 +331,7 @@ namespace osmium { if (debug()) { std::cerr << " has_closed_subring_back()\n"; } - auto end = ring.segments().end(); + const auto end = ring.segments().end(); for (auto it = ring.segments().begin() + 1; it != end - 1; ++it) { if (has_same_location(nr, it->first())) { split_off_subring(ring, it, it, end); @@ -348,7 +348,7 @@ namespace osmium { if (debug()) { std::cerr << " has_closed_subring_front()\n"; } - auto end = ring.segments().end(); + const auto end = ring.segments().end(); for (auto it = ring.segments().begin() + 1; it != end - 1; ++it) { if (has_same_location(nr, it->second())) { split_off_subring(ring, it, ring.segments().begin(), it+1); @@ -366,22 +366,22 @@ namespace osmium { osmium::area::detail::ProtoRing::segments_type segments(ring.segments().size()); std::copy(ring.segments().begin(), ring.segments().end(), segments.begin()); std::sort(segments.begin(), segments.end()); - auto it = std::adjacent_find(segments.begin(), segments.end(), [this](const osmium::area::detail::NodeRefSegment& s1, const osmium::area::detail::NodeRefSegment& s2) { + const auto it = std::adjacent_find(segments.begin(), segments.end(), [this](const osmium::area::detail::NodeRefSegment& s1, const osmium::area::detail::NodeRefSegment& s2) { return has_same_location(s1.first(), s2.first()); }); if (it == segments.end()) { return false; } - auto r1 = std::find_first_of(ring.segments().begin(), ring.segments().end(), it, it+1); + const auto r1 = std::find_first_of(ring.segments().begin(), ring.segments().end(), it, it+1); assert(r1 != ring.segments().end()); - auto r2 = std::find_first_of(ring.segments().begin(), ring.segments().end(), it+1, it+2); + const auto r2 = std::find_first_of(ring.segments().begin(), ring.segments().end(), it+1, it+2); assert(r2 != ring.segments().end()); if (debug()) { std::cerr << " found subring in ring " << ring << " at " << it->first() << "\n"; } - auto m = std::minmax(r1, r2); + const auto m = std::minmax(r1, r2); ProtoRing new_ring(m.first, m.second); ring.remove_segments(m.first, m.second); @@ -537,7 +537,7 @@ namespace osmium { } for (const auto ringptr : m_outer_rings) { - for (const auto segment : ringptr->segments()) { + for (const auto& segment : ringptr->segments()) { if (!segment.role_outer()) { ++m_inner_outer_mismatches; if (debug()) { @@ -550,7 +550,7 @@ namespace osmium { } } for (const auto ringptr : m_inner_rings) { - for (const auto segment : ringptr->segments()) { + for (const auto& segment : ringptr->segments()) { if (!segment.role_inner()) { ++m_inner_outer_mismatches; if (debug()) { diff --git a/include/osmium/area/multipolygon_collector.hpp b/include/osmium/area/multipolygon_collector.hpp index 84a52622b42..bf2a4ce86a1 100644 --- a/include/osmium/area/multipolygon_collector.hpp +++ b/include/osmium/area/multipolygon_collector.hpp @@ -41,6 +41,8 @@ DEALINGS IN THE SOFTWARE. #include #include +#include +#include #include #include #include @@ -49,8 +51,6 @@ DEALINGS IN THE SOFTWARE. namespace osmium { - struct invalid_location; - namespace relations { class RelationMeta; } @@ -107,8 +107,8 @@ namespace osmium { } /** - * We are interested in all relations tagged with type=multipolygon or - * type=boundary. + * We are interested in all relations tagged with type=multipolygon + * or type=boundary. * * Overwritten from the base class. */ @@ -142,15 +142,22 @@ namespace osmium { * Overwritten from the base class. */ void way_not_in_any_relation(const osmium::Way& way) { - if (way.nodes().size() > 3 && way.ends_have_same_location()) { - // way is closed and has enough nodes, build simple multipolygon - try { + // you need at least 4 nodes to make up a polygon + if (way.nodes().size() <= 3) { + return; + } + try { + if (!way.nodes().front().location() || !way.nodes().back().location()) { + throw osmium::invalid_location("invalid location"); + } + if (way.ends_have_same_location()) { + // way is closed and has enough nodes, build simple multipolygon TAssembler assembler(m_assembler_config); assembler(way, m_output_buffer); possibly_flush_output_buffer(); - } catch (osmium::invalid_location&) { - // XXX ignore } + } catch (osmium::invalid_location&) { + // XXX ignore } } diff --git a/include/osmium/builder/builder.hpp b/include/osmium/builder/builder.hpp index dcb95e2ac3c..4424d88e197 100644 --- a/include/osmium/builder/builder.hpp +++ b/include/osmium/builder/builder.hpp @@ -147,6 +147,7 @@ namespace osmium { * @param length Length of data in bytes. If data is a * \0-terminated string, length must contain the * \0 byte. + * @returns The number of bytes appended (length). */ osmium::memory::item_size_type append(const char* data, const osmium::memory::item_size_type length) { unsigned char* target = m_buffer.reserve_space(length); @@ -156,11 +157,24 @@ namespace osmium { /** * Append \0-terminated string to buffer. + * + * @param str \0-terminated string. + * @returns The number of bytes appended (strlen(str) + 1). */ osmium::memory::item_size_type append(const char* str) { return append(str, static_cast(std::strlen(str) + 1)); } + /** + * Append '\0' to the buffer. + * + * @returns The number of bytes appended (always 1). + */ + osmium::memory::item_size_type append_zero() { + *m_buffer.reserve_space(1) = '\0'; + return 1; + } + /// Return the buffer this builder is using. osmium::memory::Buffer& buffer() noexcept { return m_buffer; @@ -188,11 +202,11 @@ namespace osmium { * Add user name to buffer. * * @param user Pointer to user name. - * @param length Length of user name including \0 byte. + * @param length Length of user name (without \0 termination). */ void add_user(const char* user, const string_size_type length) { - object().set_user_size(length); - add_size(append(user, length)); + object().set_user_size(length + 1); + add_size(append(user, length) + append_zero()); add_padding(true); } @@ -202,7 +216,7 @@ namespace osmium { * @param user Pointer to \0-terminated user name. */ void add_user(const char* user) { - add_user(user, static_cast_with_assert(std::strlen(user) + 1)); + add_user(user, static_cast_with_assert(std::strlen(user))); } /** @@ -211,7 +225,7 @@ namespace osmium { * @param user User name. */ void add_user(const std::string& user) { - add_user(user.data(), static_cast_with_assert(user.size() + 1)); + add_user(user.data(), static_cast_with_assert(user.size())); } }; // class ObjectBuilder diff --git a/include/osmium/builder/osm_object_builder.hpp b/include/osmium/builder/osm_object_builder.hpp index 058f89e8ebc..074076ce6f5 100644 --- a/include/osmium/builder/osm_object_builder.hpp +++ b/include/osmium/builder/osm_object_builder.hpp @@ -72,13 +72,25 @@ namespace osmium { /** * Add tag to buffer. * - * @param key Tag key. - * @param value Tag value. + * @param key Tag key (0-terminated string). + * @param value Tag value (0-terminated string). */ void add_tag(const char* key, const char* value) { add_size(append(key) + append(value)); } + /** + * Add tag to buffer. + * + * @param key Pointer to tag key. + * @param key_length Length of key (not including the \0 byte). + * @param value Pointer to tag value. + * @param value_length Length of value (not including the \0 byte). + */ + void add_tag(const char* key, const string_size_type key_length, const char* value, const string_size_type value_length) { + add_size(append(key, key_length) + append_zero() + append(value, value_length) + append_zero()); + } + /** * Add tag to buffer. * @@ -128,11 +140,11 @@ namespace osmium { * @param member Relation member object where the length of the role * will be set. * @param role The role. - * @param length Length of role string including \0 termination. + * @param length Length of role (without \0 termination). */ void add_role(osmium::RelationMember& member, const char* role, const string_size_type length) { - member.set_role_size(length); - add_size(append(role, length)); + member.set_role_size(length + 1); + add_size(append(role, length) + append_zero()); add_padding(true); } @@ -144,7 +156,7 @@ namespace osmium { * @param role \0-terminated role. */ void add_role(osmium::RelationMember& member, const char* role) { - add_role(member, role, static_cast_with_assert(std::strlen(role) + 1)); + add_role(member, role, static_cast_with_assert(std::strlen(role))); } /** @@ -155,7 +167,7 @@ namespace osmium { * @param role Role. */ void add_role(osmium::RelationMember& member, const std::string& role) { - add_role(member, role.data(), static_cast_with_assert(role.size() + 1)); + add_role(member, role.data(), static_cast_with_assert(role.size())); } public: @@ -174,20 +186,35 @@ namespace osmium { * @param type The type (node, way, or relation). * @param ref The ID of the member. * @param role The role of the member. + * @param role_length Length of the role (without \0 termination). * @param full_member Optional pointer to the member object. If it * is available a copy will be added to the * relation. */ - void add_member(osmium::item_type type, object_id_type ref, const char* role, const osmium::OSMObject* full_member = nullptr) { + void add_member(osmium::item_type type, object_id_type ref, const char* role, const string_size_type role_length, const osmium::OSMObject* full_member = nullptr) { osmium::RelationMember* member = reserve_space_for(); new (member) osmium::RelationMember(ref, type, full_member != nullptr); add_size(sizeof(RelationMember)); - add_role(*member, role); + add_role(*member, role, role_length); if (full_member) { add_item(full_member); } } + /** + * Add a member to the relation. + * + * @param type The type (node, way, or relation). + * @param ref The ID of the member. + * @param role The role of the member (\0 terminated string). + * @param full_member Optional pointer to the member object. If it + * is available a copy will be added to the + * relation. + */ + void add_member(osmium::item_type type, object_id_type ref, const char* role, const osmium::OSMObject* full_member = nullptr) { + add_member(type, ref, role, strlen(role), full_member); + } + /** * Add a member to the relation. * @@ -199,13 +226,7 @@ namespace osmium { * relation. */ void add_member(osmium::item_type type, object_id_type ref, const std::string& role, const osmium::OSMObject* full_member = nullptr) { - osmium::RelationMember* member = reserve_space_for(); - new (member) osmium::RelationMember(ref, type, full_member != nullptr); - add_size(sizeof(RelationMember)); - add_role(*member, role); - if (full_member) { - add_item(full_member); - } + add_member(type, ref, role.data(), role.size(), full_member); } }; // class RelationMemberListBuilder diff --git a/include/osmium/geom/factory.hpp b/include/osmium/geom/factory.hpp index 518cbfbc265..9be050d2563 100644 --- a/include/osmium/geom/factory.hpp +++ b/include/osmium/geom/factory.hpp @@ -54,14 +54,43 @@ namespace osmium { * Exception thrown when an invalid geometry is encountered. An example * would be a linestring with less than two points. */ - struct geometry_error : public std::runtime_error { + class geometry_error : public std::runtime_error { + + std::string m_message; + osmium::object_id_type m_id; + + public: + + geometry_error(const std::string& message, const char* object_type = "", osmium::object_id_type id = 0) : + std::runtime_error(message), + m_message(message), + m_id(id) { + if (m_id != 0) { + m_message += " ("; + m_message += object_type; + m_message += "_id="; + m_message += std::to_string(m_id); + m_message += ")"; + } + } + + void set_id(const char* object_type, osmium::object_id_type id) { + if (m_id == 0 && id != 0) { + m_message += " ("; + m_message += object_type; + m_message += "_id="; + m_message += std::to_string(id); + m_message += ")"; + } + m_id = id; + } - geometry_error(const std::string& what) : - std::runtime_error(what) { + osmium::object_id_type id() const noexcept { + return m_id; } - geometry_error(const char* what) : - std::runtime_error(what) { + virtual const char* what() const noexcept override { + return m_message.c_str(); } }; // struct geometry_error @@ -174,11 +203,21 @@ namespace osmium { } point_type create_point(const osmium::Node& node) { - return create_point(node.location()); + try { + return create_point(node.location()); + } catch (osmium::geometry_error& e) { + e.set_id("node", node.id()); + throw; + } } point_type create_point(const osmium::NodeRef& node_ref) { - return create_point(node_ref.location()); + try { + return create_point(node_ref.location()); + } catch (osmium::geometry_error& e) { + e.set_id("node", node_ref.ref()); + throw; + } } /* LineString */ @@ -240,14 +279,19 @@ namespace osmium { } if (num_points < 2) { - throw osmium::geometry_error("not enough points for linestring"); + throw osmium::geometry_error("need at least two points for linestring"); } return linestring_finish(num_points); } linestring_type create_linestring(const osmium::Way& way, use_nodes un=use_nodes::unique, direction dir=direction::forward) { - return create_linestring(way.nodes(), un, dir); + try { + return create_linestring(way.nodes(), un, dir); + } catch (osmium::geometry_error& e) { + e.set_id("way", way.id()); + throw; + } } /* Polygon */ @@ -283,40 +327,86 @@ namespace osmium { return m_impl.polygon_finish(num_points); } + polygon_type create_polygon(const osmium::WayNodeList& wnl, use_nodes un = use_nodes::unique, direction dir = direction::forward) { + polygon_start(); + size_t num_points = 0; + + if (un == use_nodes::unique) { + osmium::Location last_location; + switch (dir) { + case direction::forward: + num_points = fill_polygon_unique(wnl.cbegin(), wnl.cend()); + break; + case direction::backward: + num_points = fill_polygon_unique(wnl.crbegin(), wnl.crend()); + break; + } + } else { + switch (dir) { + case direction::forward: + num_points = fill_polygon(wnl.cbegin(), wnl.cend()); + break; + case direction::backward: + num_points = fill_polygon(wnl.crbegin(), wnl.crend()); + break; + } + } + + if (num_points < 4) { + throw osmium::geometry_error("need at least four points for polygon"); + } + + return polygon_finish(num_points); + } + + polygon_type create_polygon(const osmium::Way& way, use_nodes un=use_nodes::unique, direction dir=direction::forward) { + try { + return create_polygon(way.nodes(), un, dir); + } catch (osmium::geometry_error& e) { + e.set_id("way", way.id()); + throw; + } + } + /* MultiPolygon */ multipolygon_type create_multipolygon(const osmium::Area& area) { - size_t num_polygons = 0; - size_t num_rings = 0; - m_impl.multipolygon_start(); - - for (auto it = area.cbegin(); it != area.cend(); ++it) { - const osmium::OuterRing& ring = static_cast(*it); - if (it->type() == osmium::item_type::outer_ring) { - if (num_polygons > 0) { - m_impl.multipolygon_polygon_finish(); + try { + size_t num_polygons = 0; + size_t num_rings = 0; + m_impl.multipolygon_start(); + + for (auto it = area.cbegin(); it != area.cend(); ++it) { + const osmium::OuterRing& ring = static_cast(*it); + if (it->type() == osmium::item_type::outer_ring) { + if (num_polygons > 0) { + m_impl.multipolygon_polygon_finish(); + } + m_impl.multipolygon_polygon_start(); + m_impl.multipolygon_outer_ring_start(); + add_points(ring); + m_impl.multipolygon_outer_ring_finish(); + ++num_rings; + ++num_polygons; + } else if (it->type() == osmium::item_type::inner_ring) { + m_impl.multipolygon_inner_ring_start(); + add_points(ring); + m_impl.multipolygon_inner_ring_finish(); + ++num_rings; } - m_impl.multipolygon_polygon_start(); - m_impl.multipolygon_outer_ring_start(); - add_points(ring); - m_impl.multipolygon_outer_ring_finish(); - ++num_rings; - ++num_polygons; - } else if (it->type() == osmium::item_type::inner_ring) { - m_impl.multipolygon_inner_ring_start(); - add_points(ring); - m_impl.multipolygon_inner_ring_finish(); - ++num_rings; } - } - // if there are no rings, this area is invalid - if (num_rings == 0) { - throw osmium::geometry_error("invalid area"); - } + // if there are no rings, this area is invalid + if (num_rings == 0) { + throw osmium::geometry_error("area contains no rings"); + } - m_impl.multipolygon_polygon_finish(); - return m_impl.multipolygon_finish(); + m_impl.multipolygon_polygon_finish(); + return m_impl.multipolygon_finish(); + } catch (osmium::geometry_error& e) { + e.set_id("area", area.id()); + throw; + } } }; // class GeometryFactory diff --git a/include/osmium/geom/geos.hpp b/include/osmium/geom/geos.hpp index 3c73637be05..771b08736fc 100644 --- a/include/osmium/geom/geos.hpp +++ b/include/osmium/geom/geos.hpp @@ -42,6 +42,8 @@ DEALINGS IN THE SOFTWARE. * @attention If you include this file, you'll need to link with `libgeos`. */ +#include +#include #include #include @@ -69,8 +71,8 @@ namespace osmium { struct geos_geometry_error : public geometry_error { - geos_geometry_error() : - geometry_error("geometry creation failed in GEOS library, see nested exception for details") { + geos_geometry_error(const char* message) : + geometry_error(std::string("geometry creation failed in GEOS library: ") + message) { } }; // struct geos_geometry_error @@ -81,8 +83,9 @@ namespace osmium { class GEOSFactoryImpl { - geos::geom::PrecisionModel m_precision_model; - geos::geom::GeometryFactory m_geos_factory; + std::unique_ptr m_precision_model; + std::unique_ptr m_our_geos_factory; + geos::geom::GeometryFactory* m_geos_factory; std::unique_ptr m_coordinate_sequence; std::vector> m_rings; @@ -96,18 +99,25 @@ namespace osmium { typedef std::unique_ptr multipolygon_type; typedef std::unique_ptr ring_type; + explicit GEOSFactoryImpl(geos::geom::GeometryFactory& geos_factory) : + m_precision_model(nullptr), + m_our_geos_factory(nullptr), + m_geos_factory(&geos_factory) { + } + explicit GEOSFactoryImpl(int srid = -1) : - m_precision_model(), - m_geos_factory(&m_precision_model, srid) { + m_precision_model(new geos::geom::PrecisionModel), + m_our_geos_factory(new geos::geom::GeometryFactory(m_precision_model.get(), srid)), + m_geos_factory(m_our_geos_factory.get()) { } /* Point */ point_type make_point(const osmium::geom::Coordinates& xy) const { try { - return point_type(m_geos_factory.createPoint(geos::geom::Coordinate(xy.x, xy.y))); - } catch (geos::util::GEOSException&) { - THROW(osmium::geos_geometry_error()); + return point_type(m_geos_factory->createPoint(geos::geom::Coordinate(xy.x, xy.y))); + } catch (geos::util::GEOSException& e) { + THROW(osmium::geos_geometry_error(e.what())); } } @@ -115,25 +125,25 @@ namespace osmium { void linestring_start() { try { - m_coordinate_sequence.reset(m_geos_factory.getCoordinateSequenceFactory()->create(static_cast(0), 2)); - } catch (geos::util::GEOSException&) { - THROW(osmium::geos_geometry_error()); + m_coordinate_sequence.reset(m_geos_factory->getCoordinateSequenceFactory()->create(static_cast(0), 2)); + } catch (geos::util::GEOSException& e) { + THROW(osmium::geos_geometry_error(e.what())); } } void linestring_add_location(const osmium::geom::Coordinates& xy) { try { m_coordinate_sequence->add(geos::geom::Coordinate(xy.x, xy.y)); - } catch (geos::util::GEOSException&) { - THROW(osmium::geos_geometry_error()); + } catch (geos::util::GEOSException& e) { + THROW(osmium::geos_geometry_error(e.what())); } } linestring_type linestring_finish(size_t /* num_points */) { try { - return linestring_type(m_geos_factory.createLineString(m_coordinate_sequence.release())); - } catch (geos::util::GEOSException&) { - THROW(osmium::geos_geometry_error()); + return linestring_type(m_geos_factory->createLineString(m_coordinate_sequence.release())); + } catch (geos::util::GEOSException& e) { + THROW(osmium::geos_geometry_error(e.what())); } } @@ -154,50 +164,50 @@ namespace osmium { std::transform(std::next(m_rings.begin(), 1), m_rings.end(), std::back_inserter(*inner_rings), [](std::unique_ptr& r) { return r.release(); }); - m_polygons.emplace_back(m_geos_factory.createPolygon(m_rings[0].release(), inner_rings)); + m_polygons.emplace_back(m_geos_factory->createPolygon(m_rings[0].release(), inner_rings)); m_rings.clear(); - } catch (geos::util::GEOSException&) { - THROW(osmium::geos_geometry_error()); + } catch (geos::util::GEOSException& e) { + THROW(osmium::geos_geometry_error(e.what())); } } void multipolygon_outer_ring_start() { try { - m_coordinate_sequence.reset(m_geos_factory.getCoordinateSequenceFactory()->create(static_cast(0), 2)); - } catch (geos::util::GEOSException&) { - THROW(osmium::geos_geometry_error()); + m_coordinate_sequence.reset(m_geos_factory->getCoordinateSequenceFactory()->create(static_cast(0), 2)); + } catch (geos::util::GEOSException& e) { + THROW(osmium::geos_geometry_error(e.what())); } } void multipolygon_outer_ring_finish() { try { - m_rings.emplace_back(m_geos_factory.createLinearRing(m_coordinate_sequence.release())); - } catch (geos::util::GEOSException&) { - THROW(osmium::geos_geometry_error()); + m_rings.emplace_back(m_geos_factory->createLinearRing(m_coordinate_sequence.release())); + } catch (geos::util::GEOSException& e) { + THROW(osmium::geos_geometry_error(e.what())); } } void multipolygon_inner_ring_start() { try { - m_coordinate_sequence.reset(m_geos_factory.getCoordinateSequenceFactory()->create(static_cast(0), 2)); - } catch (geos::util::GEOSException&) { - THROW(osmium::geos_geometry_error()); + m_coordinate_sequence.reset(m_geos_factory->getCoordinateSequenceFactory()->create(static_cast(0), 2)); + } catch (geos::util::GEOSException& e) { + THROW(osmium::geos_geometry_error(e.what())); } } void multipolygon_inner_ring_finish() { try { - m_rings.emplace_back(m_geos_factory.createLinearRing(m_coordinate_sequence.release())); - } catch (geos::util::GEOSException&) { - THROW(osmium::geos_geometry_error()); + m_rings.emplace_back(m_geos_factory->createLinearRing(m_coordinate_sequence.release())); + } catch (geos::util::GEOSException& e) { + THROW(osmium::geos_geometry_error(e.what())); } } void multipolygon_add_location(const osmium::geom::Coordinates& xy) { try { m_coordinate_sequence->add(geos::geom::Coordinate(xy.x, xy.y)); - } catch (geos::util::GEOSException&) { - THROW(osmium::geos_geometry_error()); + } catch (geos::util::GEOSException& e) { + THROW(osmium::geos_geometry_error(e.what())); } } @@ -208,9 +218,9 @@ namespace osmium { return p.release(); }); m_polygons.clear(); - return multipolygon_type(m_geos_factory.createMultiPolygon(polygons)); - } catch (geos::util::GEOSException&) { - THROW(osmium::geos_geometry_error()); + return multipolygon_type(m_geos_factory->createMultiPolygon(polygons)); + } catch (geos::util::GEOSException& e) { + THROW(osmium::geos_geometry_error(e.what())); } } diff --git a/include/osmium/geom/mercator_projection.hpp b/include/osmium/geom/mercator_projection.hpp index a6d1d5742c9..22a0d646b00 100644 --- a/include/osmium/geom/mercator_projection.hpp +++ b/include/osmium/geom/mercator_projection.hpp @@ -47,6 +47,7 @@ namespace osmium { namespace detail { constexpr double earth_radius_for_epsg3857 = 6378137.0; + constexpr double max_coordinate_epsg3857 = 20037508.34; constexpr inline double lon_to_x(double lon) { return earth_radius_for_epsg3857 * deg_to_rad(lon); diff --git a/include/osmium/geom/rapid_geojson.hpp b/include/osmium/geom/rapid_geojson.hpp new file mode 100644 index 00000000000..a3d46870c67 --- /dev/null +++ b/include/osmium/geom/rapid_geojson.hpp @@ -0,0 +1,190 @@ +#ifndef OSMIUM_GEOM_RAPID_GEOJSON_HPP +#define OSMIUM_GEOM_RAPID_GEOJSON_HPP + +/* + +This file is part of Osmium (http://osmcode.org/libosmium). + +Copyright 2013-2015 Jochen Topf and others (see README). + +Boost Software License - Version 1.0 - August 17th, 2003 + +Permission is hereby granted, free of charge, to any person or organization +obtaining a copy of the software and accompanying documentation covered by +this license (the "Software") to use, reproduce, display, distribute, +execute, and transmit the Software, and to prepare derivative works of the +Software, and to permit third-parties to whom the Software is furnished to +do so, all subject to the following: + +The copyright notices in the Software and this entire statement, including +the above license grant, this restriction and the following disclaimer, +must be included in all copies of the Software, in whole or in part, and +all derivative works of the Software, unless such copies or derivative +works are solely in the form of machine-executable object code generated by +a source language processor. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT +SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE +FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, +ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. + +*/ + +#include +#include + +namespace osmium { + + namespace geom { + + namespace detail { + + /** + * A geometry factory implementation that can be used with the + * RapidJSON (https://github.com/miloyip/rapidjson) JSON writer. + */ + template + class RapidGeoJSONFactoryImpl { + + TWriter* m_writer; + + public: + + typedef void point_type; + typedef void linestring_type; + typedef void polygon_type; + typedef void multipolygon_type; + typedef void ring_type; + + RapidGeoJSONFactoryImpl(TWriter& writer) : + m_writer(&writer) { + } + + /* Point */ + + // { "type": "Point", "coordinates": [100.0, 0.0] } + point_type make_point(const osmium::geom::Coordinates& xy) const { + m_writer->String("geometry"); + m_writer->StartObject(); + m_writer->String("type"); + m_writer->String("Point"); + m_writer->String("coordinates"); + m_writer->StartArray(); + m_writer->Double(xy.x); + m_writer->Double(xy.y); + m_writer->EndArray(); + m_writer->EndObject(); + } + + /* LineString */ + + // { "type": "LineString", "coordinates": [ [100.0, 0.0], [101.0, 1.0] ] } + void linestring_start() { + m_writer->String("geometry"); + m_writer->StartObject(); + m_writer->String("type"); + m_writer->String("LineString"); + m_writer->String("coordinates"); + m_writer->StartArray(); + } + + void linestring_add_location(const osmium::geom::Coordinates& xy) { + m_writer->StartArray(); + m_writer->Double(xy.x); + m_writer->Double(xy.y); + m_writer->EndArray(); + } + + linestring_type linestring_finish(size_t /* num_points */) { + m_writer->EndArray(); + m_writer->EndObject(); + } + + /* Polygon */ + + // { "type": "Polygon", "coordinates": [[[100.0, 0.0], [101.0, 1.0]]] } + void polygon_start() { + m_writer->String("geometry"); + m_writer->StartObject(); + m_writer->String("type"); + m_writer->String("Polygon"); + m_writer->String("coordinates"); + m_writer->StartArray(); + m_writer->StartArray(); + } + + void polygon_add_location(const osmium::geom::Coordinates& xy) { + m_writer->StartArray(); + m_writer->Double(xy.x); + m_writer->Double(xy.y); + m_writer->EndArray(); + } + + polygon_type polygon_finish(size_t /* num_points */) { + m_writer->EndArray(); + m_writer->EndArray(); + m_writer->EndObject(); + } + + /* MultiPolygon */ + + void multipolygon_start() { + m_writer->String("geometry"); + m_writer->StartObject(); + m_writer->String("type"); + m_writer->String("MultiPolygon"); + m_writer->String("coordinates"); + m_writer->StartArray(); + } + + void multipolygon_polygon_start() { + m_writer->StartArray(); + } + + void multipolygon_polygon_finish() { + m_writer->EndArray(); + } + + void multipolygon_outer_ring_start() { + m_writer->StartArray(); + } + + void multipolygon_outer_ring_finish() { + m_writer->EndArray(); + } + + void multipolygon_inner_ring_start() { + m_writer->StartArray(); + } + + void multipolygon_inner_ring_finish() { + m_writer->EndArray(); + } + + void multipolygon_add_location(const osmium::geom::Coordinates& xy) { + m_writer->StartArray(); + m_writer->Double(xy.x); + m_writer->Double(xy.y); + m_writer->EndArray(); + } + + multipolygon_type multipolygon_finish() { + m_writer->EndArray(); + m_writer->EndObject(); + } + + }; // class RapidGeoJSONFactoryImpl + + } // namespace detail + + template + using RapidGeoJSONFactory = GeometryFactory, TProjection>; + + } // namespace geom + +} // namespace osmium + +#endif // OSMIUM_GEOM_RAPID_GEOJSON_HPP diff --git a/include/osmium/geom/tile.hpp b/include/osmium/geom/tile.hpp new file mode 100644 index 00000000000..9cd0b282b7b --- /dev/null +++ b/include/osmium/geom/tile.hpp @@ -0,0 +1,101 @@ +#ifndef OSMIUM_GEOM_TILE_HPP +#define OSMIUM_GEOM_TILE_HPP + +/* + +This file is part of Osmium (http://osmcode.org/libosmium). + +Copyright 2013-2015 Jochen Topf and others (see README). + +Boost Software License - Version 1.0 - August 17th, 2003 + +Permission is hereby granted, free of charge, to any person or organization +obtaining a copy of the software and accompanying documentation covered by +this license (the "Software") to use, reproduce, display, distribute, +execute, and transmit the Software, and to prepare derivative works of the +Software, and to permit third-parties to whom the Software is furnished to +do so, all subject to the following: + +The copyright notices in the Software and this entire statement, including +the above license grant, this restriction and the following disclaimer, +must be included in all copies of the Software, in whole or in part, and +all derivative works of the Software, unless such copies or derivative +works are solely in the form of machine-executable object code generated by +a source language processor. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT +SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE +FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, +ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. + +*/ + +#include + +#include + +namespace osmium { + + namespace geom { + + namespace detail { + + template + inline T restrict_to_range(T value, T min, T max) { + if (value < min) return min; + if (value > max) return max; + return value; + } + + } // namespace detail + + /** + * A tile in the usual Mercator projection. + */ + struct Tile { + + uint32_t x; + uint32_t y; + uint32_t z; + + explicit Tile(uint32_t zoom, uint32_t tx, uint32_t ty) noexcept : x(tx), y(ty), z(zoom) { + } + + explicit Tile(uint32_t zoom, const osmium::Location& location) : + z(zoom) { + osmium::geom::Coordinates c = lonlat_to_mercator(location); + const int32_t n = 1LL << zoom; + const double scale = detail::max_coordinate_epsg3857 * 2 / n; + x = detail::restrict_to_range((c.x + detail::max_coordinate_epsg3857) / scale, 0, n-1); + y = detail::restrict_to_range((detail::max_coordinate_epsg3857 - c.y) / scale, 0, n-1); + } + + }; // struct Tile + + inline bool operator==(const Tile& a, const Tile& b) { + return a.z == b.z && a.x == b.x && a.y == b.y; + } + + inline bool operator!=(const Tile& a, const Tile& b) { + return ! (a == b); + } + + /** + * This defines an arbitrary order on tiles for use in std::map etc. + */ + inline bool operator<(const Tile& a, const Tile& b) { + if (a.z < b.z) return true; + if (a.z > b.z) return false; + if (a.x < b.x) return true; + if (a.x > b.x) return false; + return a.y < b.y; + } + + } // namespace geom + +} // namespace osmium + +#endif // OSMIUM_GEOM_TILE_HPP diff --git a/include/osmium/geom/wkb.hpp b/include/osmium/geom/wkb.hpp index 2f32fe33bea..a290c02d06d 100644 --- a/include/osmium/geom/wkb.hpp +++ b/include/osmium/geom/wkb.hpp @@ -37,18 +37,10 @@ DEALINGS IN THE SOFTWARE. #include #include -// Windows is only available for little endian architectures -// http://stackoverflow.com/questions/6449468/can-i-safely-assume-that-windows-installations-will-always-be-little-endian -#if !defined(_WIN32) && !defined(__APPLE__) -# include -#else -# define __LITTLE_ENDIAN 1234 -# define __BYTE_ORDER __LITTLE_ENDIAN -#endif - #include #include #include +#include namespace osmium { diff --git a/include/osmium/index/bool_vector.hpp b/include/osmium/index/bool_vector.hpp new file mode 100644 index 00000000000..94e1f726849 --- /dev/null +++ b/include/osmium/index/bool_vector.hpp @@ -0,0 +1,83 @@ +#ifndef OSMIUM_INDEX_BOOL_VECTOR_HPP +#define OSMIUM_INDEX_BOOL_VECTOR_HPP + +/* + +This file is part of Osmium (http://osmcode.org/libosmium). + +Copyright 2013-2015 Jochen Topf and others (see README). + +Boost Software License - Version 1.0 - August 17th, 2003 + +Permission is hereby granted, free of charge, to any person or organization +obtaining a copy of the software and accompanying documentation covered by +this license (the "Software") to use, reproduce, display, distribute, +execute, and transmit the Software, and to prepare derivative works of the +Software, and to permit third-parties to whom the Software is furnished to +do so, all subject to the following: + +The copyright notices in the Software and this entire statement, including +the above license grant, this restriction and the following disclaimer, +must be included in all copies of the Software, in whole or in part, and +all derivative works of the Software, unless such copies or derivative +works are solely in the form of machine-executable object code generated by +a source language processor. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT +SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE +FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, +ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. + +*/ + +#include +#include + +namespace osmium { + + namespace index { + + /** + * Index storing one bit for each Id. The index automatically scales + * with the Ids stored. Default value is 'false'. Storage uses + * std::vector and needs a minimum of memory if the Ids are + * dense. + */ + template + class BoolVector { + + static_assert(std::is_unsigned::value, "Needs unsigned type"); + + std::vector m_bits; + + public: + + BoolVector() = default; + BoolVector(const BoolVector&) = default; + BoolVector(BoolVector&&) = default; + BoolVector& operator=(const BoolVector&) = default; + BoolVector& operator=(BoolVector&&) = default; + ~BoolVector() = default; + + void set(T id, bool value = true) { + if (m_bits.size() <= id) { + m_bits.resize(id + 1024 * 1024); + } + + m_bits[id] = value; + } + + bool get(T id) const { + return id < m_bits.size() && m_bits[id]; + } + + }; // class BoolVector + + } // namespace index + +} // namespace osmium + +#endif // OSMIUM_INDEX_BOOL_VECTOR_HPP diff --git a/include/osmium/index/detail/create_map_with_fd.hpp b/include/osmium/index/detail/create_map_with_fd.hpp index 29dc1dc675a..5ccbfc80922 100644 --- a/include/osmium/index/detail/create_map_with_fd.hpp +++ b/include/osmium/index/detail/create_map_with_fd.hpp @@ -39,8 +39,6 @@ DEALINGS IN THE SOFTWARE. #include #include #include -#include -#include #include namespace osmium { diff --git a/include/osmium/index/detail/mmap_vector_anon.hpp b/include/osmium/index/detail/mmap_vector_anon.hpp index 0ea4f9db657..fc016261e7c 100644 --- a/include/osmium/index/detail/mmap_vector_anon.hpp +++ b/include/osmium/index/detail/mmap_vector_anon.hpp @@ -35,9 +35,6 @@ DEALINGS IN THE SOFTWARE. #ifdef __linux__ -#include - -#include #include namespace osmium { @@ -45,26 +42,16 @@ namespace osmium { namespace detail { /** - * This class looks and behaves like STL vector, but uses mmap internally. + * This class looks and behaves like STL vector, but uses mmap + * internally. */ template - class mmap_vector_anon : public mmap_vector_base { + class mmap_vector_anon : public mmap_vector_base { public: mmap_vector_anon() : - mmap_vector_base( - -1, - osmium::detail::mmap_vector_size_increment, - 0, - osmium::detail::typed_mmap::map(osmium::detail::mmap_vector_size_increment)) { - } - - void reserve(size_t new_capacity) { - if (new_capacity > this->capacity()) { - this->data(osmium::detail::typed_mmap::remap(this->data(), this->capacity(), new_capacity)); - this->m_capacity = new_capacity; - } + mmap_vector_base() { } }; // class mmap_vector_anon diff --git a/include/osmium/index/detail/mmap_vector_base.hpp b/include/osmium/index/detail/mmap_vector_base.hpp index 3aff26d4103..9b647681203 100644 --- a/include/osmium/index/detail/mmap_vector_base.hpp +++ b/include/osmium/index/detail/mmap_vector_base.hpp @@ -34,11 +34,10 @@ DEALINGS IN THE SOFTWARE. */ #include -#include +#include // IWYU pragma: keep #include -#include -#include +#include namespace osmium { @@ -48,40 +47,29 @@ namespace osmium { /** * This is a base class for implementing classes that look like - * STL vector but use mmap internally. This class can not be used - * on it's own. Use the derived classes mmap_vector_anon or - * mmap_vector_file. + * STL vector but use mmap internally. Do not use this class itself, + * use the derived classes mmap_vector_anon or mmap_vector_file. */ - template class TDerived> + template class mmap_vector_base { protected: - int m_fd; - size_t m_capacity; size_t m_size; - T* m_data; + osmium::util::TypedMemoryMapping m_mapping; - explicit mmap_vector_base(int fd, size_t capacity, size_t size, T* data) noexcept : - m_fd(fd), - m_capacity(capacity), - m_size(size), - m_data(data) { - } + public: - explicit mmap_vector_base(int fd, size_t capacity, size_t size) : - m_fd(fd), - m_capacity(capacity), + explicit mmap_vector_base(int fd, size_t capacity, size_t size = 0) : m_size(size), - m_data(osmium::detail::typed_mmap::grow_and_map(capacity, m_fd)) { + m_mapping(capacity, osmium::util::MemoryMapping::mapping_mode::write_shared, fd) { } - void data(T* data) { - m_data = data; + explicit mmap_vector_base(size_t capacity = mmap_vector_size_increment) : + m_size(0), + m_mapping(capacity) { } - public: - typedef T value_type; typedef T& reference; typedef const T& const_reference; @@ -90,12 +78,14 @@ namespace osmium { typedef T* iterator; typedef const T* const_iterator; - ~mmap_vector_base() { - osmium::detail::typed_mmap::unmap(m_data, m_capacity); + ~mmap_vector_base() = default; + + void close() { + m_mapping.unmap(); } size_t capacity() const noexcept { - return m_capacity; + return m_mapping.size(); } size_t size() const noexcept { @@ -106,23 +96,23 @@ namespace osmium { return m_size == 0; } - const T* data() const noexcept { - return m_data; + const T* data() const { + return m_mapping.begin(); } - T* data() noexcept { - return m_data; + T* data() { + return m_mapping.begin(); } T& operator[](size_t n) { - return m_data[n]; + return data()[n]; } T at(size_t n) const { if (n >= m_size) { throw std::out_of_range("out of range"); } - return m_data[n]; + return data()[n]; } void clear() noexcept { @@ -134,16 +124,22 @@ namespace osmium { } void push_back(const T& value) { - if (m_size >= m_capacity) { + if (m_size >= capacity()) { resize(m_size+1); } - m_data[m_size] = value; + data()[m_size] = value; ++m_size; } + void reserve(size_t new_capacity) { + if (new_capacity > capacity()) { + m_mapping.resize(new_capacity); + } + } + void resize(size_t new_size) { if (new_size > capacity()) { - static_cast*>(this)->reserve(new_size + osmium::detail::mmap_vector_size_increment); + reserve(new_size + osmium::detail::mmap_vector_size_increment); } if (new_size > size()) { new (data() + size()) T[new_size - size()]; @@ -152,27 +148,27 @@ namespace osmium { } iterator begin() noexcept { - return m_data; + return data(); } iterator end() noexcept { - return m_data + m_size; + return data() + m_size; } const_iterator begin() const noexcept { - return m_data; + return data(); } const_iterator end() const noexcept { - return m_data + m_size; + return data() + m_size; } - const_iterator cbegin() noexcept { - return m_data; + const_iterator cbegin() const noexcept { + return data(); } - const_iterator cend() noexcept { - return m_data + m_size; + const_iterator cend() const noexcept { + return data() + m_size; } }; // class mmap_vector_base diff --git a/include/osmium/index/detail/mmap_vector_file.hpp b/include/osmium/index/detail/mmap_vector_file.hpp index 55077d18d45..1dadbcb45b3 100644 --- a/include/osmium/index/detail/mmap_vector_file.hpp +++ b/include/osmium/index/detail/mmap_vector_file.hpp @@ -33,11 +33,9 @@ DEALINGS IN THE SOFTWARE. */ -#include - -#include #include #include +#include namespace osmium { @@ -48,32 +46,19 @@ namespace osmium { * internally. */ template - class mmap_vector_file : public mmap_vector_base { + class mmap_vector_file : public mmap_vector_base { public: - explicit mmap_vector_file() : - mmap_vector_base( + explicit mmap_vector_file() : mmap_vector_base( osmium::detail::create_tmp_file(), - osmium::detail::mmap_vector_size_increment, - 0) { + osmium::detail::mmap_vector_size_increment) { } - explicit mmap_vector_file(int fd) : - mmap_vector_base( + explicit mmap_vector_file(int fd) : mmap_vector_base( fd, - osmium::detail::typed_mmap::file_size(fd) == 0 ? - osmium::detail::mmap_vector_size_increment : - osmium::detail::typed_mmap::file_size(fd), - osmium::detail::typed_mmap::file_size(fd)) { - } - - void reserve(size_t new_capacity) { - if (new_capacity > this->capacity()) { - typed_mmap::unmap(this->data(), this->capacity()); - this->data(typed_mmap::grow_and_map(new_capacity, this->m_fd)); - this->m_capacity = new_capacity; - } + osmium::util::file_size(fd) / sizeof(T), + osmium::util::file_size(fd) / sizeof(T)) { } }; // class mmap_vector_file diff --git a/include/osmium/index/detail/typed_mmap.hpp b/include/osmium/index/detail/typed_mmap.hpp deleted file mode 100644 index 77b065e8e95..00000000000 --- a/include/osmium/index/detail/typed_mmap.hpp +++ /dev/null @@ -1,229 +0,0 @@ -#ifndef OSMIUM_INDEX_DETAIL_TYPED_MMAP_HPP -#define OSMIUM_INDEX_DETAIL_TYPED_MMAP_HPP - -/* - -This file is part of Osmium (http://osmcode.org/libosmium). - -Copyright 2013-2015 Jochen Topf and others (see README). - -Boost Software License - Version 1.0 - August 17th, 2003 - -Permission is hereby granted, free of charge, to any person or organization -obtaining a copy of the software and accompanying documentation covered by -this license (the "Software") to use, reproduce, display, distribute, -execute, and transmit the Software, and to prepare derivative works of the -Software, and to permit third-parties to whom the Software is furnished to -do so, all subject to the following: - -The copyright notices in the Software and this entire statement, including -the above license grant, this restriction and the following disclaimer, -must be included in all copies of the Software, in whole or in part, and -all derivative works of the Software, unless such copies or derivative -works are solely in the form of machine-executable object code generated by -a source language processor. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT -SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE -FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, -ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER -DEALINGS IN THE SOFTWARE. - -*/ - -#include -#include -#include -#include - -#include - -#ifndef _WIN32 -# include -#else -# include -#endif - -#ifndef _MSC_VER -# include -#else -# define ftruncate _chsize -#endif - -// for bsd systems -#ifndef MAP_ANONYMOUS -# define MAP_ANONYMOUS MAP_ANON -#endif - -#include - -namespace osmium { - - /** - * @brief Namespace for Osmium internal use - */ - namespace detail { - - /** - * This is a helper class for working with memory mapped files and - * anonymous shared memory. It wraps the necessary system calls - * adding: - * - error checking: all functions throw exceptions where needed - * - internal casts and size calculations allow use with user defined - * type T instead of void* - * - * This class only contains static functions. It should never be - * instantiated. - * - * @tparam T Type of objects we want to store. - */ - template - class typed_mmap { - - public: - - /** - * Create anonymous private memory mapping with enough space for size - * objects of type T. - * - * Note that no constructor is called for any of the objects in this memory! - * - * @param size Number of objects of type T that should fit into this memory - * @returns Pointer to mapped memory - * @throws std::system_error If mmap(2) failed - */ - static T* map(size_t size) { - void* addr = ::mmap(nullptr, sizeof(T) * size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wold-style-cast" - if (addr == MAP_FAILED) { - throw std::system_error(errno, std::system_category(), "mmap failed"); - } -#pragma GCC diagnostic pop - return reinterpret_cast(addr); - } - - /** - * Create shared memory mapping of a file with enough space for size - * objects of type T. The file must already have at least the - * required size. - * - * Note that no constructor is called for any of the objects in this memory! - * - * @param size Number of objects of type T that should fit into this memory - * @param fd File descriptor - * @param write True if data should be writable - * @returns Pointer to mapped memory - * @throws std::system_error If mmap(2) failed - */ - static T* map(size_t size, int fd, bool write = false) { - int prot = PROT_READ; - if (write) { - prot |= PROT_WRITE; - } - void* addr = ::mmap(nullptr, sizeof(T) * size, prot, MAP_SHARED, fd, 0); -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wold-style-cast" - if (addr == MAP_FAILED) { - throw std::system_error(errno, std::system_category(), "mmap failed"); - } -#pragma GCC diagnostic pop - return reinterpret_cast(addr); - } - -// mremap(2) is only available on linux systems -#ifdef __linux__ - /** - * Grow memory mapping created with map(). - * - * Note that no constructor is called for any of the objects in this memory! - * - * @param data Pointer to current mapping (as returned by typed_mmap()) - * @param old_size Number of objects currently stored in this memory - * @param new_size Number of objects we want to have space for - * @throws std::system_error If mremap(2) call failed - */ - static T* remap(T* data, size_t old_size, size_t new_size) { - void* addr = ::mremap(reinterpret_cast(data), sizeof(T) * old_size, sizeof(T) * new_size, MREMAP_MAYMOVE); -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wold-style-cast" - if (addr == MAP_FAILED) { - throw std::system_error(errno, std::system_category(), "mremap failed"); - } -#pragma GCC diagnostic pop - return reinterpret_cast(addr); - } -#endif - - /** - * Release memory from map() call. - * - * Note that no destructor is called for the objects in this memory! - * - * @param data Pointer to the data - * @param size Number of objects of type T stored - * @throws std::system_error If munmap(2) call failed - */ - static void unmap(T* data, size_t size) { - if (::munmap(reinterpret_cast(data), sizeof(T) * size) != 0) { - throw std::system_error(errno, std::system_category(), "munmap failed"); - } - } - - /** - * Get number of objects of type T that would fit into a file. - * - * @param fd File descriptor - * @returns Number of objects of type T in this file - * @throws std::system_error If fstat(2) call failed - * @throws std::length_error If size of the file isn't a multiple of sizeof(T) - */ - static size_t file_size(int fd) { - struct stat s; - if (fstat(fd, &s) < 0) { - throw std::system_error(errno, std::system_category(), "fstat failed"); - } - if (static_cast(s.st_size) % sizeof(T) != 0) { - throw std::length_error("file size has to be multiple of object size"); - } - return static_cast(s.st_size) / sizeof(T); - } - - /** - * Grow file so there is enough space for at least new_size objects - * of type T. If the file is large enough already, nothing is done. - * The file is never shrunk. - * - * @param new_size Number of objects of type T that should fit into this file - * @param fd File descriptor - * @throws std::system_error If ftruncate(2) call failed - */ - static void grow_file(size_t new_size, int fd) { - if (file_size(fd) < new_size) { - if (::ftruncate(fd, static_cast_with_assert(sizeof(T) * new_size)) < 0) { - throw std::system_error(errno, std::system_category(), "ftruncate failed"); - } - } - } - - /** - * Grow file to given size (if it is smaller) and mmap it. - * - * @param size Number of objects of type T that should fit into this file - * @param fd File descriptor - * @throws Errors thrown by grow_file() or map() - */ - static T* grow_and_map(size_t size, int fd) { - grow_file(size, fd); - return map(size, fd, true); - } - - }; // class typed_mmap - - } // namespace detail - -} // namespace osmium - -#endif // OSMIUM_INDEX_DETAIL_TYPED_MMAP_HPP diff --git a/include/osmium/index/detail/vector_map.hpp b/include/osmium/index/detail/vector_map.hpp index 73c5a37a579..9c1cf4ed842 100644 --- a/include/osmium/index/detail/vector_map.hpp +++ b/include/osmium/index/detail/vector_map.hpp @@ -68,7 +68,7 @@ namespace osmium { m_vector(fd) { } - ~VectorBasedDenseMap() {} + ~VectorBasedDenseMap() = default; void reserve(const size_t size) override final { m_vector.reserve(size); @@ -97,6 +97,10 @@ namespace osmium { return m_vector.size(); } + size_t byte_size() const { + return m_vector.size() * sizeof(element_type); + } + size_t used_memory() const override final { return sizeof(TValue) * size(); } @@ -106,6 +110,10 @@ namespace osmium { m_vector.shrink_to_fit(); } + void dump_as_array(const int fd) override final { + osmium::io::detail::reliable_write(fd, reinterpret_cast(m_vector.data()), byte_size()); + } + iterator begin() { return m_vector.begin(); } diff --git a/include/osmium/index/detail/vector_multimap.hpp b/include/osmium/index/detail/vector_multimap.hpp index c2b2e1f2915..1d875fcba96 100644 --- a/include/osmium/index/detail/vector_multimap.hpp +++ b/include/osmium/index/detail/vector_multimap.hpp @@ -67,6 +67,16 @@ namespace osmium { public: + VectorBasedSparseMultimap() : + m_vector() { + } + + explicit VectorBasedSparseMultimap(int fd) : + m_vector(fd) { + } + + ~VectorBasedSparseMultimap() = default; + void set(const TId id, const TValue value) override final { m_vector.push_back(element_type(id, value)); } @@ -141,6 +151,30 @@ namespace osmium { osmium::io::detail::reliable_write(fd, reinterpret_cast(m_vector.data()), byte_size()); } + iterator begin() { + return m_vector.begin(); + } + + iterator end() { + return m_vector.end(); + } + + const_iterator cbegin() const { + return m_vector.cbegin(); + } + + const_iterator cend() const { + return m_vector.cend(); + } + + const_iterator begin() const { + return m_vector.cbegin(); + } + + const_iterator end() const { + return m_vector.cend(); + } + }; // class VectorBasedSparseMultimap } // namespace multimap diff --git a/include/osmium/index/index.hpp b/include/osmium/index/index.hpp index b73b319e87b..f415192d2db 100644 --- a/include/osmium/index/index.hpp +++ b/include/osmium/index/index.hpp @@ -67,7 +67,7 @@ namespace osmium { template OSMIUM_NORETURN void not_found_error(TKey key) { std::stringstream s; - s << "id " << key << " no found"; + s << "id " << key << " not found"; throw not_found(s.str()); } diff --git a/include/osmium/index/map.hpp b/include/osmium/index/map.hpp index 7b44b8e3377..61af672cdd7 100644 --- a/include/osmium/index/map.hpp +++ b/include/osmium/index/map.hpp @@ -148,7 +148,11 @@ namespace osmium { } virtual void dump_as_list(const int /*fd*/) { - std::runtime_error("can't dump as list"); + throw std::runtime_error("can't dump as list"); + } + + virtual void dump_as_array(const int /*fd*/) { + throw std::runtime_error("can't dump as array"); } }; // class Map @@ -195,6 +199,10 @@ namespace osmium { return m_callbacks.emplace(map_type_name, func).second; } + bool has_map_type(const std::string& map_type_name) const { + return m_callbacks.count(map_type_name); + } + std::vector map_types() const { std::vector result; @@ -242,9 +250,13 @@ namespace osmium { }); } +#define OSMIUM_CONCATENATE_DETAIL_(x, y) x##y +#define OSMIUM_CONCATENATE_(x, y) OSMIUM_CONCATENATE_DETAIL_(x, y) +#define OSMIUM_MAKE_UNIQUE_(x) OSMIUM_CONCATENATE_(x, __COUNTER__) + #define REGISTER_MAP(id, value, klass, name) \ namespace { \ - const bool registered_index_map_##name = osmium::index::register_map(#name); \ + const bool OSMIUM_MAKE_UNIQUE_(registered_index_map_##name) = osmium::index::register_map(#name); \ } } // namespace index diff --git a/include/osmium/index/map/dense_mmap_array.hpp b/include/osmium/index/map/dense_mmap_array.hpp index fc60a1ef21d..a912aebd1f0 100644 --- a/include/osmium/index/map/dense_mmap_array.hpp +++ b/include/osmium/index/map/dense_mmap_array.hpp @@ -35,7 +35,7 @@ DEALINGS IN THE SOFTWARE. #ifdef __linux__ -#include +#include // IWYU pragma: keep #include #define OSMIUM_HAS_INDEX_MAP_DENSE_MMAP_ARRAY diff --git a/include/osmium/index/map/sparse_mem_map.hpp b/include/osmium/index/map/sparse_mem_map.hpp index d053155f057..2b9048b3be2 100644 --- a/include/osmium/index/map/sparse_mem_map.hpp +++ b/include/osmium/index/map/sparse_mem_map.hpp @@ -33,7 +33,7 @@ DEALINGS IN THE SOFTWARE. */ -#include +#include // IWYU pragma: keep (for std::copy) #include #include #include diff --git a/include/osmium/io/any_input.hpp b/include/osmium/io/any_input.hpp index 633fab31392..d16d069a571 100644 --- a/include/osmium/io/any_input.hpp +++ b/include/osmium/io/any_input.hpp @@ -39,8 +39,8 @@ DEALINGS IN THE SOFTWARE. * Include this file if you want to read all kinds of OSM files. * * @attention If you include this file, you'll need to link with - * `libprotobuf-lite`, `libosmpbf`, `ws2_32` (Windows only), - * `libexpat`, `libz`, `libbz2`, and enable multithreading. + * `ws2_32` (Windows only), `libexpat`, `libz`, `libbz2`, + * and enable multithreading. */ #include // IWYU pragma: export diff --git a/include/osmium/io/any_output.hpp b/include/osmium/io/any_output.hpp index 63de3ff2ebf..990a27bacfd 100644 --- a/include/osmium/io/any_output.hpp +++ b/include/osmium/io/any_output.hpp @@ -39,12 +39,13 @@ DEALINGS IN THE SOFTWARE. * Include this file if you want to write all kinds of OSM files. * * @attention If you include this file, you'll need to link with - * `libprotobuf-lite`, `libosmpbf`, `ws2_32` (Windows only), - * `libz`, `libbz2`, and enable multithreading. + * `ws2_32` (Windows only), `libz`, `libbz2`, and enable + * multithreading. */ #include // IWYU pragma: export +#include // IWYU pragma: export #include // IWYU pragma: export #include // IWYU pragma: export #include // IWYU pragma: export diff --git a/include/osmium/io/bzip2_compression.hpp b/include/osmium/io/bzip2_compression.hpp index 7e86c154358..e961a87ab9a 100644 --- a/include/osmium/io/bzip2_compression.hpp +++ b/include/osmium/io/bzip2_compression.hpp @@ -274,11 +274,16 @@ namespace osmium { namespace { +// we want the register_compression() function to run, setting the variable +// is only a side-effect, it will never be used +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wunused-variable" const bool registered_bzip2_compression = osmium::io::CompressionFactory::instance().register_compression(osmium::io::file_compression::bzip2, [](int fd) { return new osmium::io::Bzip2Compressor(fd); }, [](int fd) { return new osmium::io::Bzip2Decompressor(fd); }, [](const char* buffer, size_t size) { return new osmium::io::Bzip2BufferDecompressor(buffer, size); } ); +#pragma GCC diagnostic pop } // anonymous namespace diff --git a/include/osmium/io/compression.hpp b/include/osmium/io/compression.hpp index c1f8de2e71d..2529761810d 100644 --- a/include/osmium/io/compression.hpp +++ b/include/osmium/io/compression.hpp @@ -266,11 +266,16 @@ namespace osmium { namespace { +// we want the register_compression() function to run, setting the variable +// is only a side-effect, it will never be used +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wunused-variable" const bool registered_no_compression = osmium::io::CompressionFactory::instance().register_compression(osmium::io::file_compression::none, [](int fd) { return new osmium::io::NoCompressor(fd); }, [](int fd) { return new osmium::io::NoDecompressor(fd); }, [](const char* buffer, size_t size) { return new osmium::io::NoDecompressor(buffer, size); } ); +#pragma GCC diagnostic pop } // anonymous namespace diff --git a/include/osmium/io/debug_output.hpp b/include/osmium/io/debug_output.hpp new file mode 100644 index 00000000000..2836f79879d --- /dev/null +++ b/include/osmium/io/debug_output.hpp @@ -0,0 +1,39 @@ +#ifndef OSMIUM_IO_DEBUG_OUTPUT_HPP +#define OSMIUM_IO_DEBUG_OUTPUT_HPP + +/* + +This file is part of Osmium (http://osmcode.org/libosmium). + +Copyright 2013-2015 Jochen Topf and others (see README). + +Boost Software License - Version 1.0 - August 17th, 2003 + +Permission is hereby granted, free of charge, to any person or organization +obtaining a copy of the software and accompanying documentation covered by +this license (the "Software") to use, reproduce, display, distribute, +execute, and transmit the Software, and to prepare derivative works of the +Software, and to permit third-parties to whom the Software is furnished to +do so, all subject to the following: + +The copyright notices in the Software and this entire statement, including +the above license grant, this restriction and the following disclaimer, +must be included in all copies of the Software, in whole or in part, and +all derivative works of the Software, unless such copies or derivative +works are solely in the form of machine-executable object code generated by +a source language processor. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT +SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE +FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, +ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. + +*/ + +#include // IWYU pragma: export +#include // IWYU pragma: export + +#endif // OSMIUM_IO_DEBUG_OUTPUT_HPP diff --git a/include/osmium/io/detail/debug_output_format.hpp b/include/osmium/io/detail/debug_output_format.hpp new file mode 100644 index 00000000000..efecc583313 --- /dev/null +++ b/include/osmium/io/detail/debug_output_format.hpp @@ -0,0 +1,482 @@ +#ifndef OSMIUM_IO_DETAIL_DEBUG_OUTPUT_FORMAT_HPP +#define OSMIUM_IO_DETAIL_DEBUG_OUTPUT_FORMAT_HPP + +/* + +This file is part of Osmium (http://osmcode.org/libosmium). + +Copyright 2013-2015 Jochen Topf and others (see README). + +Boost Software License - Version 1.0 - August 17th, 2003 + +Permission is hereby granted, free of charge, to any person or organization +obtaining a copy of the software and accompanying documentation covered by +this license (the "Software") to use, reproduce, display, distribute, +execute, and transmit the Software, and to prepare derivative works of the +Software, and to permit third-parties to whom the Software is furnished to +do so, all subject to the following: + +The copyright notices in the Software and this entire statement, including +the above license grant, this restriction and the following disclaimer, +must be included in all copies of the Software, in whole or in part, and +all derivative works of the Software, unless such copies or derivative +works are solely in the form of machine-executable object code generated by +a source language processor. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT +SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE +FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, +ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. + +*/ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace osmium { + + namespace io { + + class File; + + namespace detail { + + constexpr const char* color_bold = "\x1b[1m"; + constexpr const char* color_black = "\x1b[30m"; + constexpr const char* color_gray = "\x1b[30;1m"; + constexpr const char* color_red = "\x1b[31m"; + constexpr const char* color_green = "\x1b[32m"; + constexpr const char* color_yellow = "\x1b[33m"; + constexpr const char* color_blue = "\x1b[34m"; + constexpr const char* color_magenta = "\x1b[35m"; + constexpr const char* color_cyan = "\x1b[36m"; + constexpr const char* color_white = "\x1b[37m"; + constexpr const char* color_reset = "\x1b[0m"; + + /** + * Writes out one buffer with OSM data in Debug format. + */ + class DebugOutputBlock : public osmium::handler::Handler { + + static constexpr size_t tmp_buffer_size = 50; + + std::shared_ptr m_input_buffer; + + std::shared_ptr m_out; + + char m_tmp_buffer[tmp_buffer_size+1]; + + bool m_add_metadata; + bool m_use_color; + + template + void output_formatted(const char* format, TArgs&&... args) { +#ifndef NDEBUG + int len = +#endif +#ifndef _MSC_VER + snprintf(m_tmp_buffer, tmp_buffer_size, format, std::forward(args)...); +#else + _snprintf(m_tmp_buffer, tmp_buffer_size, format, std::forward(args)...); +#endif + assert(len > 0 && static_cast(len) < tmp_buffer_size); + *m_out += m_tmp_buffer; + } + + void append_encoded_string(const char* data) { + const char* end = data + std::strlen(data); + + while (data != end) { + const char* last = data; + uint32_t c = utf8::next(data, end); + + // This is a list of Unicode code points that we let + // through instead of escaping them. It is incomplete + // and can be extended later. + // Generally we don't want to let through any + // non-printing characters. + if ((0x0020 <= c && c <= 0x0021) || + (0x0023 <= c && c <= 0x003b) || + (0x003d == c) || + (0x003f <= c && c <= 0x007e) || + (0x00a1 <= c && c <= 0x00ac) || + (0x00ae <= c && c <= 0x05ff)) { + m_out->append(last, data); + } else { + write_color(color_red); + output_formatted("", c); + write_color(color_blue); + } + } + } + + void write_color(const char* color) { + if (m_use_color) { + *m_out += color; + } + } + + void write_string(const char* string) { + *m_out += '"'; + write_color(color_blue); + append_encoded_string(string); + write_color(color_reset); + *m_out += '"'; + } + + void write_object_type(const char* object_type, bool visible = true) { + if (visible) { + write_color(color_bold); + } else { + write_color(color_white); + } + *m_out += object_type; + write_color(color_reset); + *m_out += ' '; + } + + void write_fieldname(const char* name) { + *m_out += " "; + write_color(color_cyan); + *m_out += name; + write_color(color_reset); + *m_out += ": "; + } + + void write_error(const char* msg) { + write_color(color_red); + *m_out += msg; + write_color(color_reset); + } + + void write_meta(const osmium::OSMObject& object) { + output_formatted("%" PRId64 "\n", object.id()); + if (m_add_metadata) { + write_fieldname("version"); + output_formatted(" %d", object.version()); + if (object.visible()) { + *m_out += " visible\n"; + } else { + write_error(" deleted\n"); + } + write_fieldname("changeset"); + output_formatted("%d\n", object.changeset()); + write_fieldname("timestamp"); + *m_out += object.timestamp().to_iso(); + output_formatted(" (%d)\n", object.timestamp()); + write_fieldname("user"); + output_formatted(" %d ", object.uid()); + write_string(object.user()); + *m_out += '\n'; + } + } + + void write_tags(const osmium::TagList& tags, const char* padding="") { + if (!tags.empty()) { + write_fieldname("tags"); + *m_out += padding; + output_formatted(" %d\n", tags.size()); + + osmium::max_op max; + for (const auto& tag : tags) { + max.update(std::strlen(tag.key())); + } + for (const auto& tag : tags) { + *m_out += " "; + write_string(tag.key()); + int spacing = max() - std::strlen(tag.key()); + while (spacing--) { + *m_out += " "; + } + *m_out += " = "; + write_string(tag.value()); + *m_out += '\n'; + } + } + } + + void write_location(const osmium::Location& location) { + write_fieldname("lon/lat"); + output_formatted(" %.7f,%.7f", location.lon_without_check(), location.lat_without_check()); + if (!location.valid()) { + write_error(" INVALID LOCATION!"); + } + *m_out += '\n'; + } + + void write_box(const osmium::Box& box) { + write_fieldname("box l/b/r/t"); + if (!box) { + write_error("BOX NOT SET!\n"); + return; + } + const auto& bl = box.bottom_left(); + const auto& tr = box.top_right(); + output_formatted("%.7f,%.7f %.7f,%.7f", bl.lon_without_check(), bl.lat_without_check(), tr.lon_without_check(), tr.lat_without_check()); + if (!box.valid()) { + write_error(" INVALID BOX!"); + } + *m_out += '\n'; + } + + public: + + explicit DebugOutputBlock(osmium::memory::Buffer&& buffer, bool add_metadata, bool use_color) : + m_input_buffer(std::make_shared(std::move(buffer))), + m_out(std::make_shared()), + m_tmp_buffer(), + m_add_metadata(add_metadata), + m_use_color(use_color) { + } + + DebugOutputBlock(const DebugOutputBlock&) = default; + DebugOutputBlock& operator=(const DebugOutputBlock&) = default; + + DebugOutputBlock(DebugOutputBlock&&) = default; + DebugOutputBlock& operator=(DebugOutputBlock&&) = default; + + ~DebugOutputBlock() = default; + + std::string operator()() { + osmium::apply(m_input_buffer->cbegin(), m_input_buffer->cend(), *this); + + std::string out; + std::swap(out, *m_out); + return out; + } + + void node(const osmium::Node& node) { + write_object_type("node", node.visible()); + write_meta(node); + + if (node.visible()) { + write_location(node.location()); + } + + write_tags(node.tags()); + + *m_out += '\n'; + } + + void way(const osmium::Way& way) { + write_object_type("way", way.visible()); + write_meta(way); + write_tags(way.tags()); + + write_fieldname("nodes"); + + output_formatted(" %d", way.nodes().size()); + if (way.nodes().size() < 2) { + write_error(" LESS THAN 2 NODES!\n"); + } else if (way.nodes().size() > 2000) { + write_error(" MORE THAN 2000 NODES!\n"); + } else if (way.nodes().is_closed()) { + *m_out += " (closed)\n"; + } else { + *m_out += " (open)\n"; + } + + int width = int(log10(way.nodes().size())) + 1; + int n = 0; + for (const auto& node_ref : way.nodes()) { + output_formatted(" %0*d: %10" PRId64, width, n++, node_ref.ref()); + if (node_ref.location().valid()) { + output_formatted(" (%.7f,%.7f)", node_ref.location().lon_without_check(), node_ref.location().lat_without_check()); + } + *m_out += '\n'; + } + + *m_out += '\n'; + } + + void relation(const osmium::Relation& relation) { + static const char* short_typename[] = { "node", "way ", "rel " }; + write_object_type("relation", relation.visible()); + write_meta(relation); + write_tags(relation.tags()); + + write_fieldname("members"); + output_formatted(" %d\n", relation.members().size()); + + int width = int(log10(relation.members().size())) + 1; + int n = 0; + for (const auto& member : relation.members()) { + output_formatted(" %0*d: ", width, n++); + *m_out += short_typename[item_type_to_nwr_index(member.type())]; + output_formatted(" %10" PRId64 " ", member.ref()); + write_string(member.role()); + *m_out += '\n'; + } + + *m_out += '\n'; + } + + void changeset(const osmium::Changeset& changeset) { + write_object_type("changeset"); + output_formatted("%d\n", changeset.id()); + write_fieldname("num changes"); + output_formatted("%d", changeset.num_changes()); + if (changeset.num_changes() == 0) { + write_error(" NO CHANGES!"); + } + *m_out += '\n'; + write_fieldname("created at"); + *m_out += ' '; + *m_out += changeset.created_at().to_iso(); + output_formatted(" (%d)\n", changeset.created_at()); + write_fieldname("closed at"); + *m_out += " "; + if (changeset.closed()) { + *m_out += changeset.closed_at().to_iso(); + output_formatted(" (%d)\n", changeset.closed_at()); + } else { + write_error("OPEN!\n"); + } + write_fieldname("user"); + output_formatted(" %d ", changeset.uid()); + write_string(changeset.user()); + *m_out += '\n'; + + write_box(changeset.bounds()); + write_tags(changeset.tags(), " "); + + *m_out += '\n'; + } + + }; // DebugOutputBlock + + class DebugOutputFormat : public osmium::io::detail::OutputFormat { + + bool m_add_metadata; + bool m_use_color; + + public: + + DebugOutputFormat(const osmium::io::File& file, data_queue_type& output_queue) : + OutputFormat(file, output_queue), + m_add_metadata(file.get("add_metadata") != "false"), + m_use_color(file.get("color") == "true") { + } + + DebugOutputFormat(const DebugOutputFormat&) = delete; + DebugOutputFormat& operator=(const DebugOutputFormat&) = delete; + + void write_buffer(osmium::memory::Buffer&& buffer) override final { + m_output_queue.push(osmium::thread::Pool::instance().submit(DebugOutputBlock{std::move(buffer), m_add_metadata, m_use_color})); + } + + void write_fieldname(std::string& out, const char* name) { + out += " "; + if (m_use_color) { + out += color_cyan; + } + out += name; + if (m_use_color) { + out += color_reset; + } + out += ": "; + } + + void write_header(const osmium::io::Header& header) override final { + std::string out; + + if (m_use_color) { + out += color_bold; + } + out += "header\n"; + if (m_use_color) { + out += color_reset; + } + + write_fieldname(out, "multiple object versions"); + out += header.has_multiple_object_versions() ? "yes" : "no"; + out += '\n'; + write_fieldname(out, "bounding boxes"); + out += '\n'; + for (const auto& box : header.boxes()) { + out += " "; + box.bottom_left().as_string(std::back_inserter(out), ','); + out += " "; + box.top_right().as_string(std::back_inserter(out), ','); + out += '\n'; + } + write_fieldname(out, "options"); + out += '\n'; + for (const auto& opt : header) { + out += " "; + out += opt.first; + out += " = "; + out += opt.second; + out += '\n'; + } + out += "\n=============================================\n\n"; + + std::promise promise; + m_output_queue.push(promise.get_future()); + promise.set_value(std::move(out)); + } + + void close() override final { + std::string out; + std::promise promise; + m_output_queue.push(promise.get_future()); + promise.set_value(out); + } + + }; // class DebugOutputFormat + + namespace { + +// we want the register_output_format() function to run, setting the variable +// is only a side-effect, it will never be used +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wunused-variable" + const bool registered_debug_output = osmium::io::detail::OutputFormatFactory::instance().register_output_format(osmium::io::file_format::debug, + [](const osmium::io::File& file, data_queue_type& output_queue) { + return new osmium::io::detail::DebugOutputFormat(file, output_queue); + }); +#pragma GCC diagnostic pop + + } // anonymous namespace + + } // namespace detail + + } // namespace io + +} // namespace osmium + +#endif // OSMIUM_IO_DETAIL_DEBUG_OUTPUT_FORMAT_HPP diff --git a/include/osmium/io/detail/opl_output_format.hpp b/include/osmium/io/detail/opl_output_format.hpp index cf92e1382d8..a3103d9bf67 100644 --- a/include/osmium/io/detail/opl_output_format.hpp +++ b/include/osmium/io/detail/opl_output_format.hpp @@ -46,23 +46,7 @@ DEALINGS IN THE SOFTWARE. #include #include -#include - -#ifdef __clang__ -# pragma clang diagnostic push -# pragma clang diagnostic ignored "-Wmissing-noreturn" -# pragma clang diagnostic ignored "-Wsign-conversion" -#endif - -#if BOOST_VERSION >= 104800 -# include -#else -# include -#endif - -#ifdef __clang__ -# pragma clang diagnostic pop -#endif +#include #include #include @@ -103,6 +87,8 @@ namespace osmium { char m_tmp_buffer[tmp_buffer_size+1]; + bool m_add_metadata; + template void output_formatted(const char* format, TArgs&&... args) { #ifndef NDEBUG @@ -117,13 +103,12 @@ namespace osmium { *m_out += m_tmp_buffer; } - void append_encoded_string(const std::string& data) { - boost::u8_to_u32_iterator it(data.cbegin(), data.cbegin(), data.cend()); - boost::u8_to_u32_iterator end(data.cend(), data.cend(), data.cend()); - boost::utf8_output_iterator> oit(std::back_inserter(*m_out)); + void append_encoded_string(const char* data) { + const char* end = data + std::strlen(data); - for (; it != end; ++it) { - uint32_t c = *it; + while (data != end) { + const char* last = data; + uint32_t c = utf8::next(data, end); // This is a list of Unicode code points that we let // through instead of escaping them. It is incomplete @@ -138,21 +123,29 @@ namespace osmium { (0x0041 <= c && c <= 0x007e) || (0x00a1 <= c && c <= 0x00ac) || (0x00ae <= c && c <= 0x05ff)) { - *oit = c; + m_out->append(last, data); } else { *m_out += '%'; - output_formatted("%04x", c); + if (c <= 0xff) { + output_formatted("%02x", c); + } else { + output_formatted("%04x", c); + } + *m_out += '%'; } } } void write_meta(const osmium::OSMObject& object) { - output_formatted("%" PRId64 " v%d d", object.id(), object.version()); - *m_out += (object.visible() ? 'V' : 'D'); - output_formatted(" c%d t", object.changeset()); - *m_out += object.timestamp().to_iso(); - output_formatted(" i%d u", object.uid()); - append_encoded_string(object.user()); + output_formatted("%" PRId64, object.id()); + if (m_add_metadata) { + output_formatted(" v%d d", object.version()); + *m_out += (object.visible() ? 'V' : 'D'); + output_formatted(" c%d t", object.changeset()); + *m_out += object.timestamp().to_iso(); + output_formatted(" i%d u", object.uid()); + append_encoded_string(object.user()); + } *m_out += " T"; bool first = true; for (const auto& tag : object.tags()) { @@ -180,10 +173,11 @@ namespace osmium { public: - explicit OPLOutputBlock(osmium::memory::Buffer&& buffer) : + explicit OPLOutputBlock(osmium::memory::Buffer&& buffer, bool add_metadata) : m_input_buffer(std::make_shared(std::move(buffer))), m_out(std::make_shared()), - m_tmp_buffer() { + m_tmp_buffer(), + m_add_metadata(add_metadata) { } OPLOutputBlock(const OPLOutputBlock&) = default; @@ -240,7 +234,7 @@ namespace osmium { } *m_out += item_type_to_char(member.type()); output_formatted("%" PRId64 "@", member.ref()); - *m_out += member.role(); + append_encoded_string(member.role()); } *m_out += '\n'; } @@ -274,17 +268,20 @@ namespace osmium { class OPLOutputFormat : public osmium::io::detail::OutputFormat { - OPLOutputFormat(const OPLOutputFormat&) = delete; - OPLOutputFormat& operator=(const OPLOutputFormat&) = delete; + bool m_add_metadata; public: OPLOutputFormat(const osmium::io::File& file, data_queue_type& output_queue) : - OutputFormat(file, output_queue) { + OutputFormat(file, output_queue), + m_add_metadata(file.get("add_metadata") != "false") { } + OPLOutputFormat(const OPLOutputFormat&) = delete; + OPLOutputFormat& operator=(const OPLOutputFormat&) = delete; + void write_buffer(osmium::memory::Buffer&& buffer) override final { - m_output_queue.push(osmium::thread::Pool::instance().submit(OPLOutputBlock{std::move(buffer)})); + m_output_queue.push(osmium::thread::Pool::instance().submit(OPLOutputBlock{std::move(buffer), m_add_metadata})); } void close() override final { @@ -298,6 +295,8 @@ namespace osmium { namespace { +// we want the register_output_format() function to run, setting the variable +// is only a side-effect, it will never be used #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wunused-variable" const bool registered_opl_output = osmium::io::detail::OutputFormatFactory::instance().register_output_format(osmium::io::file_format::opl, diff --git a/include/osmium/io/detail/pbf.hpp b/include/osmium/io/detail/pbf.hpp index e64e51a57c0..15e457a1235 100644 --- a/include/osmium/io/detail/pbf.hpp +++ b/include/osmium/io/detail/pbf.hpp @@ -33,9 +33,7 @@ DEALINGS IN THE SOFTWARE. */ -#include - -#include +#include // needed for htonl and ntohl #ifndef _WIN32 @@ -45,38 +43,10 @@ DEALINGS IN THE SOFTWARE. #endif #include -#include +#include namespace osmium { -// avoid g++ false positive -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wreturn-type" - inline item_type osmpbf_membertype_to_item_type(const OSMPBF::Relation::MemberType mt) { - switch (mt) { - case OSMPBF::Relation::NODE: - return item_type::node; - case OSMPBF::Relation::WAY: - return item_type::way; - case OSMPBF::Relation::RELATION: - return item_type::relation; - } - } -#pragma GCC diagnostic pop - - inline OSMPBF::Relation::MemberType item_type_to_osmpbf_membertype(const item_type type) { - switch (type) { - case item_type::node: - return OSMPBF::Relation::NODE; - case item_type::way: - return OSMPBF::Relation::WAY; - case item_type::relation: - return OSMPBF::Relation::RELATION; - default: - throw std::runtime_error("Unknown relation member type"); - } - } - /** * Exception thrown when there was a problem with parsing the PBF format of * a file. @@ -93,6 +63,26 @@ namespace osmium { }; // struct pbf_error + namespace io { + + namespace detail { + + // the maximum size of a blob header in bytes + const int max_blob_header_size = 64 * 1024; // 64 kB + + // the maximum size of an uncompressed blob in bytes + const uint64_t max_uncompressed_blob_size = 32 * 1024 * 1024; // 32 MB + + // resolution for longitude/latitude used for conversion + // between representation as double and as int + const int64_t lonlat_resolution = 1000 * 1000 * 1000; + + const int64_t resolution_convert = lonlat_resolution / osmium::Location::coordinate_precision; + + } + + } + } // namespace osmium #endif // OSMIUM_IO_DETAIL_PBF_HPP diff --git a/include/osmium/io/detail/pbf_decoder.hpp b/include/osmium/io/detail/pbf_decoder.hpp new file mode 100644 index 00000000000..79e899ff85e --- /dev/null +++ b/include/osmium/io/detail/pbf_decoder.hpp @@ -0,0 +1,760 @@ +#ifndef OSMIUM_IO_DETAIL_PBF_DECODER_HPP +#define OSMIUM_IO_DETAIL_PBF_DECODER_HPP + +/* + +This file is part of Osmium (http://osmcode.org/libosmium). + +Copyright 2013-2015 Jochen Topf and others (see README). + +Boost Software License - Version 1.0 - August 17th, 2003 + +Permission is hereby granted, free of charge, to any person or organization +obtaining a copy of the software and accompanying documentation covered by +this license (the "Software") to use, reproduce, display, distribute, +execute, and transmit the Software, and to prepare derivative works of the +Software, and to permit third-parties to whom the Software is furnished to +do so, all subject to the following: + +The copyright notices in the Software and this entire statement, including +the above license grant, this restriction and the following disclaimer, +must be included in all copies of the Software, in whole or in part, and +all derivative works of the Software, unless such copies or derivative +works are solely in the form of machine-executable object code generated by +a source language processor. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT +SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE +FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, +ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. + +*/ + +#include +#include +#include +#include +#include +#include + +#include + +#include +#include // IWYU pragma: export +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace osmium { + + namespace io { + + namespace detail { + + using ptr_len_type = std::pair; + + class PBFPrimitiveBlockDecoder { + + static constexpr size_t initial_buffer_size = 2 * 1024 * 1024; + + ptr_len_type m_data; + std::vector m_stringtable; + + int64_t m_lon_offset = 0; + int64_t m_lat_offset = 0; + int64_t m_date_factor = 1000; + int32_t m_granularity = 100; + + osmium::osm_entity_bits::type m_read_types; + + osmium::memory::Buffer m_buffer { initial_buffer_size }; + + void decode_stringtable(const ptr_len_type& data) { + if (!m_stringtable.empty()) { + throw osmium::pbf_error("more than one stringtable in pbf file"); + } + + protozero::pbf_message pbf_string_table(data); + while (pbf_string_table.next(OSMFormat::StringTable::repeated_bytes_s)) { + m_stringtable.push_back(pbf_string_table.get_data()); + } + } + + void decode_primitive_block_metadata() { + protozero::pbf_message pbf_primitive_block(m_data); + while (pbf_primitive_block.next()) { + switch (pbf_primitive_block.tag()) { + case OSMFormat::PrimitiveBlock::required_StringTable_stringtable: + decode_stringtable(pbf_primitive_block.get_data()); + break; + case OSMFormat::PrimitiveBlock::optional_int32_granularity: + m_granularity = pbf_primitive_block.get_int32(); + break; + case OSMFormat::PrimitiveBlock::optional_int32_date_granularity: + m_date_factor = pbf_primitive_block.get_int32(); + break; + case OSMFormat::PrimitiveBlock::optional_int64_lat_offset: + m_lat_offset = pbf_primitive_block.get_int64(); + break; + case OSMFormat::PrimitiveBlock::optional_int64_lon_offset: + m_lon_offset = pbf_primitive_block.get_int64(); + break; + default: + pbf_primitive_block.skip(); + } + } + } + + void decode_primitive_block_data() { + protozero::pbf_message pbf_primitive_block(m_data); + while (pbf_primitive_block.next(OSMFormat::PrimitiveBlock::repeated_PrimitiveGroup_primitivegroup)) { + protozero::pbf_message pbf_primitive_group = pbf_primitive_block.get_message(); + while (pbf_primitive_group.next()) { + switch (pbf_primitive_group.tag()) { + case OSMFormat::PrimitiveGroup::repeated_Node_nodes: + if (m_read_types & osmium::osm_entity_bits::node) { + decode_node(pbf_primitive_group.get_data()); + } else { + pbf_primitive_group.skip(); + } + break; + case OSMFormat::PrimitiveGroup::optional_DenseNodes_dense: + if (m_read_types & osmium::osm_entity_bits::node) { + decode_dense_nodes(pbf_primitive_group.get_data()); + } else { + pbf_primitive_group.skip(); + } + break; + case OSMFormat::PrimitiveGroup::repeated_Way_ways: + if (m_read_types & osmium::osm_entity_bits::way) { + decode_way(pbf_primitive_group.get_data()); + } else { + pbf_primitive_group.skip(); + } + break; + case OSMFormat::PrimitiveGroup::repeated_Relation_relations: + if (m_read_types & osmium::osm_entity_bits::relation) { + decode_relation(pbf_primitive_group.get_data()); + } else { + pbf_primitive_group.skip(); + } + break; + default: + pbf_primitive_group.skip(); + } + } + } + } + + ptr_len_type decode_info(const ptr_len_type& data, osmium::OSMObject& object) { + ptr_len_type user = std::make_pair("", 0); + + protozero::pbf_message pbf_info(data); + while (pbf_info.next()) { + switch (pbf_info.tag()) { + case OSMFormat::Info::optional_int32_version: + { + auto version = pbf_info.get_int32(); + if (version < 0) { + throw osmium::pbf_error("object version must not be negative"); + } + object.set_version(static_cast_with_assert(version)); + } + break; + case OSMFormat::Info::optional_int64_timestamp: + object.set_timestamp(pbf_info.get_int64() * m_date_factor / 1000); + break; + case OSMFormat::Info::optional_int64_changeset: + { + auto changeset_id = pbf_info.get_int64(); + if (changeset_id < 0) { + throw osmium::pbf_error("object changeset_id must not be negative"); + } + object.set_changeset(static_cast_with_assert(changeset_id)); + } + break; + case OSMFormat::Info::optional_int32_uid: + object.set_uid_from_signed(pbf_info.get_int32()); + break; + case OSMFormat::Info::optional_uint32_user_sid: + user = m_stringtable.at(pbf_info.get_uint32()); + break; + case OSMFormat::Info::optional_bool_visible: + object.set_visible(pbf_info.get_bool()); + break; + default: + pbf_info.skip(); + } + } + + return user; + } + + using kv_type = std::pair; + + void build_tag_list(osmium::builder::Builder& builder, const kv_type& keys, const kv_type& vals) { + if (keys.first != keys.second) { + osmium::builder::TagListBuilder tl_builder(m_buffer, &builder); + auto kit = keys.first; + auto vit = vals.first; + while (kit != keys.second) { + if (vit == vals.second) { + // this is against the spec, must have same number of elements + throw osmium::pbf_error("PBF format error"); + } + const auto& k = m_stringtable.at(*kit++); + const auto& v = m_stringtable.at(*vit++); + tl_builder.add_tag(k.first, k.second, v.first, v.second); + } + } + } + + int32_t convert_pbf_coordinate(int64_t c) const { + return (c * m_granularity + m_lon_offset) / resolution_convert; + } + + void decode_node(const ptr_len_type& data) { + osmium::builder::NodeBuilder builder(m_buffer); + osmium::Node& node = builder.object(); + + kv_type keys; + kv_type vals; + int64_t lon = std::numeric_limits::max(); + int64_t lat = std::numeric_limits::max(); + + ptr_len_type user = { "", 0 }; + + protozero::pbf_message pbf_node(data); + while (pbf_node.next()) { + switch (pbf_node.tag()) { + case OSMFormat::Node::required_sint64_id: + node.set_id(pbf_node.get_sint64()); + break; + case OSMFormat::Node::packed_uint32_keys: + keys = pbf_node.get_packed_uint32(); + break; + case OSMFormat::Node::packed_uint32_vals: + vals = pbf_node.get_packed_uint32(); + break; + case OSMFormat::Node::optional_Info_info: + user = decode_info(pbf_node.get_data(), builder.object()); + break; + case OSMFormat::Node::required_sint64_lat: + lat = pbf_node.get_sint64(); + break; + case OSMFormat::Node::required_sint64_lon: + lon = pbf_node.get_sint64(); + break; + default: + pbf_node.skip(); + } + } + + if (node.visible()) { + if (lon == std::numeric_limits::max() || + lat == std::numeric_limits::max()) { + throw osmium::pbf_error("illegal coordinate format"); + } + node.set_location(osmium::Location( + convert_pbf_coordinate(lon), + convert_pbf_coordinate(lat) + )); + } + + builder.add_user(user.first, user.second); + + build_tag_list(builder, keys, vals); + + m_buffer.commit(); + } + + void decode_way(const ptr_len_type& data) { + osmium::builder::WayBuilder builder(m_buffer); + + kv_type keys; + kv_type vals; + std::pair refs; + + ptr_len_type user = { "", 0 }; + + protozero::pbf_message pbf_way(data); + while (pbf_way.next()) { + switch (pbf_way.tag()) { + case OSMFormat::Way::required_int64_id: + builder.object().set_id(pbf_way.get_int64()); + break; + case OSMFormat::Way::packed_uint32_keys: + keys = pbf_way.get_packed_uint32(); + break; + case OSMFormat::Way::packed_uint32_vals: + vals = pbf_way.get_packed_uint32(); + break; + case OSMFormat::Way::optional_Info_info: + user = decode_info(pbf_way.get_data(), builder.object()); + break; + case OSMFormat::Way::packed_sint64_refs: + refs = pbf_way.get_packed_sint64(); + break; + default: + pbf_way.skip(); + } + } + + builder.add_user(user.first, user.second); + + if (refs.first != refs.second) { + osmium::builder::WayNodeListBuilder wnl_builder(m_buffer, &builder); + osmium::util::DeltaDecode ref; + while (refs.first != refs.second) { + wnl_builder.add_node_ref(ref.update(*refs.first++)); + } + } + + build_tag_list(builder, keys, vals); + + m_buffer.commit(); + } + + void decode_relation(const ptr_len_type& data) { + osmium::builder::RelationBuilder builder(m_buffer); + + kv_type keys; + kv_type vals; + std::pair roles; + std::pair refs; + std::pair types; + + ptr_len_type user = { "", 0 }; + + protozero::pbf_message pbf_relation(data); + while (pbf_relation.next()) { + switch (pbf_relation.tag()) { + case OSMFormat::Relation::required_int64_id: + builder.object().set_id(pbf_relation.get_int64()); + break; + case OSMFormat::Relation::packed_uint32_keys: + keys = pbf_relation.get_packed_uint32(); + break; + case OSMFormat::Relation::packed_uint32_vals: + vals = pbf_relation.get_packed_uint32(); + break; + case OSMFormat::Relation::optional_Info_info: + user = decode_info(pbf_relation.get_data(), builder.object()); + break; + case OSMFormat::Relation::packed_int32_roles_sid: + roles = pbf_relation.get_packed_int32(); + break; + case OSMFormat::Relation::packed_sint64_memids: + refs = pbf_relation.get_packed_sint64(); + break; + case OSMFormat::Relation::packed_MemberType_types: + types = pbf_relation.get_packed_enum(); + break; + default: + pbf_relation.skip(); + } + } + + builder.add_user(user.first, user.second); + + if (refs.first != refs.second) { + osmium::builder::RelationMemberListBuilder rml_builder(m_buffer, &builder); + osmium::util::DeltaDecode ref; + while (roles.first != roles.second && refs.first != refs.second && types.first != types.second) { + const auto& r = m_stringtable.at(*roles.first++); + int type = *types.first++; + if (type < 0 || type > 2) { + throw osmium::pbf_error("unknown relation member type"); + } + rml_builder.add_member( + osmium::item_type(type + 1), + ref.update(*refs.first++), + r.first, + r.second + ); + } + } + + build_tag_list(builder, keys, vals); + + m_buffer.commit(); + } + + void decode_dense_nodes(const ptr_len_type& data) { + bool has_info = false; + bool has_visibles = false; + + std::pair ids; + std::pair lats; + std::pair lons; + + std::pair tags; + + std::pair versions; + std::pair timestamps; + std::pair changesets; + std::pair uids; + std::pair user_sids; + std::pair visibles; + + protozero::pbf_message pbf_dense_nodes(data); + while (pbf_dense_nodes.next()) { + switch (pbf_dense_nodes.tag()) { + case OSMFormat::DenseNodes::packed_sint64_id: + ids = pbf_dense_nodes.get_packed_sint64(); + break; + case OSMFormat::DenseNodes::optional_DenseInfo_denseinfo: + { + has_info = true; + protozero::pbf_message pbf_dense_info = pbf_dense_nodes.get_message(); + while (pbf_dense_info.next()) { + switch (pbf_dense_info.tag()) { + case OSMFormat::DenseInfo::packed_int32_version: + versions = pbf_dense_info.get_packed_int32(); + break; + case OSMFormat::DenseInfo::packed_sint64_timestamp: + timestamps = pbf_dense_info.get_packed_sint64(); + break; + case OSMFormat::DenseInfo::packed_sint64_changeset: + changesets = pbf_dense_info.get_packed_sint64(); + break; + case OSMFormat::DenseInfo::packed_sint32_uid: + uids = pbf_dense_info.get_packed_sint32(); + break; + case OSMFormat::DenseInfo::packed_sint32_user_sid: + user_sids = pbf_dense_info.get_packed_sint32(); + break; + case OSMFormat::DenseInfo::packed_bool_visible: + has_visibles = true; + visibles = pbf_dense_info.get_packed_bool(); + break; + default: + pbf_dense_info.skip(); + } + } + } + break; + case OSMFormat::DenseNodes::packed_sint64_lat: + lats = pbf_dense_nodes.get_packed_sint64(); + break; + case OSMFormat::DenseNodes::packed_sint64_lon: + lons = pbf_dense_nodes.get_packed_sint64(); + break; + case OSMFormat::DenseNodes::packed_int32_keys_vals: + tags = pbf_dense_nodes.get_packed_int32(); + break; + default: + pbf_dense_nodes.skip(); + } + } + + osmium::util::DeltaDecode dense_id; + osmium::util::DeltaDecode dense_latitude; + osmium::util::DeltaDecode dense_longitude; + osmium::util::DeltaDecode dense_uid; + osmium::util::DeltaDecode dense_user_sid; + osmium::util::DeltaDecode dense_changeset; + osmium::util::DeltaDecode dense_timestamp; + + auto tag_it = tags.first; + + while (ids.first != ids.second) { + if (lons.first == lons.second || + lats.first == lats.second) { + // this is against the spec, must have same number of elements + throw osmium::pbf_error("PBF format error"); + } + + bool visible = true; + + osmium::builder::NodeBuilder builder(m_buffer); + osmium::Node& node = builder.object(); + + node.set_id(dense_id.update(*ids.first++)); + + if (has_info) { + if (versions.first == versions.second || + changesets.first == changesets.second || + timestamps.first == timestamps.second || + uids.first == uids.second || + user_sids.first == user_sids.second) { + // this is against the spec, must have same number of elements + throw osmium::pbf_error("PBF format error"); + } + + auto version = *versions.first++; + if (version < 0) { + throw osmium::pbf_error("object version must not be negative"); + } + node.set_version(static_cast(version)); + + auto changeset_id = dense_changeset.update(*changesets.first++); + if (changeset_id < 0) { + throw osmium::pbf_error("object changeset_id must not be negative"); + } + node.set_changeset(static_cast(changeset_id)); + + node.set_timestamp(dense_timestamp.update(*timestamps.first++) * m_date_factor / 1000); + node.set_uid_from_signed(static_cast(dense_uid.update(*uids.first++))); + + if (has_visibles) { + if (visibles.first == visibles.second) { + // this is against the spec, must have same number of elements + throw osmium::pbf_error("PBF format error"); + } + visible = *visibles.first++; + } + node.set_visible(visible); + + const auto& u = m_stringtable.at(dense_user_sid.update(*user_sids.first++)); + builder.add_user(u.first, u.second); + } else { + builder.add_user(""); + } + + if (visible) { + builder.object().set_location(osmium::Location( + convert_pbf_coordinate(dense_longitude.update(*lons.first++)), + convert_pbf_coordinate(dense_latitude.update(*lats.first++)) + )); + } + + if (tag_it != tags.second) { + osmium::builder::TagListBuilder tl_builder(m_buffer, &builder); + while (tag_it != tags.second && *tag_it != 0) { + const auto& k = m_stringtable.at(*tag_it++); + if (tag_it == tags.second) { + throw osmium::pbf_error("PBF format error"); // this is against the spec, keys/vals must come in pairs + } + const auto& v = m_stringtable.at(*tag_it++); + tl_builder.add_tag(k.first, k.second, v.first, v.second); + } + + if (tag_it != tags.second) { + ++tag_it; + } + } + + m_buffer.commit(); + } + + } + + public: + + explicit PBFPrimitiveBlockDecoder(const ptr_len_type& data, osmium::osm_entity_bits::type read_types) : + m_data(data), + m_read_types(read_types) { + } + + PBFPrimitiveBlockDecoder(const PBFPrimitiveBlockDecoder&) = delete; + PBFPrimitiveBlockDecoder& operator=(const PBFPrimitiveBlockDecoder&) = delete; + + PBFPrimitiveBlockDecoder(PBFPrimitiveBlockDecoder&&) = delete; + PBFPrimitiveBlockDecoder& operator=(PBFPrimitiveBlockDecoder&&) = delete; + + ~PBFPrimitiveBlockDecoder() = default; + + osmium::memory::Buffer operator()() { + try { + decode_primitive_block_metadata(); + decode_primitive_block_data(); + } catch (std::out_of_range&) { + throw osmium::pbf_error("string id out of range"); + } + + return std::move(m_buffer); + } + + }; // class PBFPrimitiveBlockDecoder + + inline ptr_len_type decode_blob(const std::string& blob_data, std::string& output) { + int32_t raw_size; + std::pair zlib_data; + + protozero::pbf_message pbf_blob(blob_data); + while (pbf_blob.next()) { + switch (pbf_blob.tag()) { + case FileFormat::Blob::optional_bytes_raw: + { + auto data_len = pbf_blob.get_data(); + if (data_len.second > max_uncompressed_blob_size) { + throw osmium::pbf_error("illegal blob size"); + } + return data_len; + } + case FileFormat::Blob::optional_int32_raw_size: + raw_size = pbf_blob.get_int32(); + if (raw_size <= 0 || uint32_t(raw_size) > max_uncompressed_blob_size) { + throw osmium::pbf_error("illegal blob size"); + } + break; + case FileFormat::Blob::optional_bytes_zlib_data: + zlib_data = pbf_blob.get_data(); + break; + case FileFormat::Blob::optional_bytes_lzma_data: + throw osmium::pbf_error("lzma blobs not implemented"); + default: + throw osmium::pbf_error("unknown compression"); + } + } + + if (zlib_data.second != 0) { + return osmium::io::detail::zlib_uncompress_string( + zlib_data.first, + static_cast(zlib_data.second), + static_cast(raw_size), + output + ); + } + + throw osmium::pbf_error("blob contains no data"); + } + + inline osmium::Box decode_header_bbox(const ptr_len_type& data) { + int64_t left = std::numeric_limits::max(); + int64_t right = std::numeric_limits::max(); + int64_t top = std::numeric_limits::max(); + int64_t bottom = std::numeric_limits::max(); + + protozero::pbf_message pbf_header_bbox(data); + while (pbf_header_bbox.next()) { + switch (pbf_header_bbox.tag()) { + case OSMFormat::HeaderBBox::required_sint64_left: + left = pbf_header_bbox.get_sint64(); + break; + case OSMFormat::HeaderBBox::required_sint64_right: + right = pbf_header_bbox.get_sint64(); + break; + case OSMFormat::HeaderBBox::required_sint64_top: + top = pbf_header_bbox.get_sint64(); + break; + case OSMFormat::HeaderBBox::required_sint64_bottom: + bottom = pbf_header_bbox.get_sint64(); + break; + default: + pbf_header_bbox.skip(); + } + } + + if (left == std::numeric_limits::max() || + right == std::numeric_limits::max() || + top == std::numeric_limits::max() || + bottom == std::numeric_limits::max()) { + throw osmium::pbf_error("invalid bbox"); + } + + osmium::Box box; + box.extend(osmium::Location(left / resolution_convert, bottom / resolution_convert)); + box.extend(osmium::Location(right / resolution_convert, top / resolution_convert)); + + return box; + } + + inline osmium::io::Header decode_header_block(const ptr_len_type& data) { + osmium::io::Header header; + int i = 0; + + protozero::pbf_message pbf_header_block(data); + while (pbf_header_block.next()) { + switch (pbf_header_block.tag()) { + case OSMFormat::HeaderBlock::optional_HeaderBBox_bbox: + header.add_box(decode_header_bbox(pbf_header_block.get_data())); + break; + case OSMFormat::HeaderBlock::repeated_string_required_features: + { + auto feature = pbf_header_block.get_data(); + if (!strncmp("OsmSchema-V0.6", feature.first, feature.second)) { + // intentionally left blank + } else if (!strncmp("DenseNodes", feature.first, feature.second)) { + header.set("pbf_dense_nodes", true); + } else if (!strncmp("HistoricalInformation", feature.first, feature.second)) { + header.set_has_multiple_object_versions(true); + } else { + std::string msg("required feature not supported: "); + msg.append(feature.first, feature.second); + throw osmium::pbf_error(msg); + } + } + break; + case OSMFormat::HeaderBlock::repeated_string_optional_features: + header.set("pbf_optional_feature_" + std::to_string(i++), pbf_header_block.get_string()); + break; + case OSMFormat::HeaderBlock::optional_string_writingprogram: + header.set("generator", pbf_header_block.get_string()); + break; + case OSMFormat::HeaderBlock::optional_int64_osmosis_replication_timestamp: + header.set("osmosis_replication_timestamp", osmium::Timestamp(pbf_header_block.get_int64()).to_iso()); + break; + case OSMFormat::HeaderBlock::optional_int64_osmosis_replication_sequence_number: + header.set("osmosis_replication_sequence_number", std::to_string(pbf_header_block.get_int64())); + break; + case OSMFormat::HeaderBlock::optional_string_osmosis_replication_base_url: + header.set("osmosis_replication_base_url", pbf_header_block.get_string()); + break; + default: + pbf_header_block.skip(); + } + } + + return header; + } + + /** + * Decode HeaderBlock. + * + * @param header_block_data Input data + * @returns Header object + * @throws osmium::pbf_error If there was a parsing error + */ + inline osmium::io::Header decode_header(const std::string& header_block_data) { + std::string output; + + return decode_header_block(decode_blob(header_block_data, output)); + } + + class PBFDataBlobDecoder { + + std::shared_ptr m_input_buffer; + osmium::osm_entity_bits::type m_read_types; + + public: + + PBFDataBlobDecoder(std::string&& input_buffer, osmium::osm_entity_bits::type read_types) : + m_input_buffer(std::make_shared(std::move(input_buffer))), + m_read_types(read_types) { + } + + PBFDataBlobDecoder(const PBFDataBlobDecoder&) = default; + PBFDataBlobDecoder& operator=(const PBFDataBlobDecoder&) = default; + + PBFDataBlobDecoder(PBFDataBlobDecoder&&) = default; + PBFDataBlobDecoder& operator=(PBFDataBlobDecoder&&) = default; + + ~PBFDataBlobDecoder() = default; + + osmium::memory::Buffer operator()() { + std::string output; + PBFPrimitiveBlockDecoder decoder(decode_blob(*m_input_buffer, output), m_read_types); + return decoder(); + } + + }; // class PBFDataBlobDecoder + + } // namespace detail + + } // namespace io + +} // namespace osmium + +#endif // OSMIUM_IO_DETAIL_PBF_DECODER_HPP diff --git a/include/osmium/io/detail/pbf_input_format.hpp b/include/osmium/io/detail/pbf_input_format.hpp index ba8fb42a2e6..7817d27d164 100644 --- a/include/osmium/io/detail/pbf_input_format.hpp +++ b/include/osmium/io/detail/pbf_input_format.hpp @@ -49,9 +49,12 @@ DEALINGS IN THE SOFTWARE. #include #include +#include + #include #include // IWYU pragma: export -#include +#include +#include #include #include #include @@ -76,13 +79,13 @@ namespace osmium { namespace detail { - typedef osmium::thread::Queue> queue_type; - /** * Class for parsing PBF files. */ class PBFInputFormat : public osmium::io::detail::InputFormat { + typedef osmium::thread::Queue> queue_type; + bool m_use_thread_pool; bool m_eof { false }; queue_type m_queue; @@ -115,15 +118,10 @@ namespace osmium { } /** - * Read BlobHeader by first reading the size and then the - * BlobHeader. The BlobHeader contains a type field (which is - * checked against the expected type) and a size field. - * - * @param expected_type Expected type of data ("OSMHeader" or - * "OSMData"). - * @returns Size of the data read from BlobHeader (0 on EOF). + * Read 4 bytes in network byte order from file. They contain + * the length of the following BlobHeader. */ - size_t read_blob_header(const char* expected_type) { + uint32_t read_blob_header_size_from_file() { uint32_t size_in_network_byte_order; try { @@ -133,37 +131,76 @@ namespace osmium { return 0; // EOF } - uint32_t size = ntohl(size_in_network_byte_order); - if (size > static_cast(OSMPBF::max_blob_header_size)) { + const uint32_t size = ntohl(size_in_network_byte_order); + if (size > static_cast(max_blob_header_size)) { throw osmium::pbf_error("invalid BlobHeader size (> max_blob_header_size)"); } - OSMPBF::BlobHeader blob_header; - if (!blob_header.ParseFromString(read_from_input_queue(size))) { - throw osmium::pbf_error("failed to parse BlobHeader"); + return size; + } + + /** + * Decode the BlobHeader. Make sure it contains the expected + * type. Return the size of the following Blob. + */ + size_t decode_blob_header(protozero::pbf_message&& pbf_blob_header, const char* expected_type) { + std::pair blob_header_type; + size_t blob_header_datasize = 0; + + while (pbf_blob_header.next()) { + switch (pbf_blob_header.tag()) { + case FileFormat::BlobHeader::required_string_type: + blob_header_type = pbf_blob_header.get_data(); + break; + case FileFormat::BlobHeader::required_int32_datasize: + blob_header_datasize = pbf_blob_header.get_int32(); + break; + default: + pbf_blob_header.skip(); + } } - if (blob_header.type() != expected_type) { + if (blob_header_datasize == 0) { + throw osmium::pbf_error("PBF format error: BlobHeader.datasize missing or zero."); + } + + if (strncmp(expected_type, blob_header_type.first, blob_header_type.second)) { throw osmium::pbf_error("blob does not have expected type (OSMHeader in first blob, OSMData in following blobs)"); } - return static_cast(blob_header.datasize()); + return blob_header_datasize; + } + + size_t check_type_and_get_blob_size(const char* expected_type) { + assert(expected_type); + + auto size = read_blob_header_size_from_file(); + if (size == 0) { // EOF + return 0; + } + + std::string blob_header = read_from_input_queue(size); + + return decode_blob_header(protozero::pbf_message(blob_header), expected_type); } void parse_osm_data(osmium::osm_entity_bits::type read_types) { osmium::thread::set_thread_name("_osmium_pbf_in"); - int n = 0; - while (auto size = read_blob_header("OSMData")) { + + while (auto size = check_type_and_get_blob_size("OSMData")) { + std::string input_buffer = read_from_input_queue(size); + if (input_buffer.size() > max_uncompressed_blob_size) { + throw osmium::pbf_error(std::string("invalid blob size: " + std::to_string(input_buffer.size()))); + } if (m_use_thread_pool) { - m_queue.push(osmium::thread::Pool::instance().submit(DataBlobParser{read_from_input_queue(size), read_types})); + m_queue.push(osmium::thread::Pool::instance().submit(PBFDataBlobDecoder{ std::move(input_buffer), read_types })); } else { std::promise promise; m_queue.push(promise.get_future()); - DataBlobParser data_blob_parser{read_from_input_queue(size), read_types}; + PBFDataBlobDecoder data_blob_parser{ std::move(input_buffer), read_types }; promise.set_value(data_blob_parser()); } - ++n; if (m_quit_input_thread) { return; @@ -197,11 +234,10 @@ namespace osmium { m_quit_input_thread(false), m_input_queue(input_queue), m_input_buffer() { - GOOGLE_PROTOBUF_VERIFY_VERSION; // handle OSMHeader - auto size = read_blob_header("OSMHeader"); - m_header = parse_header_blob(read_from_input_queue(size)); + const auto size = check_type_and_get_blob_size("OSMHeader"); + m_header = decode_header(read_from_input_queue(size)); if (m_read_which_entities != osmium::osm_entity_bits::nothing) { m_reader = std::thread(&PBFInputFormat::parse_osm_data, this, m_read_which_entities); @@ -246,10 +282,15 @@ namespace osmium { namespace { +// we want the register_input_format() function to run, setting the variable +// is only a side-effect, it will never be used +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wunused-variable" const bool registered_pbf_input = osmium::io::detail::InputFormatFactory::instance().register_input_format(osmium::io::file_format::pbf, [](const osmium::io::File& file, osmium::osm_entity_bits::type read_which_entities, osmium::thread::Queue& input_queue) { return new osmium::io::detail::PBFInputFormat(file, read_which_entities, input_queue); }); +#pragma GCC diagnostic pop } // anonymous namespace diff --git a/include/osmium/io/detail/pbf_output_format.hpp b/include/osmium/io/detail/pbf_output_format.hpp index 288008f6ad6..8d8a079b494 100644 --- a/include/osmium/io/detail/pbf_output_format.hpp +++ b/include/osmium/io/detail/pbf_output_format.hpp @@ -33,75 +33,13 @@ DEALINGS IN THE SOFTWARE. */ -/* - -About the .osm.pbf file format -This is an excerpt of - -The .osm.pbf format and it's derived formats (.osh.pbf and .osc.pbf) are encoded -using googles protobuf library for the low-level storage. They are constructed -by nesting data on two levels: - -On the lower level the file is constructed using BlobHeaders and Blobs. A .osm.pbf -file contains multiple sequences of - 1. a 4-byte header size, stored in network-byte-order - 2. a BlobHeader of exactly this size - 3. a Blob - -The BlobHeader tells the reader about the type and size of the following Blob. The -Blob can contain data in raw or zlib-compressed form. After uncompressing the blob -it is treated differently depending on the type specified in the BlobHeader. - -The contents of the Blob belongs to the higher level. It contains either an HeaderBlock -(type="OSMHeader") or an PrimitiveBlock (type="OSMData"). The file needs to have -at least one HeaderBlock before the first PrimitiveBlock. - -The HeaderBlock contains meta-information like the writing program or a bbox. It may -also contain multiple "required features" that describe what kinds of input a -reading program needs to handle in order to fully understand the files' contents. - -The PrimitiveBlock can store multiple types of objects (i.e. 5 nodes, 2 ways and -1 relation). It contains one or more PrimitiveGroup which in turn contain multiple -nodes, ways or relations. A PrimitiveGroup should only contain one kind of object. - -There's a special kind of "object type" called dense-nodes. It is used to store nodes -in a very dense format, avoiding message overheads and using delta-encoding for nearly -all ids. - -All Strings are stored as indexes to rows in a StringTable. The StringTable contains -one row for each used string, so strings that are used multiple times need to be -stored only once. The StringTable is sorted by usage-count, so the most often used -string is stored at index 1. - -A simple outline of a .osm.pbf file could look like this: - - 4-bytes header size - BlobHeader - Blob - HeaderBlock - 4-bytes header size - BlobHeader - Blob - PrimitiveBlock - StringTable - PrimitiveGroup - 5 nodes - PrimitiveGroup - 2 ways - PrimitiveGroup - 1 relation - -More complete outlines of real .osm.pbf files can be created using the osmpbf-outline tool: - -*/ - #include #include #include #include #include #include -#include +#include #include #include #include @@ -109,10 +47,15 @@ More complete outlines of real .osm.pbf files can be created using the osmpbf-ou #include #include +#include + +#include + #include #include #include // IWYU pragma: export -#include +#include +#include #include #include #include @@ -129,6 +72,7 @@ More complete outlines of real .osm.pbf files can be created using the osmpbf-ou #include #include #include +#include #include namespace osmium { @@ -137,815 +81,493 @@ namespace osmium { namespace detail { - namespace { - - /** - * Serialize a protobuf message into a Blob, optionally apply compression - * and return it together with a BlobHeader ready to be written to a file. - * - * @param type Type-string used in the BlobHeader. - * @param msg Protobuf-message. - * @param use_compression Should the output be compressed using zlib? - */ - std::string serialize_blob(const std::string& type, const google::protobuf::MessageLite& msg, bool use_compression) { - OSMPBF::Blob pbf_blob; - - { - std::string content; - msg.SerializeToString(&content); - - pbf_blob.set_raw_size(static_cast_with_assert<::google::protobuf::int32>(content.size())); - - if (use_compression) { - pbf_blob.set_zlib_data(osmium::io::detail::zlib_compress(content)); - } else { - pbf_blob.set_raw(content); - } - } - - std::string blob_data; - pbf_blob.SerializeToString(&blob_data); - - OSMPBF::BlobHeader pbf_blob_header; - pbf_blob_header.set_type(type); - pbf_blob_header.set_datasize(static_cast_with_assert<::google::protobuf::int32>(blob_data.size())); - - std::string blob_header_data; - pbf_blob_header.SerializeToString(&blob_header_data); - - uint32_t sz = htonl(static_cast_with_assert(blob_header_data.size())); - - // write to output: the 4-byte BlobHeader-Size followed by the BlobHeader followed by the Blob - std::string output; - output.reserve(sizeof(sz) + blob_header_data.size() + blob_data.size()); - output.append(reinterpret_cast(&sz), sizeof(sz)); - output.append(blob_header_data); - output.append(blob_data); - - return output; + /** + * Maximum number of items in a primitive block. + * + * The uncompressed length of a Blob *should* be less + * than 16 megabytes and *must* be less than 32 megabytes. + * + * A block may contain any number of entities, as long as + * the size limits for the surrounding blob are obeyed. + * However, for simplicity, the current Osmosis (0.38) + * as well as Osmium implementation always + * uses at most 8k entities in a block. + */ + constexpr int32_t max_entities_per_block = 8000; + + constexpr int location_granularity = 100; + + /** + * convert a double lat or lon value to an int, respecting the granularity + */ + inline int64_t lonlat2int(double lonlat) { + return static_cast(std::round(lonlat * lonlat_resolution / location_granularity)); + } + + /** + * Serialize a protobuf message into a Blob, optionally apply compression + * and return it together with a BlobHeader ready to be written to a file. + * + * @param type Type-string used in the BlobHeader. + * @param msg Protobuf-message. + * @param use_compression Should the output be compressed using zlib? + */ + inline std::string serialize_blob(const std::string& type, const std::string& msg, bool use_compression) { + std::string blob_data; + protozero::pbf_builder pbf_blob(blob_data); + + if (use_compression) { + pbf_blob.add_int32(FileFormat::Blob::optional_int32_raw_size, msg.size()); + pbf_blob.add_bytes(FileFormat::Blob::optional_bytes_zlib_data, osmium::io::detail::zlib_compress(msg)); + } else { + pbf_blob.add_bytes(FileFormat::Blob::optional_bytes_raw, msg); } - } // anonymous namespace + std::string blob_header_data; + protozero::pbf_builder pbf_blob_header(blob_header_data); - class PBFOutputFormat : public osmium::io::detail::OutputFormat, public osmium::handler::Handler { + pbf_blob_header.add_string(FileFormat::BlobHeader::required_string_type, type); + pbf_blob_header.add_int32(FileFormat::BlobHeader::required_int32_datasize, blob_data.size()); - /** - * This class models a variable that keeps track of the value - * it was last set to and returns the delta between old and - * new value from the update() call. - */ - template - class Delta { + uint32_t sz = htonl(static_cast_with_assert(blob_header_data.size())); - T m_value; + // write to output: the 4-byte BlobHeader-Size followed by the BlobHeader followed by the Blob + std::string output; + output.reserve(sizeof(sz) + blob_header_data.size() + blob_data.size()); + output.append(reinterpret_cast(&sz), sizeof(sz)); + output.append(blob_header_data); + output.append(blob_data); - public: + return output; + } - Delta() : - m_value(0) { - } + class DenseNodes { - void clear() { - m_value = 0; - } + StringTable& m_stringtable; - T update(T new_value) { - using std::swap; - swap(m_value, new_value); - return m_value - new_value; - } + std::vector m_ids; - }; // class Delta + std::vector m_versions; + std::vector m_timestamps; + std::vector m_changesets; + std::vector m_uids; + std::vector m_user_sids; + std::vector m_visibles; - /** - * Maximum number of items in a primitive block. - * - * The uncompressed length of a Blob *should* be less - * than 16 megabytes and *must* be less than 32 megabytes. - * - * A block may contain any number of entities, as long as - * the size limits for the surrounding blob are obeyed. - * However, for simplicity, the current Osmosis (0.38) - * as well as Osmium implementation always - * uses at most 8k entities in a block. - */ - static constexpr uint32_t max_block_contents = 8000; + std::vector m_lats; + std::vector m_lons; + std::vector m_tags; - /** - * The output buffer (block) will be filled to about - * 95% and then written to disk. This leaves more than - * enough space for the string table (which typically - * needs about 0.1 to 0.3% of the block size). - */ - static constexpr int64_t buffer_fill_percent = 95; + osmium::util::DeltaEncode m_delta_id; - /** - * protobuf-struct of a HeaderBlock - */ - OSMPBF::HeaderBlock pbf_header_block; + osmium::util::DeltaEncode m_delta_timestamp; + osmium::util::DeltaEncode m_delta_changeset; + osmium::util::DeltaEncode m_delta_uid; + osmium::util::DeltaEncode m_delta_user_sid; - /** - * protobuf-struct of a PrimitiveBlock - */ - OSMPBF::PrimitiveBlock pbf_primitive_block; + osmium::util::DeltaEncode m_delta_lat; + osmium::util::DeltaEncode m_delta_lon; - /** - * pointer to PrimitiveGroups inside the current PrimitiveBlock, - * used for writing nodes, ways or relations - */ - OSMPBF::PrimitiveGroup* pbf_nodes; - OSMPBF::PrimitiveGroup* pbf_ways; - OSMPBF::PrimitiveGroup* pbf_relations; + bool m_add_metadata; + bool m_add_visible; - /** - * To flexibly handle multiple resolutions, the granularity, or - * resolution used for representing locations is adjustable in - * multiples of 1 nanodegree. The default scaling factor is 100 - * nanodegrees, corresponding to about ~1cm at the equator. - * This is the current resolution of the OSM database. - */ - int m_location_granularity; + public: - /** - * The granularity used for representing timestamps is also adjustable in - * multiples of 1 millisecond. The default scaling factor is 1000 - * milliseconds, which is the current resolution of the OSM database. - */ - int m_date_granularity; + DenseNodes(StringTable& stringtable, bool add_metadata, bool add_visible) : + m_stringtable(stringtable), + m_add_metadata(add_metadata), + m_add_visible(add_visible) { + } - /** - * should nodes be serialized into the dense format? - * - * nodes can be encoded one of two ways, as a Node - * (m_use_dense_nodes = false) and a special dense format. - * In the dense format, all information is stored 'column wise', - * as an array of ID's, array of latitudes, and array of - * longitudes. Each column is delta-encoded. This reduces - * header overheads and allows delta-coding to work very effectively. - */ - bool m_use_dense_nodes {true}; + void clear() { + m_ids.clear(); - /** - * should the PBF blobs contain zlib compressed data? - * - * the zlib compression is optional, it's possible to store the - * blobs in raw format. Disabling the compression can improve the - * writing speed a little but the output will be 2x to 3x bigger. - */ - bool m_use_compression {true}; + m_versions.clear(); + m_timestamps.clear(); + m_changesets.clear(); + m_uids.clear(); + m_user_sids.clear(); + m_visibles.clear(); - /** - * Should the string tables in the data blocks be sorted? - * - * Not sorting the string tables makes writing PBF files - * slightly faster. - */ - bool m_sort_stringtables { true }; + m_lats.clear(); + m_lons.clear(); + m_tags.clear(); - /** - * While the .osm.pbf-format is able to carry all meta information, it is - * also able to omit this information to reduce size. - */ - bool m_should_add_metadata {true}; - - /** - * Should the visible flag be added on objects? - */ - bool m_add_visible; - - /** - * counter used to quickly check the number of objects stored inside - * the current PrimitiveBlock. When the counter reaches max_block_contents - * the PrimitiveBlock is serialized into a Blob and flushed to the file. - * - * this check is performed in check_block_contents_counter() which is - * called once for each object. - */ - uint16_t primitive_block_contents; - int primitive_block_size; + m_delta_id.clear(); - // StringTable management - StringTable string_table; + m_delta_timestamp.clear(); + m_delta_changeset.clear(); + m_delta_uid.clear(); + m_delta_user_sid.clear(); - /** - * These variables are used to calculate the - * delta-encoding while storing dense-nodes. It holds the last seen values - * from which the difference is stored into the protobuf. - */ - Delta m_delta_id; - Delta m_delta_lat; - Delta m_delta_lon; - Delta m_delta_timestamp; - Delta m_delta_changeset; - Delta m_delta_uid; - Delta<::google::protobuf::int32> m_delta_user_sid; - - bool debug; - - bool has_debug_level(int) { - return false; + m_delta_lat.clear(); + m_delta_lon.clear(); } - ///// Blob writing ///// - - void delta_encode_string_ids() { - if (pbf_nodes && pbf_nodes->has_dense()) { - OSMPBF::DenseNodes* dense = pbf_nodes->mutable_dense(); - - if (dense->has_denseinfo()) { - OSMPBF::DenseInfo* denseinfo = dense->mutable_denseinfo(); - - for (int i = 0, l=denseinfo->user_sid_size(); iuser_sid(i); - denseinfo->set_user_sid(i, m_delta_user_sid.update(user_sid)); - } - } - } + size_t size() const { + return m_ids.size() * 3 * sizeof(int64_t); } - /** - * Before a PrimitiveBlock gets serialized, all interim StringTable-ids needs to be - * mapped to the associated real StringTable ids. This is done in this function. - * - * This function needs to know about the concrete structure of all item types to find - * all occurrences of string-ids. - */ - void map_string_ids() { - // test, if the node-block has been allocated - if (pbf_nodes) { - // iterate over all nodes, passing them to the map_common_string_ids function - for (int i = 0, l=pbf_nodes->nodes_size(); imutable_nodes(i)); - } - - // test, if the node-block has a densenodes structure - if (pbf_nodes->has_dense()) { - // get a pointer to the densenodes structure - OSMPBF::DenseNodes* dense = pbf_nodes->mutable_dense(); - - // in the densenodes structure keys and vals are encoded in an intermixed - // array, individual nodes are seperated by a value of 0 (0 in the StringTable - // is always unused). String-ids of 0 are thus kept alone. - for (int i = 0, l=dense->keys_vals_size(); i 0 to real string ids - auto sid = dense->keys_vals(i); - if (sid > 0) { - dense->set_keys_vals(i, string_table.map_string_id(sid)); - } - } - - // test if the densenodes block has meta infos - if (dense->has_denseinfo()) { - // get a pointer to the denseinfo structure - OSMPBF::DenseInfo* denseinfo = dense->mutable_denseinfo(); - - // iterate over all username string-ids - for (int i = 0, l=denseinfo->user_sid_size(); i 0 to real string ids - auto user_sid = string_table.map_string_id(denseinfo->user_sid(i)); - - // delta encode the string-id - denseinfo->set_user_sid(i, m_delta_user_sid.update(user_sid)); - } - } - } - } - - // test, if the ways-block has been allocated - if (pbf_ways) { - // iterate over all ways, passing them to the map_common_string_ids function - for (int i = 0, l=pbf_ways->ways_size(); imutable_ways(i)); - } - } + void add_node(const osmium::Node& node) { + m_ids.push_back(m_delta_id.update(node.id())); - // test, if the relations-block has been allocated - if (pbf_relations) { - // iterate over all relations - for (int i = 0, l=pbf_relations->relations_size(); imutable_relations(i); - - // pass them to the map_common_string_ids function - map_common_string_ids(relation); - - // iterate over all relation members, mapping the interim string-ids - // of the role to real string ids - for (int mi = 0; mi < relation->roles_sid_size(); ++mi) { - relation->set_roles_sid(mi, string_table.map_string_id(relation->roles_sid(mi))); - } + if (m_add_metadata) { + m_versions.push_back(node.version()); + m_timestamps.push_back(m_delta_timestamp.update(node.timestamp())); + m_changesets.push_back(m_delta_changeset.update(node.changeset())); + m_uids.push_back(m_delta_uid.update(node.uid())); + m_user_sids.push_back(m_delta_user_sid.update(m_stringtable.add(node.user()))); + if (m_add_visible) { + m_visibles.push_back(node.visible()); } } - } - /** - * a helper function used in map_string_ids to map common interim string-ids of the - * user name and all tags to real string ids. - * - * TPBFObject is either OSMPBF::Node, OSMPBF::Way or OSMPBF::Relation. - */ - template - void map_common_string_ids(TPBFObject* in) { - // if the object has meta-info attached - if (in->has_info()) { - // map the interim-id of the user name to a real id - OSMPBF::Info* info = in->mutable_info(); - info->set_user_sid(string_table.map_string_id(info->user_sid())); - } + m_lats.push_back(m_delta_lat.update(lonlat2int(node.location().lat_without_check()))); + m_lons.push_back(m_delta_lon.update(lonlat2int(node.location().lon_without_check()))); - // iterate over all tags and map the interim-ids of the key and the value to real ids - for (int i = 0, l=in->keys_size(); iset_keys(i, string_table.map_string_id(in->keys(i))); - in->set_vals(i, string_table.map_string_id(in->vals(i))); + for (const auto& tag : node.tags()) { + m_tags.push_back(m_stringtable.add(tag.key())); + m_tags.push_back(m_stringtable.add(tag.value())); } + m_tags.push_back(0); } + std::string serialize() const { + std::string data; + protozero::pbf_builder pbf_dense_nodes(data); - ///// MetaData helper ///// - - /** - * convert a double lat or lon value to an int, respecting the current blocks granularity - */ - int64_t lonlat2int(double lonlat) { - return static_cast(std::round(lonlat * OSMPBF::lonlat_resolution / location_granularity())); - } - - /** - * convert a timestamp to an int, respecting the current blocks granularity - */ - int64_t timestamp2int(time_t timestamp) { - return static_cast(std::round(timestamp * (1000.0 / date_granularity()))); - } + pbf_dense_nodes.add_packed_sint64(OSMFormat::DenseNodes::packed_sint64_id, m_ids.cbegin(), m_ids.cend()); - /** - * helper function used in the write()-calls to apply common information from an osmium-object - * onto a pbf-object. - * - * TPBFObject is either OSMPBF::Node, OSMPBF::Way or OSMPBF::Relation. - */ - template - void apply_common_info(const osmium::OSMObject& in, TPBFObject* out) { - // set the object-id - out->set_id(in.id()); - - // iterate over all tags and set the keys and vals, recording the strings in the - // interim StringTable and storing the interim ids - for (const auto& tag : in.tags()) { - out->add_keys(string_table.record_string(tag.key())); - out->add_vals(string_table.record_string(tag.value())); - } + if (m_add_metadata) { + protozero::pbf_builder pbf_dense_info(pbf_dense_nodes, OSMFormat::DenseNodes::optional_DenseInfo_denseinfo); + pbf_dense_info.add_packed_int32(OSMFormat::DenseInfo::packed_int32_version, m_versions.cbegin(), m_versions.cend()); + pbf_dense_info.add_packed_sint64(OSMFormat::DenseInfo::packed_sint64_timestamp, m_timestamps.cbegin(), m_timestamps.cend()); + pbf_dense_info.add_packed_sint64(OSMFormat::DenseInfo::packed_sint64_changeset, m_changesets.cbegin(), m_changesets.cend()); + pbf_dense_info.add_packed_sint32(OSMFormat::DenseInfo::packed_sint32_uid, m_uids.cbegin(), m_uids.cend()); + pbf_dense_info.add_packed_sint32(OSMFormat::DenseInfo::packed_sint32_user_sid, m_user_sids.cbegin(), m_user_sids.cend()); - if (m_should_add_metadata) { - // add an info-section to the pbf object and set the meta-info on it - OSMPBF::Info* out_info = out->mutable_info(); if (m_add_visible) { - out_info->set_visible(in.visible()); + pbf_dense_info.add_packed_bool(OSMFormat::DenseInfo::packed_bool_visible, m_visibles.cbegin(), m_visibles.cend()); } - out_info->set_version(static_cast<::google::protobuf::int32>(in.version())); - out_info->set_timestamp(timestamp2int(in.timestamp())); - out_info->set_changeset(in.changeset()); - out_info->set_uid(static_cast<::google::protobuf::int32>(in.uid())); - out_info->set_user_sid(string_table.record_string(in.user())); } + + pbf_dense_nodes.add_packed_sint64(OSMFormat::DenseNodes::packed_sint64_lat, m_lats.cbegin(), m_lats.cend()); + pbf_dense_nodes.add_packed_sint64(OSMFormat::DenseNodes::packed_sint64_lon, m_lons.cbegin(), m_lons.cend()); + + pbf_dense_nodes.add_packed_int32(OSMFormat::DenseNodes::packed_int32_keys_vals, m_tags.cbegin(), m_tags.cend()); + + return data; } + }; // class DenseNodes - ///// High-Level Block writing ///// + class PrimitiveBlock { - /** - * store the current pbf_header_block into a Blob and clear this struct afterwards. - */ - void store_header_block() { - if (debug && has_debug_level(1)) { - std::cerr << "storing header block" << std::endl; - } + std::string m_pbf_primitive_group_data; + protozero::pbf_builder m_pbf_primitive_group; + StringTable m_stringtable; + DenseNodes m_dense_nodes; + OSMFormat::PrimitiveGroup m_type; + int m_count; - std::promise promise; - m_output_queue.push(promise.get_future()); - promise.set_value(serialize_blob("OSMHeader", pbf_header_block, m_use_compression)); + public: - pbf_header_block.Clear(); + PrimitiveBlock(bool add_metadata, bool add_visible) : + m_pbf_primitive_group_data(), + m_pbf_primitive_group(m_pbf_primitive_group_data), + m_stringtable(), + m_dense_nodes(m_stringtable, add_metadata, add_visible), + m_type(OSMFormat::PrimitiveGroup::unknown), + m_count(0) { } - /** - * store the interim StringTable to the current pbf_primitive_block, map all interim string ids - * to real StringTable ids and then store the current pbf_primitive_block into a Blob and clear - * this struct and all related pointers and maps afterwards. - */ - void store_primitive_block() { - if (debug && has_debug_level(1)) { - std::cerr << "storing primitive block with " << primitive_block_contents << " items" << std::endl; + const std::string& group_data() { + if (type() == OSMFormat::PrimitiveGroup::optional_DenseNodes_dense) { + m_pbf_primitive_group.add_message(OSMFormat::PrimitiveGroup::optional_DenseNodes_dense, m_dense_nodes.serialize()); } + return m_pbf_primitive_group_data; + } - // set the granularity - pbf_primitive_block.set_granularity(location_granularity()); - pbf_primitive_block.set_date_granularity(date_granularity()); - - string_table.store_stringtable(pbf_primitive_block.mutable_stringtable(), m_sort_stringtables); + void reset(OSMFormat::PrimitiveGroup type) { + m_pbf_primitive_group_data.clear(); + m_stringtable.clear(); + m_dense_nodes.clear(); + m_type = type; + m_count = 0; + } - if (m_sort_stringtables) { - map_string_ids(); - } else { - delta_encode_string_ids(); + void write_stringtable(protozero::pbf_builder& pbf_string_table) { + for (const char* s : m_stringtable) { + pbf_string_table.add_bytes(OSMFormat::StringTable::repeated_bytes_s, s); } + } - std::promise promise; - m_output_queue.push(promise.get_future()); - promise.set_value(serialize_blob("OSMData", pbf_primitive_block, m_use_compression)); + protozero::pbf_builder& group() { + ++m_count; + return m_pbf_primitive_group; + } - // clear the PrimitiveBlock struct - pbf_primitive_block.Clear(); + void add_dense_node(const osmium::Node& node) { + m_dense_nodes.add_node(node); + ++m_count; + } - // clear the interim StringTable and its id map - string_table.clear(); + size_t add_string(const char* s) { + return m_stringtable.add(s); + } - // reset the delta variables - m_delta_id.clear(); - m_delta_lat.clear(); - m_delta_lon.clear(); - m_delta_timestamp.clear(); - m_delta_changeset.clear(); - m_delta_uid.clear(); - m_delta_user_sid.clear(); + int count() const { + return m_count; + } - // reset the contents-counter to zero - primitive_block_contents = 0; - primitive_block_size = 0; + OSMFormat::PrimitiveGroup type() const { + return m_type; + } - // reset the node/way/relation pointers to nullptr - pbf_nodes = nullptr; - pbf_ways = nullptr; - pbf_relations = nullptr; + size_t size() const { + return m_pbf_primitive_group_data.size() + m_stringtable.size() + m_dense_nodes.size(); } /** - * this little function checks primitive_block_contents counter against its maximum and calls - * store_primitive_block to flush the block to the disk when it's reached. It's also responsible - * for increasing this counter. - * - * this function also checks the estimated size of the current block and calls store_primitive_block - * when the estimated size reaches buffer_fill_percent of the maximum uncompressed blob size. + * The output buffer (block) will be filled to about + * 95% and then written to disk. This leaves more than + * enough space for the string table (which typically + * needs about 0.1 to 0.3% of the block size). */ - void check_block_contents_counter() { - if (primitive_block_contents >= max_block_contents) { - store_primitive_block(); - } else if (primitive_block_size > OSMPBF::max_uncompressed_blob_size * buffer_fill_percent / 100) { - if (debug && has_debug_level(1)) { - std::cerr << "storing primitive_block with only " << primitive_block_contents << " items, because its ByteSize (" << primitive_block_size << ") reached " << - (static_cast(primitive_block_size) / static_cast(OSMPBF::max_uncompressed_blob_size) * 100.0) << "% of the maximum blob-size" << std::endl; - } + constexpr static size_t max_used_blob_size = max_uncompressed_blob_size * 95 / 100; - store_primitive_block(); + bool can_add(OSMFormat::PrimitiveGroup type) const { + if (type != m_type) { + return false; } - - ++primitive_block_contents; + if (count() >= max_entities_per_block) { + return false; + } + return size() < max_used_blob_size; } + }; // class PrimitiveBlock - ///// Block content writing ///// - - /** - * Add a node to the block. - * - * @param node The node to add. - */ - void write_node(const osmium::Node& node) { - // add a way to the group - OSMPBF::Node* pbf_node = pbf_nodes->add_nodes(); - - // copy the common meta-info from the osmium-object to the pbf-object - apply_common_info(node, pbf_node); + class PBFOutputFormat : public osmium::io::detail::OutputFormat, public osmium::handler::Handler { - // modify lat & lon to integers, respecting the block's granularity and copy - // the ints to the pbf-object - pbf_node->set_lon(lonlat2int(node.location().lon_without_check())); - pbf_node->set_lat(lonlat2int(node.location().lat_without_check())); - } + /// Should nodes be encoded in DenseNodes? + bool m_use_dense_nodes; /** - * Add a node to the block using DenseNodes. + * Should the PBF blobs contain zlib compressed data? * - * @param node The node to add. + * The zlib compression is optional, it's possible to store the + * blobs in raw format. Disabling the compression can improve + * the writing speed a little but the output will be 2x to 3x + * bigger. */ - void write_dense_node(const osmium::Node& node) { - // add a DenseNodes-Section to the PrimitiveGroup - OSMPBF::DenseNodes* dense = pbf_nodes->mutable_dense(); + bool m_use_compression; - // copy the id, delta encoded - dense->add_id(m_delta_id.update(node.id())); + /// Should metadata of objects be written? + bool m_add_metadata; - // copy the longitude, delta encoded - dense->add_lon(m_delta_lon.update(lonlat2int(node.location().lon_without_check()))); + /// Should the visible flag be added to objects? + bool m_add_visible; - // copy the latitude, delta encoded - dense->add_lat(m_delta_lat.update(lonlat2int(node.location().lat_without_check()))); + PrimitiveBlock m_primitive_block; - // in the densenodes structure keys and vals are encoded in an intermixed - // array, individual nodes are seperated by a value of 0 (0 in the StringTable - // is always unused) - // so for three nodes the keys_vals array may look like this: 3 5 2 1 0 0 8 5 - // the first node has two tags (3=>5 and 2=>1), the second node does not - // have any tags and the third node has a single tag (8=>5) - for (const auto& tag : node.tags()) { - dense->add_keys_vals(string_table.record_string(tag.key())); - dense->add_keys_vals(string_table.record_string(tag.value())); + void store_primitive_block() { + if (m_primitive_block.count() == 0) { + return; } - dense->add_keys_vals(0); - - if (m_should_add_metadata) { - // add a DenseInfo-Section to the PrimitiveGroup - OSMPBF::DenseInfo* denseinfo = dense->mutable_denseinfo(); - - denseinfo->add_version(static_cast<::google::protobuf::int32>(node.version())); - - if (m_add_visible) { - denseinfo->add_visible(node.visible()); - } - - // copy the timestamp, delta encoded - denseinfo->add_timestamp(m_delta_timestamp.update(timestamp2int(node.timestamp()))); - // copy the changeset, delta encoded - denseinfo->add_changeset(m_delta_changeset.update(node.changeset())); + std::string primitive_block_data; + protozero::pbf_builder primitive_block(primitive_block_data); - // copy the user id, delta encoded - denseinfo->add_uid(static_cast<::google::protobuf::int32>(m_delta_uid.update(node.uid()))); - - // record the user-name to the interim stringtable and copy the - // interim string-id to the pbf-object - denseinfo->add_user_sid(string_table.record_string(node.user())); + { + protozero::pbf_builder pbf_string_table(primitive_block, OSMFormat::PrimitiveBlock::required_StringTable_stringtable); + m_primitive_block.write_stringtable(pbf_string_table); } - } - - /** - * Add a way to the block. - * - * @param way The way to add. - */ - void write_way(const osmium::Way& way) { - // add a way to the group - OSMPBF::Way* pbf_way = pbf_ways->add_ways(); - - // copy the common meta-info from the osmium-object to the pbf-object - apply_common_info(way, pbf_way); - // last way-node-id used for delta-encoding - Delta delta_id; + primitive_block.add_message(OSMFormat::PrimitiveBlock::repeated_PrimitiveGroup_primitivegroup, m_primitive_block.group_data()); - for (const auto& node_ref : way.nodes()) { - // copy the way-node-id, delta encoded - pbf_way->add_refs(delta_id.update(node_ref.ref())); - } - - // count up blob size by the size of the Way - primitive_block_size += pbf_way->ByteSize(); + std::promise promise; + m_output_queue.push(promise.get_future()); + promise.set_value(serialize_blob("OSMData", primitive_block_data, m_use_compression)); } - /** - * Add a relation to the block. - * - * @param relation The relation to add. - */ - void write_relation(const osmium::Relation& relation) { - // add a relation to the group - OSMPBF::Relation* pbf_relation = pbf_relations->add_relations(); - - // copy the common meta-info from the osmium-object to the pbf-object - apply_common_info(relation, pbf_relation); - - Delta delta_id; - - for (const auto& member : relation.members()) { - // record the relation-member role to the interim stringtable and copy the - // interim string-id to the pbf-object - pbf_relation->add_roles_sid(string_table.record_string(member.role())); - - // copy the relation-member-id, delta encoded - pbf_relation->add_memids(delta_id.update(member.ref())); - - // copy the relation-member-type, mapped to the OSMPBF enum - pbf_relation->add_types(item_type_to_osmpbf_membertype(member.type())); + template + void add_meta(const osmium::OSMObject& object, T& pbf_object) { + const osmium::TagList& tags = object.tags(); + + auto map_tag_key = [this](const osmium::Tag& tag) -> size_t { + return m_primitive_block.add_string(tag.key()); + }; + auto map_tag_value = [this](const osmium::Tag& tag) -> size_t { + return m_primitive_block.add_string(tag.value()); + }; + + pbf_object.add_packed_uint32(T::enum_type::packed_uint32_keys, + boost::make_transform_iterator(tags.begin(), map_tag_key), + boost::make_transform_iterator(tags.end(), map_tag_key)); + + pbf_object.add_packed_uint32(T::enum_type::packed_uint32_vals, + boost::make_transform_iterator(tags.begin(), map_tag_value), + boost::make_transform_iterator(tags.end(), map_tag_value)); + + if (m_add_metadata) { + protozero::pbf_builder pbf_info(pbf_object, T::enum_type::optional_Info_info); + + pbf_info.add_int32(OSMFormat::Info::optional_int32_version, object.version()); + pbf_info.add_int64(OSMFormat::Info::optional_int64_timestamp, object.timestamp()); + pbf_info.add_int64(OSMFormat::Info::optional_int64_changeset, object.changeset()); + pbf_info.add_int32(OSMFormat::Info::optional_int32_uid, object.uid()); + pbf_info.add_uint32(OSMFormat::Info::optional_uint32_user_sid, m_primitive_block.add_string(object.user())); + if (m_add_visible) { + pbf_info.add_bool(OSMFormat::Info::optional_bool_visible, object.visible()); + } } - - // count up blob size by the size of the Relation - primitive_block_size += pbf_relation->ByteSize(); } - // objects of this class can't be copied PBFOutputFormat(const PBFOutputFormat&) = delete; PBFOutputFormat& operator=(const PBFOutputFormat&) = delete; public: - /** - * Create PBFOutputFormat object from File. - */ explicit PBFOutputFormat(const osmium::io::File& file, data_queue_type& output_queue) : OutputFormat(file, output_queue), - pbf_header_block(), - pbf_primitive_block(), - pbf_nodes(nullptr), - pbf_ways(nullptr), - pbf_relations(nullptr), - m_location_granularity(pbf_primitive_block.granularity()), - m_date_granularity(pbf_primitive_block.date_granularity()), + m_use_dense_nodes(file.get("pbf_dense_nodes") != "false"), + m_use_compression(file.get("pbf_compression") != "none" && file.get("pbf_compression") != "false"), + m_add_metadata(file.get("pbf_add_metadata") != "false" && file.get("add_metadata") != "false"), m_add_visible(file.has_multiple_object_versions()), - primitive_block_contents(0), - primitive_block_size(0), - string_table(), - m_delta_id(), - m_delta_lat(), - m_delta_lon(), - m_delta_timestamp(), - m_delta_changeset(), - m_delta_uid(), - m_delta_user_sid(), - debug(true) { - GOOGLE_PROTOBUF_VERIFY_VERSION; - if (file.get("pbf_dense_nodes") == "false") { - m_use_dense_nodes = false; - } - if (file.get("pbf_compression") == "none" || file.get("pbf_compression") == "false") { - m_use_compression = false; - } - if (file.get("pbf_sort_stringtables") == "false") { - m_sort_stringtables = false; - } - if (file.get("pbf_add_metadata") == "false") { - m_should_add_metadata = false; - } + m_primitive_block(m_add_metadata, m_add_visible) { } void write_buffer(osmium::memory::Buffer&& buffer) override final { osmium::apply(buffer.cbegin(), buffer.cend(), *this); } + void write_header(const osmium::io::Header& header) override final { + std::string data; + protozero::pbf_builder pbf_header_block(data); - /** - * getter to access the granularity - */ - int location_granularity() const { - return m_location_granularity; - } - - /** - * setter to set the granularity - */ - PBFOutputFormat& location_granularity(int g) { - m_location_granularity = g; - return *this; - } - - - /** - * getter to access the date_granularity - */ - int date_granularity() const { - return m_date_granularity; - } - - /** - * Set date granularity. - */ - PBFOutputFormat& date_granularity(int g) { - m_date_granularity = g; - return *this; - } + if (!header.boxes().empty()) { + protozero::pbf_builder pbf_header_bbox(pbf_header_block, OSMFormat::HeaderBlock::optional_HeaderBBox_bbox); + osmium::Box box = header.joined_boxes(); + pbf_header_bbox.add_sint64(OSMFormat::HeaderBBox::required_sint64_left, box.bottom_left().lon() * lonlat_resolution); + pbf_header_bbox.add_sint64(OSMFormat::HeaderBBox::required_sint64_right, box.top_right().lon() * lonlat_resolution); + pbf_header_bbox.add_sint64(OSMFormat::HeaderBBox::required_sint64_top, box.top_right().lat() * lonlat_resolution); + pbf_header_bbox.add_sint64(OSMFormat::HeaderBBox::required_sint64_bottom, box.bottom_left().lat() * lonlat_resolution); + } - /** - * Initialize the writing process. - * - * This initializes the header-block, sets the required-features and - * the writing-program and adds the obligatory StringTable-Index 0. - */ - void write_header(const osmium::io::Header& header) override final { - // add the schema version as required feature to the HeaderBlock - pbf_header_block.add_required_features("OsmSchema-V0.6"); + pbf_header_block.add_string(OSMFormat::HeaderBlock::repeated_string_required_features, "OsmSchema-V0.6"); - // when the densenodes-feature is used, add DenseNodes as required feature if (m_use_dense_nodes) { - pbf_header_block.add_required_features("DenseNodes"); + pbf_header_block.add_string(OSMFormat::HeaderBlock::repeated_string_required_features, "DenseNodes"); } - // when the resulting file will carry history information, add - // HistoricalInformation as required feature if (m_file.has_multiple_object_versions()) { - pbf_header_block.add_required_features("HistoricalInformation"); + pbf_header_block.add_string(OSMFormat::HeaderBlock::repeated_string_required_features, "HistoricalInformation"); } - // set the writing program - pbf_header_block.set_writingprogram(header.get("generator")); - - if (!header.boxes().empty()) { - OSMPBF::HeaderBBox* pbf_bbox = pbf_header_block.mutable_bbox(); - osmium::Box box = header.joined_boxes(); - pbf_bbox->set_left(static_cast<::google::protobuf::int64>(box.bottom_left().lon() * OSMPBF::lonlat_resolution)); - pbf_bbox->set_bottom(static_cast<::google::protobuf::int64>(box.bottom_left().lat() * OSMPBF::lonlat_resolution)); - pbf_bbox->set_right(static_cast<::google::protobuf::int64>(box.top_right().lon() * OSMPBF::lonlat_resolution)); - pbf_bbox->set_top(static_cast<::google::protobuf::int64>(box.top_right().lat() * OSMPBF::lonlat_resolution)); - } + pbf_header_block.add_string(OSMFormat::HeaderBlock::optional_string_writingprogram, header.get("generator")); std::string osmosis_replication_timestamp = header.get("osmosis_replication_timestamp"); if (!osmosis_replication_timestamp.empty()) { osmium::Timestamp ts(osmosis_replication_timestamp.c_str()); - pbf_header_block.set_osmosis_replication_timestamp(ts); + pbf_header_block.add_int64(OSMFormat::HeaderBlock::optional_int64_osmosis_replication_timestamp, ts); } std::string osmosis_replication_sequence_number = header.get("osmosis_replication_sequence_number"); if (!osmosis_replication_sequence_number.empty()) { - pbf_header_block.set_osmosis_replication_sequence_number(std::atoll(osmosis_replication_sequence_number.c_str())); + pbf_header_block.add_int64(OSMFormat::HeaderBlock::optional_int64_osmosis_replication_sequence_number, std::atoll(osmosis_replication_sequence_number.c_str())); } std::string osmosis_replication_base_url = header.get("osmosis_replication_base_url"); if (!osmosis_replication_base_url.empty()) { - pbf_header_block.set_osmosis_replication_base_url(osmosis_replication_base_url); + pbf_header_block.add_string(OSMFormat::HeaderBlock::optional_string_osmosis_replication_base_url, osmosis_replication_base_url); } - store_header_block(); + std::promise promise; + m_output_queue.push(promise.get_future()); + promise.set_value(serialize_blob("OSMHeader", data, m_use_compression)); } - /** - * Add a node to the pbf. - * - * A call to this method won't write the node to the file directly but - * cache it for later bulk-writing. Calling final() ensures that everything - * gets written and every file pointer is closed. - */ - void node(const osmium::Node& node) { - // first of we check the contents-counter which may flush the cached nodes to - // disk if the limit is reached. This call also increases the contents-counter - check_block_contents_counter(); - - if (debug && has_debug_level(2)) { - std::cerr << "node " << node.id() << " v" << node.version() << std::endl; - } - - // if no PrimitiveGroup for nodes has been added, add one and save the pointer - if (!pbf_nodes) { - pbf_nodes = pbf_primitive_block.add_primitivegroup(); + void switch_primitive_block_type(OSMFormat::PrimitiveGroup type) { + if (!m_primitive_block.can_add(type)) { + store_primitive_block(); + m_primitive_block.reset(type); } + } + void node(const osmium::Node& node) { if (m_use_dense_nodes) { - write_dense_node(node); - } else { - write_node(node); + switch_primitive_block_type(OSMFormat::PrimitiveGroup::optional_DenseNodes_dense); + m_primitive_block.add_dense_node(node); + return; } + + switch_primitive_block_type(OSMFormat::PrimitiveGroup::repeated_Node_nodes); + protozero::pbf_builder pbf_node{ m_primitive_block.group(), OSMFormat::PrimitiveGroup::repeated_Node_nodes }; + + pbf_node.add_sint64(OSMFormat::Node::required_sint64_id, node.id()); + add_meta(node, pbf_node); + + pbf_node.add_sint64(OSMFormat::Node::required_sint64_lat, lonlat2int(node.location().lat_without_check())); + pbf_node.add_sint64(OSMFormat::Node::required_sint64_lon, lonlat2int(node.location().lon_without_check())); } - /** - * Add a way to the pbf. - * - * A call to this method won't write the way to the file directly but - * cache it for later bulk-writing. Calling final() ensures that everything - * gets written and every file pointer is closed. - */ void way(const osmium::Way& way) { - // first of we check the contents-counter which may flush the cached ways to - // disk if the limit is reached. This call also increases the contents-counter - check_block_contents_counter(); + switch_primitive_block_type(OSMFormat::PrimitiveGroup::repeated_Way_ways); + protozero::pbf_builder pbf_way{ m_primitive_block.group(), OSMFormat::PrimitiveGroup::repeated_Way_ways }; - // if no PrimitiveGroup for nodes has been added, add one and save the pointer - if (!pbf_ways) { - pbf_ways = pbf_primitive_block.add_primitivegroup(); - } + pbf_way.add_int64(OSMFormat::Way::required_int64_id, way.id()); + add_meta(way, pbf_way); + + static auto map_node_ref = [](osmium::NodeRefList::const_iterator node_ref) noexcept -> osmium::object_id_type { + return node_ref->ref(); + }; + typedef osmium::util::DeltaEncodeIterator it_type; - write_way(way); + const auto& nodes = way.nodes(); + it_type first { nodes.cbegin(), nodes.cend(), map_node_ref }; + it_type last { nodes.cend(), nodes.cend(), map_node_ref }; + pbf_way.add_packed_sint64(OSMFormat::Way::packed_sint64_refs, first, last); } - /** - * Add a relation to the pbf. - * - * A call to this method won't write the way to the file directly but - * cache it for later bulk-writing. Calling final() ensures that everything - * gets written and every file pointer is closed. - */ void relation(const osmium::Relation& relation) { - // first of we check the contents-counter which may flush the cached relations to - // disk if the limit is reached. This call also increases the contents-counter - check_block_contents_counter(); - - // if no PrimitiveGroup for relations has been added, add one and save the pointer - if (!pbf_relations) { - pbf_relations = pbf_primitive_block.add_primitivegroup(); - } - - write_relation(relation); + switch_primitive_block_type(OSMFormat::PrimitiveGroup::repeated_Relation_relations); + protozero::pbf_builder pbf_relation { m_primitive_block.group(), OSMFormat::PrimitiveGroup::repeated_Relation_relations }; + + pbf_relation.add_int64(OSMFormat::Relation::required_int64_id, relation.id()); + add_meta(relation, pbf_relation); + + auto map_member_role = [this](const osmium::RelationMember& member) -> size_t { + return m_primitive_block.add_string(member.role()); + }; + pbf_relation.add_packed_int32(OSMFormat::Relation::packed_int32_roles_sid, + boost::make_transform_iterator(relation.members().begin(), map_member_role), + boost::make_transform_iterator(relation.members().end(), map_member_role)); + + static auto map_member_ref = [](osmium::RelationMemberList::const_iterator member) noexcept -> osmium::object_id_type { + return member->ref(); + }; + typedef osmium::util::DeltaEncodeIterator it_type; + const auto& members = relation.members(); + it_type first { members.cbegin(), members.cend(), map_member_ref }; + it_type last { members.cend(), members.cend(), map_member_ref }; + pbf_relation.add_packed_sint64(OSMFormat::Relation::packed_sint64_memids, first, last); + + static auto map_member_type = [](const osmium::RelationMember& member) noexcept -> int { + return osmium::item_type_to_nwr_index(member.type()); + }; + pbf_relation.add_packed_int32(OSMFormat::Relation::packed_MemberType_types, + boost::make_transform_iterator(relation.members().begin(), map_member_type), + boost::make_transform_iterator(relation.members().end(), map_member_type)); } /** - * Finalize the writing process, flush any open primitive blocks to the file and - * close the file. + * Finalize the writing process, flush any open primitive + * blocks to the file and close the file. */ void close() override final { - if (debug && has_debug_level(1)) { - std::cerr << "finishing" << std::endl; - } - - // if the current block contains any elements, flush it to the protobuf - if (primitive_block_contents > 0) { - store_primitive_block(); - } + store_primitive_block(); std::promise promise; m_output_queue.push(promise.get_future()); @@ -956,10 +578,15 @@ namespace osmium { namespace { +// we want the register_output_format() function to run, setting the variable +// is only a side-effect, it will never be used +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wunused-variable" const bool registered_pbf_output = osmium::io::detail::OutputFormatFactory::instance().register_output_format(osmium::io::file_format::pbf, [](const osmium::io::File& file, data_queue_type& output_queue) { return new osmium::io::detail::PBFOutputFormat(file, output_queue); }); +#pragma GCC diagnostic pop } // anonymous namespace diff --git a/include/osmium/io/detail/pbf_parser.hpp b/include/osmium/io/detail/pbf_parser.hpp deleted file mode 100644 index 65a11e15581..00000000000 --- a/include/osmium/io/detail/pbf_parser.hpp +++ /dev/null @@ -1,455 +0,0 @@ -#ifndef OSMIUM_IO_DETAIL_PBF_PRIMITIVE_BLOCK_PARSER_HPP -#define OSMIUM_IO_DETAIL_PBF_PRIMITIVE_BLOCK_PARSER_HPP - -/* - -This file is part of Osmium (http://osmcode.org/libosmium). - -Copyright 2013-2015 Jochen Topf and others (see README). - -Boost Software License - Version 1.0 - August 17th, 2003 - -Permission is hereby granted, free of charge, to any person or organization -obtaining a copy of the software and accompanying documentation covered by -this license (the "Software") to use, reproduce, display, distribute, -execute, and transmit the Software, and to prepare derivative works of the -Software, and to permit third-parties to whom the Software is furnished to -do so, all subject to the following: - -The copyright notices in the Software and this entire statement, including -the above license grant, this restriction and the following disclaimer, -must be included in all copies of the Software, in whole or in part, and -all derivative works of the Software, unless such copies or derivative -works are solely in the form of machine-executable object code generated by -a source language processor. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT -SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE -FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, -ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER -DEALINGS IN THE SOFTWARE. - -*/ - -#include -#include -#include -#include - -#include - -#include -#include // IWYU pragma: export -#include -#include -#include -#include -#include -#include -#include -#include - -namespace osmium { - - namespace io { - - namespace detail { - - class PBFPrimitiveBlockParser { - - static constexpr size_t initial_buffer_size = 2 * 1024 * 1024; - - const std::string& m_data; - - const OSMPBF::StringTable* m_stringtable; - int64_t m_lon_offset; - int64_t m_lat_offset; - int64_t m_date_factor; - int32_t m_granularity; - - osmium::osm_entity_bits::type m_read_types; - - osmium::memory::Buffer m_buffer; - - PBFPrimitiveBlockParser(const PBFPrimitiveBlockParser&) = delete; - PBFPrimitiveBlockParser(PBFPrimitiveBlockParser&&) = delete; - - PBFPrimitiveBlockParser& operator=(const PBFPrimitiveBlockParser&) = delete; - PBFPrimitiveBlockParser& operator=(PBFPrimitiveBlockParser&&) = delete; - - public: - - explicit PBFPrimitiveBlockParser(const std::string& data, osmium::osm_entity_bits::type read_types) : - m_data(data), - m_stringtable(nullptr), - m_lon_offset(0), - m_lat_offset(0), - m_date_factor(1000), - m_granularity(100), - m_read_types(read_types), - m_buffer(initial_buffer_size) { - } - - ~PBFPrimitiveBlockParser() = default; - - osmium::memory::Buffer operator()() { - OSMPBF::PrimitiveBlock pbf_primitive_block; - if (!pbf_primitive_block.ParseFromString(m_data)) { - throw osmium::pbf_error("failed to parse PrimitiveBlock"); - } - - m_stringtable = &pbf_primitive_block.stringtable(); - m_lon_offset = pbf_primitive_block.lon_offset(); - m_lat_offset = pbf_primitive_block.lat_offset(); - m_date_factor = pbf_primitive_block.date_granularity() / 1000; - m_granularity = pbf_primitive_block.granularity(); - - for (int i = 0; i < pbf_primitive_block.primitivegroup_size(); ++i) { - const OSMPBF::PrimitiveGroup& group = pbf_primitive_block.primitivegroup(i); - - if (group.has_dense()) { - if (m_read_types & osmium::osm_entity_bits::node) parse_dense_node_group(group); - } else if (group.ways_size() != 0) { - if (m_read_types & osmium::osm_entity_bits::way) parse_way_group(group); - } else if (group.relations_size() != 0) { - if (m_read_types & osmium::osm_entity_bits::relation) parse_relation_group(group); - } else if (group.nodes_size() != 0) { - if (m_read_types & osmium::osm_entity_bits::node) parse_node_group(group); - } else { - throw osmium::pbf_error("group of unknown type"); - } - } - - return std::move(m_buffer); - } - - private: - - template - void parse_attributes(TBuilder& builder, const TPBFObject& pbf_object) { - auto& object = builder.object(); - - object.set_id(pbf_object.id()); - - if (pbf_object.has_info()) { - object.set_version(static_cast_with_assert(pbf_object.info().version())) - .set_changeset(static_cast_with_assert(pbf_object.info().changeset())) - .set_timestamp(pbf_object.info().timestamp() * m_date_factor) - .set_uid_from_signed(pbf_object.info().uid()); - if (pbf_object.info().has_visible()) { - object.set_visible(pbf_object.info().visible()); - } - builder.add_user(m_stringtable->s(static_cast_with_assert(pbf_object.info().user_sid()))); - } else { - builder.add_user("", 1); - } - } - - void parse_node_group(const OSMPBF::PrimitiveGroup& group) { - for (int i = 0; i < group.nodes_size(); ++i) { - osmium::builder::NodeBuilder builder(m_buffer); - const OSMPBF::Node& pbf_node = group.nodes(i); - parse_attributes(builder, pbf_node); - - if (builder.object().visible()) { - builder.object().set_location(osmium::Location( - (pbf_node.lon() * m_granularity + m_lon_offset) / (OSMPBF::lonlat_resolution / osmium::Location::coordinate_precision), - (pbf_node.lat() * m_granularity + m_lat_offset) / (OSMPBF::lonlat_resolution / osmium::Location::coordinate_precision))); - } - - if (pbf_node.keys_size() > 0) { - osmium::builder::TagListBuilder tl_builder(m_buffer, &builder); - for (int tag = 0; tag < pbf_node.keys_size(); ++tag) { - tl_builder.add_tag(m_stringtable->s(static_cast(pbf_node.keys(tag))), - m_stringtable->s(static_cast(pbf_node.vals(tag)))); - } - } - - m_buffer.commit(); - } - } - - void parse_way_group(const OSMPBF::PrimitiveGroup& group) { - for (int i = 0; i < group.ways_size(); ++i) { - osmium::builder::WayBuilder builder(m_buffer); - const OSMPBF::Way& pbf_way = group.ways(i); - parse_attributes(builder, pbf_way); - - if (pbf_way.refs_size() > 0) { - osmium::builder::WayNodeListBuilder wnl_builder(m_buffer, &builder); - int64_t ref = 0; - for (int n = 0; n < pbf_way.refs_size(); ++n) { - ref += pbf_way.refs(n); - wnl_builder.add_node_ref(ref); - } - } - - if (pbf_way.keys_size() > 0) { - osmium::builder::TagListBuilder tl_builder(m_buffer, &builder); - for (int tag = 0; tag < pbf_way.keys_size(); ++tag) { - tl_builder.add_tag(m_stringtable->s(static_cast(pbf_way.keys(tag))), - m_stringtable->s(static_cast(pbf_way.vals(tag)))); - } - } - - m_buffer.commit(); - } - } - - void parse_relation_group(const OSMPBF::PrimitiveGroup& group) { - for (int i = 0; i < group.relations_size(); ++i) { - osmium::builder::RelationBuilder builder(m_buffer); - const OSMPBF::Relation& pbf_relation = group.relations(i); - parse_attributes(builder, pbf_relation); - - if (pbf_relation.types_size() > 0) { - osmium::builder::RelationMemberListBuilder rml_builder(m_buffer, &builder); - int64_t ref = 0; - for (int n = 0; n < pbf_relation.types_size(); ++n) { - ref += pbf_relation.memids(n); - rml_builder.add_member(osmpbf_membertype_to_item_type(pbf_relation.types(n)), ref, m_stringtable->s(pbf_relation.roles_sid(n))); - } - } - - if (pbf_relation.keys_size() > 0) { - osmium::builder::TagListBuilder tl_builder(m_buffer, &builder); - for (int tag = 0; tag < pbf_relation.keys_size(); ++tag) { - tl_builder.add_tag(m_stringtable->s(static_cast(pbf_relation.keys(tag))), - m_stringtable->s(static_cast(pbf_relation.vals(tag)))); - } - } - - m_buffer.commit(); - } - } - - int add_tags(const OSMPBF::DenseNodes& dense, int n, osmium::builder::NodeBuilder* builder) { - if (n >= dense.keys_vals_size()) { - return n; - } - - if (dense.keys_vals(n) == 0) { - return n+1; - } - - osmium::builder::TagListBuilder tl_builder(m_buffer, builder); - - while (n < dense.keys_vals_size()) { - int tag_key_pos = dense.keys_vals(n++); - - if (tag_key_pos == 0) { - break; - } - - tl_builder.add_tag(m_stringtable->s(tag_key_pos), - m_stringtable->s(dense.keys_vals(n))); - - ++n; - } - - return n; - } - - void parse_dense_node_group(const OSMPBF::PrimitiveGroup& group) { - int64_t last_dense_id = 0; - int64_t last_dense_latitude = 0; - int64_t last_dense_longitude = 0; - int64_t last_dense_uid = 0; - int64_t last_dense_user_sid = 0; - int64_t last_dense_changeset = 0; - int64_t last_dense_timestamp = 0; - int last_dense_tag = 0; - - const OSMPBF::DenseNodes& dense = group.dense(); - - for (int i = 0; i < dense.id_size(); ++i) { - bool visible = true; - - last_dense_id += dense.id(i); - last_dense_latitude += dense.lat(i); - last_dense_longitude += dense.lon(i); - - if (dense.has_denseinfo()) { - last_dense_changeset += dense.denseinfo().changeset(i); - last_dense_timestamp += dense.denseinfo().timestamp(i); - last_dense_uid += dense.denseinfo().uid(i); - last_dense_user_sid += dense.denseinfo().user_sid(i); - if (dense.denseinfo().visible_size() > 0) { - visible = dense.denseinfo().visible(i); - } - assert(last_dense_changeset >= 0); - assert(last_dense_timestamp >= 0); - assert(last_dense_uid >= -1); - assert(last_dense_user_sid >= 0); - } - - osmium::builder::NodeBuilder builder(m_buffer); - osmium::Node& node = builder.object(); - - node.set_id(last_dense_id); - - if (dense.has_denseinfo()) { - auto v = dense.denseinfo().version(i); - assert(v > 0); - node.set_version(static_cast(v)); - node.set_changeset(static_cast(last_dense_changeset)); - node.set_timestamp(last_dense_timestamp * m_date_factor); - node.set_uid_from_signed(static_cast(last_dense_uid)); - node.set_visible(visible); - builder.add_user(m_stringtable->s(static_cast(last_dense_user_sid))); - } else { - builder.add_user("", 1); - } - - if (visible) { - builder.object().set_location(osmium::Location( - (last_dense_longitude * m_granularity + m_lon_offset) / (OSMPBF::lonlat_resolution / osmium::Location::coordinate_precision), - (last_dense_latitude * m_granularity + m_lat_offset) / (OSMPBF::lonlat_resolution / osmium::Location::coordinate_precision))); - } - - last_dense_tag = add_tags(dense, last_dense_tag, &builder); - m_buffer.commit(); - } - } - - }; // class PBFPrimitiveBlockParser - - /** - * PBF blobs can optionally be packed with the zlib algorithm. - * This function returns the raw data (if it was unpacked) or - * the unpacked data (if it was packed). - * - * @param input_data Reference to input data. - * @returns Unpacked data - * @throws osmium::pbf_error If there was a problem parsing the PBF - */ - inline std::unique_ptr unpack_blob(const std::string& input_data) { - OSMPBF::Blob pbf_blob; - if (!pbf_blob.ParseFromString(input_data)) { - throw osmium::pbf_error("failed to parse blob"); - } - - if (pbf_blob.has_raw()) { - return std::unique_ptr(pbf_blob.release_raw()); - } else if (pbf_blob.has_zlib_data()) { - auto raw_size = pbf_blob.raw_size(); - assert(raw_size >= 0); - assert(raw_size <= OSMPBF::max_uncompressed_blob_size); - return osmium::io::detail::zlib_uncompress(pbf_blob.zlib_data(), static_cast(raw_size)); - } else if (pbf_blob.has_lzma_data()) { - throw osmium::pbf_error("lzma blobs not implemented"); - } else { - throw osmium::pbf_error("blob contains no data"); - } - } - - /** - * Parse blob as a HeaderBlock. - * - * @param input_buffer Blob data - * @returns Header object - * @throws osmium::pbf_error If there was a parsing error - */ - inline osmium::io::Header parse_header_blob(const std::string& input_buffer) { - const std::unique_ptr data = unpack_blob(input_buffer); - - OSMPBF::HeaderBlock pbf_header_block; - if (!pbf_header_block.ParseFromString(*data)) { - throw osmium::pbf_error("failed to parse HeaderBlock"); - } - - osmium::io::Header header; - for (int i = 0; i < pbf_header_block.required_features_size(); ++i) { - const std::string& feature = pbf_header_block.required_features(i); - - if (feature == "OsmSchema-V0.6") continue; - if (feature == "DenseNodes") { - header.set("pbf_dense_nodes", true); - continue; - } - if (feature == "HistoricalInformation") { - header.set_has_multiple_object_versions(true); - continue; - } - - throw osmium::pbf_error(std::string("required feature not supported: ") + feature); - } - - for (int i = 0; i < pbf_header_block.optional_features_size(); ++i) { - const std::string& feature = pbf_header_block.optional_features(i); - header.set("pbf_optional_feature_" + std::to_string(i), feature); - } - - if (pbf_header_block.has_writingprogram()) { - header.set("generator", pbf_header_block.writingprogram()); - } - - if (pbf_header_block.has_bbox()) { - const OSMPBF::HeaderBBox& pbf_bbox = pbf_header_block.bbox(); - const int64_t resolution_convert = OSMPBF::lonlat_resolution / osmium::Location::coordinate_precision; - osmium::Box box; - box.extend(osmium::Location(pbf_bbox.left() / resolution_convert, pbf_bbox.bottom() / resolution_convert)); - box.extend(osmium::Location(pbf_bbox.right() / resolution_convert, pbf_bbox.top() / resolution_convert)); - header.add_box(box); - } - - if (pbf_header_block.has_osmosis_replication_timestamp()) { - header.set("osmosis_replication_timestamp", osmium::Timestamp(pbf_header_block.osmosis_replication_timestamp()).to_iso()); - } - - if (pbf_header_block.has_osmosis_replication_sequence_number()) { - header.set("osmosis_replication_sequence_number", std::to_string(pbf_header_block.osmosis_replication_sequence_number())); - } - - if (pbf_header_block.has_osmosis_replication_base_url()) { - header.set("osmosis_replication_base_url", pbf_header_block.osmosis_replication_base_url()); - } - - return header; - } - - class DataBlobParser { - - std::shared_ptr m_input_buffer; - osmium::osm_entity_bits::type m_read_types; - - public: - - DataBlobParser(std::string&& input_buffer, osmium::osm_entity_bits::type read_types) : - m_input_buffer(std::make_shared(std::move(input_buffer))), - m_read_types(read_types) { - if (input_buffer.size() > OSMPBF::max_uncompressed_blob_size) { - throw osmium::pbf_error(std::string("invalid blob size: " + std::to_string(input_buffer.size()))); - } - } -/* - DataBlobParser(const DataBlobParser& other) : - m_input_buffer(std::move(other.m_input_buffer)), - m_read_types(other.m_read_types) { - }*/ - - DataBlobParser(const DataBlobParser&) = default; - DataBlobParser& operator=(const DataBlobParser&) = default; - - DataBlobParser(DataBlobParser&&) = default; - DataBlobParser& operator=(DataBlobParser&&) = default; - - ~DataBlobParser() = default; - - osmium::memory::Buffer operator()() { - const std::unique_ptr data = unpack_blob(*m_input_buffer); - PBFPrimitiveBlockParser parser(*data, m_read_types); - return parser(); - } - - }; // class DataBlobParser - - } // namespace detail - - } // namespace io - -} // namespace osmium - -#endif // OSMIUM_IO_DETAIL_PBF_PRIMITIVE_BLOCK_PARSER_HPP diff --git a/include/osmium/io/detail/pbf_stringtable.hpp b/include/osmium/io/detail/pbf_stringtable.hpp deleted file mode 100644 index 5f540f1271b..00000000000 --- a/include/osmium/io/detail/pbf_stringtable.hpp +++ /dev/null @@ -1,218 +0,0 @@ -#ifndef OSMIUM_IO_DETAIL_PBF_STRINGTABLE_HPP -#define OSMIUM_IO_DETAIL_PBF_STRINGTABLE_HPP - -/* - -This file is part of Osmium (http://osmcode.org/libosmium). - -Copyright 2013-2015 Jochen Topf and others (see README). - -Boost Software License - Version 1.0 - August 17th, 2003 - -Permission is hereby granted, free of charge, to any person or organization -obtaining a copy of the software and accompanying documentation covered by -this license (the "Software") to use, reproduce, display, distribute, -execute, and transmit the Software, and to prepare derivative works of the -Software, and to permit third-parties to whom the Software is furnished to -do so, all subject to the following: - -The copyright notices in the Software and this entire statement, including -the above license grant, this restriction and the following disclaimer, -must be included in all copies of the Software, in whole or in part, and -all derivative works of the Software, unless such copies or derivative -works are solely in the form of machine-executable object code generated by -a source language processor. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT -SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE -FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, -ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER -DEALINGS IN THE SOFTWARE. - -*/ - -#include -#include -#include -#include -#include -#include -#include - -#include - -#include - -namespace osmium { - - namespace io { - - namespace detail { - - /** - * StringTable management for PBF writer - * - * All strings are stored as indexes to rows in a StringTable. The StringTable contains - * one row for each used string, so strings that are used multiple times need to be - * stored only once. The StringTable is sorted by usage-count, so the most often used - * string is stored at index 1. - */ - class StringTable { - - public: - - /// type for string IDs (interim and final) - typedef uint16_t string_id_type; - - private: - - /** - * this is the struct used to build the StringTable. It is stored as - * the value-part in the strings-map. - * - * when a new string is added to the map, its count is set to 0 and - * the interim_id is set to the current size of the map. This interim_id - * is then stored into the pbf-objects. - * - * before the PrimitiveBlock is serialized, the map is sorted by count - * and stored into the pbf-StringTable. Afterwards the interim-ids are - * mapped to the "real" id in the StringTable. - * - * this way often used strings get lower ids in the StringTable. As the - * protobuf-serializer stores numbers in variable bit-lengths, lower - * IDs means less used space in the resulting file. - */ - struct string_info { - - /// number of occurrences of this string - uint16_t count; - - /// an intermediate-id - string_id_type interim_id; - - }; // struct string_info - - /** - * Interim StringTable, storing all strings that should be written to - * the StringTable once the block is written to disk. - */ - typedef std::map string2string_info_type; - string2string_info_type m_strings; - - /** - * This vector is used to map the interim IDs to real StringTable IDs after - * writing all strings to the StringTable. - */ - typedef std::vector interim_id2id_type; - interim_id2id_type m_id2id_map; - - size_t m_size = 0; - - public: - - StringTable() { - } - - friend bool operator<(const string_info& lhs, const string_info& rhs) { - return lhs.count > rhs.count; - } - - /** - * record a string in the interim StringTable if it's missing, otherwise just increase its counter, - * return the interim-id assigned to the string. - */ - string_id_type record_string(const std::string& string) { - string_info& info = m_strings[string]; - if (info.interim_id == 0) { - ++m_size; - info.interim_id = static_cast_with_assert(m_size); - } else { - info.count++; - } - return info.interim_id; - } - - /** - * Sort the interim StringTable and store it to the real protobuf StringTable. - * while storing to the real table, this function fills the id2id_map with - * pairs, mapping the interim-ids to final and real StringTable ids. - * - * Note that the m_strings table is a std::map and as such is sorted lexicographically. - * When the transformation into the sortedby multimap is done, it gets sorted by - * the count. The end result (at least with the glibc standard container/algorithm - * implementation) is that the string table is sorted first by reverse count (ie descending) - * and then by reverse lexicographic order. - */ - void store_stringtable(OSMPBF::StringTable* st, bool sort) { - // add empty StringTable entry at index 0 - // StringTable index 0 is reserved as delimiter in the densenodes key/value list - // this line also ensures that there's always a valid StringTable - st->add_s(""); - - if (sort) { - std::multimap sortedbycount; - - m_id2id_map.resize(m_size+1); - - std::transform(m_strings.begin(), m_strings.end(), - std::inserter(sortedbycount, sortedbycount.begin()), - [](const std::pair& p) { - return std::pair(p.second, p.first); - }); - - string_id_type n = 0; - - for (const auto& mapping : sortedbycount) { - // add the string of the current item to the pbf StringTable - st->add_s(mapping.second); - - // store the mapping from the interim-id to the real id - m_id2id_map[mapping.first.interim_id] = ++n; - } - } else { - std::vector> sortedbyid; - sortedbyid.reserve(m_strings.size()); - - for (const auto& p : m_strings) { - sortedbyid.emplace_back(p.second.interim_id, p.first.c_str()); - } - - std::sort(sortedbyid.begin(), sortedbyid.end()); - for (const auto& mapping : sortedbyid) { - st->add_s(mapping.second); - } - } - } - - /** - * Map from an interim ID to a real string ID. - */ - string_id_type map_string_id(const string_id_type interim_id) const { - return m_id2id_map[interim_id]; - } - - template - string_id_type map_string_id(const T interim_id) const { - return map_string_id(static_cast_with_assert(interim_id)); - } - - /** - * Clear the stringtable, preparing for the next block. - */ - void clear() { - m_strings.clear(); - m_id2id_map.clear(); - m_size = 0; - } - - }; // class StringTable - - } // namespace detail - - } // namespace io - -} // namespace osmium - -#endif // OSMIUM_IO_DETAIL_PBF_STRINGTABLE_HPP diff --git a/include/osmium/io/detail/protobuf_tags.hpp b/include/osmium/io/detail/protobuf_tags.hpp new file mode 100644 index 00000000000..3f230876639 --- /dev/null +++ b/include/osmium/io/detail/protobuf_tags.hpp @@ -0,0 +1,170 @@ +#ifndef OSMIUM_IO_DETAIL_PROTOBUF_TAGS_HPP +#define OSMIUM_IO_DETAIL_PROTOBUF_TAGS_HPP + +/* + +This file is part of Osmium (http://osmcode.org/libosmium). + +Copyright 2013-2015 Jochen Topf and others (see README). + +Boost Software License - Version 1.0 - August 17th, 2003 + +Permission is hereby granted, free of charge, to any person or organization +obtaining a copy of the software and accompanying documentation covered by +this license (the "Software") to use, reproduce, display, distribute, +execute, and transmit the Software, and to prepare derivative works of the +Software, and to permit third-parties to whom the Software is furnished to +do so, all subject to the following: + +The copyright notices in the Software and this entire statement, including +the above license grant, this restriction and the following disclaimer, +must be included in all copies of the Software, in whole or in part, and +all derivative works of the Software, unless such copies or derivative +works are solely in the form of machine-executable object code generated by +a source language processor. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT +SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE +FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, +ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. + +*/ + +#include + +namespace osmium { + + namespace io { + + namespace detail { + + // directly translated from + // https://github.com/scrosby/OSM-binary/blob/master/src/fileformat.proto + + namespace FileFormat { + + enum class Blob : protozero::pbf_tag_type { + optional_bytes_raw = 1, + optional_int32_raw_size = 2, + optional_bytes_zlib_data = 3, + optional_bytes_lzma_data = 4 + }; + + enum class BlobHeader : protozero::pbf_tag_type { + required_string_type = 1, + optional_bytes_indexdata = 2, + required_int32_datasize = 3 + }; + + } // namespace FileFormat + + // directly translated from + // https://github.com/scrosby/OSM-binary/blob/master/src/osmformat.proto + + namespace OSMFormat { + + enum class HeaderBlock : protozero::pbf_tag_type { + optional_HeaderBBox_bbox = 1, + repeated_string_required_features = 4, + repeated_string_optional_features = 5, + optional_string_writingprogram = 16, + optional_string_source = 17, + optional_int64_osmosis_replication_timestamp = 32, + optional_int64_osmosis_replication_sequence_number = 33, + optional_string_osmosis_replication_base_url = 34 + }; + + enum class HeaderBBox : protozero::pbf_tag_type { + required_sint64_left = 1, + required_sint64_right = 2, + required_sint64_top = 3, + required_sint64_bottom = 4 + }; + + enum class PrimitiveBlock : protozero::pbf_tag_type { + required_StringTable_stringtable = 1, + repeated_PrimitiveGroup_primitivegroup = 2, + optional_int32_granularity = 17, + optional_int32_date_granularity = 18, + optional_int64_lat_offset = 19, + optional_int64_lon_offset = 20 + }; + + enum class PrimitiveGroup : protozero::pbf_tag_type { + unknown = 0, + repeated_Node_nodes = 1, + optional_DenseNodes_dense = 2, + repeated_Way_ways = 3, + repeated_Relation_relations = 4, + repeated_ChangeSet_changesets = 5 + }; + + enum class StringTable : protozero::pbf_tag_type { + repeated_bytes_s = 1 + }; + + enum class Info : protozero::pbf_tag_type { + optional_int32_version = 1, + optional_int64_timestamp = 2, + optional_int64_changeset = 3, + optional_int32_uid = 4, + optional_uint32_user_sid = 5, + optional_bool_visible = 6 + }; + + enum class DenseInfo : protozero::pbf_tag_type { + packed_int32_version = 1, + packed_sint64_timestamp = 2, + packed_sint64_changeset = 3, + packed_sint32_uid = 4, + packed_sint32_user_sid = 5, + packed_bool_visible = 6 + }; + + enum class Node : protozero::pbf_tag_type { + required_sint64_id = 1, + packed_uint32_keys = 2, + packed_uint32_vals = 3, + optional_Info_info = 4, + required_sint64_lat = 8, + required_sint64_lon = 9 + }; + + enum class DenseNodes : protozero::pbf_tag_type { + packed_sint64_id = 1, + optional_DenseInfo_denseinfo = 5, + packed_sint64_lat = 8, + packed_sint64_lon = 9, + packed_int32_keys_vals = 10 + }; + + enum class Way : protozero::pbf_tag_type { + required_int64_id = 1, + packed_uint32_keys = 2, + packed_uint32_vals = 3, + optional_Info_info = 4, + packed_sint64_refs = 8 + }; + + enum class Relation : protozero::pbf_tag_type { + required_int64_id = 1, + packed_uint32_keys = 2, + packed_uint32_vals = 3, + optional_Info_info = 4, + packed_int32_roles_sid = 8, + packed_sint64_memids = 9, + packed_MemberType_types = 10 + }; + + } // namespace OSMFormat + + } // namespace detail + + } // namespace io + +} // namespace osmium + +#endif // OSMIUM_IO_DETAIL_PROTOBUF_TAGS_HPP diff --git a/include/osmium/io/detail/read_write.hpp b/include/osmium/io/detail/read_write.hpp index 6651cced9b7..9863bd719cc 100644 --- a/include/osmium/io/detail/read_write.hpp +++ b/include/osmium/io/detail/read_write.hpp @@ -122,7 +122,7 @@ namespace osmium { * @throws std::system_error On error. */ inline void reliable_write(const int fd, const unsigned char* output_buffer, const size_t size) { - constexpr size_t max_write = 100 * 1024 * 1024; // Max 100 MByte per write + constexpr size_t max_write = 100L * 1024L * 1024L; // Max 100 MByte per write size_t offset = 0; do { auto write_count = size - offset; diff --git a/include/osmium/io/detail/string_table.hpp b/include/osmium/io/detail/string_table.hpp new file mode 100644 index 00000000000..ae9d5f0ce3f --- /dev/null +++ b/include/osmium/io/detail/string_table.hpp @@ -0,0 +1,250 @@ +#ifndef OSMIUM_IO_DETAIL_STRING_TABLE_HPP +#define OSMIUM_IO_DETAIL_STRING_TABLE_HPP + +/* + +This file is part of Osmium (http://osmcode.org/libosmium). + +Copyright 2013-2015 Jochen Topf and others (see README). + +Boost Software License - Version 1.0 - August 17th, 2003 + +Permission is hereby granted, free of charge, to any person or organization +obtaining a copy of the software and accompanying documentation covered by +this license (the "Software") to use, reproduce, display, distribute, +execute, and transmit the Software, and to prepare derivative works of the +Software, and to permit third-parties to whom the Software is furnished to +do so, all subject to the following: + +The copyright notices in the Software and this entire statement, including +the above license grant, this restriction and the following disclaimer, +must be included in all copies of the Software, in whole or in part, and +all derivative works of the Software, unless such copies or derivative +works are solely in the form of machine-executable object code generated by +a source language processor. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT +SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE +FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, +ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. + +*/ + +#include +#include +#include +#include +#include +#include +#include + +namespace osmium { + + namespace io { + + namespace detail { + + /** + * class StringStore + * + * Storage of lots of strings (const char *). Memory is allocated in chunks. + * If a string is added and there is no space in the current chunk, a new + * chunk will be allocated. Strings added to the store must not be larger + * than the chunk size. + * + * All memory is released when the destructor is called. There is no other way + * to release all or part of the memory. + * + */ + class StringStore { + + size_t m_chunk_size; + + std::list m_chunks; + + void add_chunk() { + m_chunks.push_front(std::string()); + m_chunks.front().reserve(m_chunk_size); + } + + public: + + StringStore(size_t chunk_size) : + m_chunk_size(chunk_size), + m_chunks() { + add_chunk(); + } + + void clear() noexcept { + m_chunks.erase(std::next(m_chunks.begin()), m_chunks.end()); + m_chunks.front().clear(); + } + + /** + * Add a null terminated string to the store. This will + * automatically get more memory if we are out. + * Returns a pointer to the copy of the string we have + * allocated. + */ + const char* add(const char* string) { + size_t len = std::strlen(string) + 1; + + assert(len <= m_chunk_size); + + size_t chunk_len = m_chunks.front().size(); + if (chunk_len + len > m_chunks.front().capacity()) { + add_chunk(); + chunk_len = 0; + } + + m_chunks.front().append(string); + m_chunks.front().append(1, '\0'); + + return m_chunks.front().c_str() + chunk_len; + } + + class const_iterator : public std::iterator { + + typedef std::list::const_iterator it_type; + it_type m_it; + const it_type m_last; + const char* m_pos; + + public: + + const_iterator(it_type it, it_type last) : + m_it(it), + m_last(last), + m_pos(it == last ? nullptr : m_it->c_str()) { + } + + const_iterator& operator++() { + assert(m_it != m_last); + auto last_pos = m_it->c_str() + m_it->size(); + while (m_pos != last_pos && *m_pos) ++m_pos; + if (m_pos != last_pos) ++m_pos; + if (m_pos == last_pos) { + ++m_it; + if (m_it != m_last) { + m_pos = m_it->c_str(); + } else { + m_pos = nullptr; + } + } + return *this; + } + + const_iterator operator++(int) { + const_iterator tmp(*this); + operator++(); + return tmp; + } + + bool operator==(const const_iterator& rhs) const { + return m_it == rhs.m_it && m_pos == rhs.m_pos; + } + + bool operator!=(const const_iterator& rhs) const { + return !(*this == rhs); + } + + const char* operator*() const { + assert(m_it != m_last); + assert(m_pos != nullptr); + return m_pos; + } + + }; // class const_iterator + + const_iterator begin() const { + if (m_chunks.front().empty()) { + return end(); + } + return const_iterator(m_chunks.begin(), m_chunks.end()); + } + + const_iterator end() const { + return const_iterator(m_chunks.end(), m_chunks.end()); + } + + // These functions get you some idea how much memory was + // used. + int get_chunk_size() const noexcept { + return m_chunk_size; + } + + int get_chunk_count() const noexcept { + return m_chunks.size(); + } + + int get_used_bytes_in_last_chunk() const noexcept { + return m_chunks.front().size(); + } + + }; // class StringStore + + struct StrComp { + + bool operator()(const char* lhs, const char* rhs) const { + return strcmp(lhs, rhs) < 0; + } + + }; // struct StrComp + + class StringTable { + + StringStore m_strings; + std::map m_index; + size_t m_size; + + public: + + StringTable() : + m_strings(1024 * 1024), + m_index(), + m_size(0) { + m_strings.add(""); + } + + void clear() { + m_strings.clear(); + m_index.clear(); + m_size = 0; + m_strings.add(""); + } + + size_t size() const noexcept { + return m_size + 1; + } + + size_t add(const char* s) { + auto f = m_index.find(s); + if (f != m_index.end()) { + return f->second; + } + + const char* cs = m_strings.add(s); + m_index[cs] = ++m_size; + return m_size; + } + + StringStore::const_iterator begin() const { + return m_strings.begin(); + } + + StringStore::const_iterator end() const { + return m_strings.end(); + } + + }; // class StringTable + + } // namespace detail + + } // namespace io + +} // namespace osmium + +#endif // OSMIUM_IO_DETAIL_STRING_TABLE_HPP diff --git a/include/osmium/io/detail/xml_input_format.hpp b/include/osmium/io/detail/xml_input_format.hpp index c03f84d9b49..b0f3da33927 100644 --- a/include/osmium/io/detail/xml_input_format.hpp +++ b/include/osmium/io/detail/xml_input_format.hpp @@ -66,6 +66,7 @@ DEALINGS IN THE SOFTWARE. #include #include #include +#include #include #include #include @@ -191,6 +192,8 @@ namespace osmium { std::atomic& m_done; + bool m_header_is_done; + /** * A C++ wrapper for the Expat parser that makes sure no memory is leaked. */ @@ -246,16 +249,25 @@ namespace osmium { T& m_data; std::promise& m_promise; + bool m_done; public: PromiseKeeper(T& data, std::promise& promise) : m_data(data), - m_promise(promise) { + m_promise(promise), + m_done(false) { + } + + void fullfill_promise() { + if (!m_done) { + m_promise.set_value(m_data); + m_done = true; + } } ~PromiseKeeper() { - m_promise.set_value(m_data); + fullfill_promise(); } }; // class PromiseKeeper @@ -279,7 +291,8 @@ namespace osmium { m_queue(queue), m_header_promise(header_promise), m_read_types(read_types), - m_done(done) { + m_done(done), + m_header_is_done(false) { } /** @@ -305,7 +318,8 @@ namespace osmium { m_queue(other.m_queue), m_header_promise(other.m_header_promise), m_read_types(other.m_read_types), - m_done(other.m_done) { + m_done(other.m_done), + m_header_is_done(other.m_header_is_done) { } XMLParser(XMLParser&&) = default; @@ -326,6 +340,9 @@ namespace osmium { last = data.empty(); try { parser(data, last); + if (m_header_is_done) { + promise_keeper.fullfill_promise(); + } } catch (ParserIsDone&) { return true; } catch (...) { @@ -343,8 +360,7 @@ namespace osmium { private: const char* init_object(osmium::OSMObject& object, const XML_Char** attrs) { - static const char* empty = ""; - const char* user = empty; + const char* user = ""; if (m_in_delete_section) { object.set_visible(false); @@ -371,8 +387,7 @@ namespace osmium { } void init_changeset(osmium::builder::ChangesetBuilder* builder, const XML_Char** attrs) { - static const char* empty = ""; - const char* user = empty; + const char* user = ""; osmium::Changeset& new_changeset = builder->object(); osmium::Location min; @@ -421,6 +436,7 @@ namespace osmium { } void header_is_done() { + m_header_is_done = true; if (m_read_types == osmium::osm_entity_bits::nothing) { throw ParserIsDone(); } @@ -722,10 +738,15 @@ namespace osmium { namespace { +// we want the register_input_format() function to run, setting the variable +// is only a side-effect, it will never be used +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wunused-variable" const bool registered_xml_input = osmium::io::detail::InputFormatFactory::instance().register_input_format(osmium::io::file_format::xml, [](const osmium::io::File& file, osmium::osm_entity_bits::type read_which_entities, osmium::thread::Queue& input_queue) { return new osmium::io::detail::XMLInputFormat(file, read_which_entities, input_queue); }); +#pragma GCC diagnostic pop } // anonymous namespace diff --git a/include/osmium/io/detail/xml_output_format.hpp b/include/osmium/io/detail/xml_output_format.hpp index 65ba171b8b1..2a381d5ab17 100644 --- a/include/osmium/io/detail/xml_output_format.hpp +++ b/include/osmium/io/detail/xml_output_format.hpp @@ -85,6 +85,9 @@ namespace osmium { case '\'': out += "'"; break; case '<': out += "<"; break; case '>': out += ">"; break; + case '\n': out += " "; break; + case '\r': out += " "; break; + case '\t': out += " "; break; default: out += *in; break; } } @@ -126,6 +129,7 @@ namespace osmium { operation m_last_op {operation::op_none}; + const bool m_add_metadata; const bool m_write_visible_flag; const bool m_write_change_ops; @@ -146,31 +150,33 @@ namespace osmium { void write_meta(const osmium::OSMObject& object) { oprintf(*m_out, " id=\"%" PRId64 "\"", object.id()); - if (object.version()) { - oprintf(*m_out, " version=\"%d\"", object.version()); - } + if (m_add_metadata) { + if (object.version()) { + oprintf(*m_out, " version=\"%d\"", object.version()); + } - if (object.timestamp()) { - *m_out += " timestamp=\""; - *m_out += object.timestamp().to_iso(); - *m_out += "\""; - } + if (object.timestamp()) { + *m_out += " timestamp=\""; + *m_out += object.timestamp().to_iso(); + *m_out += "\""; + } - if (!object.user_is_anonymous()) { - oprintf(*m_out, " uid=\"%d\" user=\"", object.uid()); - xml_string(*m_out, object.user()); - *m_out += "\""; - } + if (!object.user_is_anonymous()) { + oprintf(*m_out, " uid=\"%d\" user=\"", object.uid()); + xml_string(*m_out, object.user()); + *m_out += "\""; + } - if (object.changeset()) { - oprintf(*m_out, " changeset=\"%d\"", object.changeset()); - } + if (object.changeset()) { + oprintf(*m_out, " changeset=\"%d\"", object.changeset()); + } - if (m_write_visible_flag) { - if (object.visible()) { - *m_out += " visible=\"true\""; - } else { - *m_out += " visible=\"false\""; + if (m_write_visible_flag) { + if (object.visible()) { + *m_out += " visible=\"true\""; + } else { + *m_out += " visible=\"false\""; + } } } } @@ -224,9 +230,10 @@ namespace osmium { public: - explicit XMLOutputBlock(osmium::memory::Buffer&& buffer, bool write_visible_flag, bool write_change_ops) : + explicit XMLOutputBlock(osmium::memory::Buffer&& buffer, bool add_metadata, bool write_visible_flag, bool write_change_ops) : m_input_buffer(std::make_shared(std::move(buffer))), m_out(std::make_shared()), + m_add_metadata(add_metadata), m_write_visible_flag(write_visible_flag && !write_change_ops), m_write_change_ops(write_change_ops) { } @@ -392,12 +399,14 @@ namespace osmium { class XMLOutputFormat : public osmium::io::detail::OutputFormat, public osmium::handler::Handler { + bool m_add_metadata; bool m_write_visible_flag; public: XMLOutputFormat(const osmium::io::File& file, data_queue_type& output_queue) : OutputFormat(file, output_queue), + m_add_metadata(file.get("add_metadata") != "false"), m_write_visible_flag(file.has_multiple_object_versions() || m_file.is_true("force_visible_flag")) { } @@ -408,7 +417,7 @@ namespace osmium { } void write_buffer(osmium::memory::Buffer&& buffer) override final { - m_output_queue.push(osmium::thread::Pool::instance().submit(XMLOutputBlock{std::move(buffer), m_write_visible_flag, m_file.is_true("xml_change_format")})); + m_output_queue.push(osmium::thread::Pool::instance().submit(XMLOutputBlock{std::move(buffer), m_add_metadata, m_write_visible_flag, m_file.is_true("xml_change_format")})); } void write_header(const osmium::io::Header& header) override final { @@ -468,10 +477,15 @@ namespace osmium { namespace { +// we want the register_output_format() function to run, setting the variable +// is only a side-effect, it will never be used +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wunused-variable" const bool registered_xml_output = osmium::io::detail::OutputFormatFactory::instance().register_output_format(osmium::io::file_format::xml, [](const osmium::io::File& file, data_queue_type& output_queue) { return new osmium::io::detail::XMLOutputFormat(file, output_queue); }); +#pragma GCC diagnostic pop } // anonymous namespace diff --git a/include/osmium/io/detail/zlib.hpp b/include/osmium/io/detail/zlib.hpp index a402bf7e636..fc0baf344a4 100644 --- a/include/osmium/io/detail/zlib.hpp +++ b/include/osmium/io/detail/zlib.hpp @@ -85,23 +85,24 @@ namespace osmium { * * @param input Compressed input data. * @param raw_size Size of uncompressed data. - * @returns Uncompressed data. + * @param output Uncompressed result data. + * @returns Pointer and size to incompressed data. */ - inline std::unique_ptr zlib_uncompress(const std::string& input, unsigned long raw_size) { - auto output = std::unique_ptr(new std::string(raw_size, '\0')); + inline std::pair zlib_uncompress_string(const char* input, unsigned long input_size, unsigned long raw_size, std::string& output) { + output.resize(raw_size); auto result = ::uncompress( - reinterpret_cast(const_cast(output->data())), + reinterpret_cast(&*output.begin()), &raw_size, - reinterpret_cast(input.data()), - osmium::static_cast_with_assert(input.size()) + reinterpret_cast(input), + input_size ); if (result != Z_OK) { throw std::runtime_error(std::string("failed to uncompress data: ") + zError(result)); } - return output; + return std::make_pair(output.data(), output.size()); } } // namespace detail diff --git a/include/osmium/io/file.hpp b/include/osmium/io/file.hpp index 5b6c02f1324..3bbfacc6def 100644 --- a/include/osmium/io/file.hpp +++ b/include/osmium/io/file.hpp @@ -97,7 +97,9 @@ namespace osmium { * of the file will be taken from the suffix. * An empty filename or "-" means stdin or stdout. * @param format File format as string. See the description of the - * parse_format() function for details. + * parse_format() function for details. If this is + * empty the format will be deduced from the suffix + * of the filename. */ explicit File(const std::string& filename = "", const std::string& format = "") : Options(), @@ -107,20 +109,19 @@ namespace osmium { m_format_string(format) { // stdin/stdout - if (filename == "" || filename == "-") { + if (m_filename == "-") { m_filename = ""; - default_settings_for_stdinout(); } - // filename is actually a URL + // if filename is a URL, default to XML format std::string protocol = m_filename.substr(0, m_filename.find_first_of(':')); if (protocol == "http" || protocol == "https") { - default_settings_for_url(); + m_file_format = file_format::xml; } - detect_format_from_suffix(m_filename); - - if (format != "") { + if (format.empty()) { + detect_format_from_suffix(m_filename); + } else { parse_format(format); } } @@ -140,9 +141,6 @@ namespace osmium { m_buffer(buffer), m_buffer_size(size), m_format_string(format) { - - default_settings_for_stdinout(); - if (format != "") { parse_format(format); } @@ -220,6 +218,20 @@ namespace osmium { } else if (suffixes.back() == "opl") { m_file_format = file_format::opl; suffixes.pop_back(); + } else if (suffixes.back() == "json") { + m_file_format = file_format::json; + suffixes.pop_back(); + } else if (suffixes.back() == "o5m") { + m_file_format = file_format::o5m; + suffixes.pop_back(); + } else if (suffixes.back() == "o5c") { + m_file_format = file_format::o5m; + m_has_multiple_object_versions = true; + set("o5c_change_format", true); + suffixes.pop_back(); + } else if (suffixes.back() == "debug") { + m_file_format = file_format::debug; + suffixes.pop_back(); } if (suffixes.empty()) return; @@ -240,8 +252,8 @@ namespace osmium { } /** - * Check file format etc. for consistency and throw exception if there - * is a problem. + * Check file format etc. for consistency and throw exception if + * there is a problem. * * @throws std::runtime_error */ @@ -265,36 +277,6 @@ namespace osmium { } } - /** - * Set default settings for type and encoding when the filename is - * empty or "-". If you want to have a different default setting - * override this in a subclass. - */ - void default_settings_for_stdinout() { - m_file_format = file_format::unknown; - m_file_compression = file_compression::none; - } - - /** - * Set default settings for type and encoding when the filename is - * a normal file. If you want to have a different default setting - * override this in a subclass. - */ - void default_settings_for_file() { - m_file_format = file_format::unknown; - m_file_compression = file_compression::none; - } - - /** - * Set default settings for type and encoding when the filename is a URL. - * If you want to have a different default setting override this in a - * subclass. - */ - void default_settings_for_url() { - m_file_format = file_format::xml; - m_file_compression = file_compression::none; - } - file_format format() const noexcept { return m_file_format; } diff --git a/include/osmium/io/file_format.hpp b/include/osmium/io/file_format.hpp index 1a63a5e49d2..edfb1ff9f7d 100644 --- a/include/osmium/io/file_format.hpp +++ b/include/osmium/io/file_format.hpp @@ -44,7 +44,9 @@ namespace osmium { xml = 1, pbf = 2, opl = 3, - json = 4 + json = 4, + o5m = 5, + debug = 6 }; // avoid g++ false positive @@ -62,6 +64,10 @@ namespace osmium { return "OPL"; case file_format::json: return "JSON"; + case file_format::o5m: + return "O5M"; + case file_format::debug: + return "DEBUG"; } } #pragma GCC diagnostic pop diff --git a/include/osmium/io/gzip_compression.hpp b/include/osmium/io/gzip_compression.hpp index 272397707d9..204bfd52193 100644 --- a/include/osmium/io/gzip_compression.hpp +++ b/include/osmium/io/gzip_compression.hpp @@ -231,11 +231,16 @@ namespace osmium { namespace { +// we want the register_compression() function to run, setting the variable +// is only a side-effect, it will never be used +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wunused-variable" const bool registered_gzip_compression = osmium::io::CompressionFactory::instance().register_compression(osmium::io::file_compression::gzip, [](int fd) { return new osmium::io::GzipCompressor(fd); }, [](int fd) { return new osmium::io::GzipDecompressor(fd); }, [](const char* buffer, size_t size) { return new osmium::io::GzipBufferDecompressor(buffer, size); } ); +#pragma GCC diagnostic pop } // anonymous namespace diff --git a/include/osmium/io/pbf_input.hpp b/include/osmium/io/pbf_input.hpp index 766153edef9..d7f3787ae0e 100644 --- a/include/osmium/io/pbf_input.hpp +++ b/include/osmium/io/pbf_input.hpp @@ -39,7 +39,6 @@ DEALINGS IN THE SOFTWARE. * Include this file if you want to read OSM PBF files. * * @attention If you include this file, you'll need to link with - * `libprotobuf-lite`, `libosmpbf`, `ws2_32` (Windows only), * `libz`, and enable multithreading. */ diff --git a/include/osmium/io/pbf_output.hpp b/include/osmium/io/pbf_output.hpp index 5f46ede2b29..dad1013d7a9 100644 --- a/include/osmium/io/pbf_output.hpp +++ b/include/osmium/io/pbf_output.hpp @@ -39,7 +39,6 @@ DEALINGS IN THE SOFTWARE. * Include this file if you want to write OSM PBF files. * * @attention If you include this file, you'll need to link with - * `libprotobuf-lite`, `libosmpbf`, `ws2_32` (Windows only), * `libz`, and enable multithreading. */ diff --git a/include/osmium/memory/buffer.hpp b/include/osmium/memory/buffer.hpp index 85a3a46d4da..d800c685c2e 100644 --- a/include/osmium/memory/buffer.hpp +++ b/include/osmium/memory/buffer.hpp @@ -37,7 +37,6 @@ DEALINGS IN THE SOFTWARE. #include #include #include -#include #include #include #include @@ -83,7 +82,7 @@ namespace osmium { * Buffers exist in two flavours, those with external memory management and * those with internal memory management. If you already have some memory * with data in it (for instance read from disk), you create a Buffer with - * external memory managment. It is your job then to free the memory once + * external memory management. It is your job then to free the memory once * the buffer isn't used any more. If you don't have memory already, you can * create a Buffer object and have it manage the memory internally. It will * dynamically allocate memory and free it again after use. @@ -413,6 +412,15 @@ namespace osmium { return iterator(m_data, m_data + m_committed); } + template + t_iterator get_iterator(size_t offset) { + return t_iterator(m_data + offset, m_data + m_committed); + } + + iterator get_iterator(size_t offset) { + return iterator(m_data + offset, m_data + m_committed); + } + template t_iterator end() { return t_iterator(m_data + m_committed, m_data + m_committed); @@ -431,6 +439,15 @@ namespace osmium { return const_iterator(m_data, m_data + m_committed); } + template + t_const_iterator get_iterator(size_t offset) const { + return t_const_iterator(m_data + offset, m_data + m_committed); + } + + const_iterator get_iterator(size_t offset) const { + return const_iterator(m_data + offset, m_data + m_committed); + } + template t_const_iterator cend() const { return t_const_iterator(m_data + m_committed, m_data + m_committed); diff --git a/include/osmium/memory/collection.hpp b/include/osmium/memory/collection.hpp index 7deb88b458c..5cf3cc6e17a 100644 --- a/include/osmium/memory/collection.hpp +++ b/include/osmium/memory/collection.hpp @@ -38,7 +38,6 @@ DEALINGS IN THE SOFTWARE. #include #include -#include namespace osmium { diff --git a/include/osmium/memory/item.hpp b/include/osmium/memory/item.hpp index 2679ca6f20c..dc544049a6f 100644 --- a/include/osmium/memory/item.hpp +++ b/include/osmium/memory/item.hpp @@ -33,7 +33,6 @@ DEALINGS IN THE SOFTWARE. */ -#include #include #include diff --git a/include/osmium/osm/changeset.hpp b/include/osmium/osm/changeset.hpp index 0ab4e9bade8..07bc0dd95c1 100644 --- a/include/osmium/osm/changeset.hpp +++ b/include/osmium/osm/changeset.hpp @@ -33,7 +33,6 @@ DEALINGS IN THE SOFTWARE. */ -#include #include #include @@ -44,6 +43,7 @@ DEALINGS IN THE SOFTWARE. #include #include #include +#include namespace osmium { diff --git a/include/osmium/osm/crc.hpp b/include/osmium/osm/crc.hpp new file mode 100644 index 00000000000..eefa4a13ed8 --- /dev/null +++ b/include/osmium/osm/crc.hpp @@ -0,0 +1,223 @@ +#ifndef OSMIUM_OSM_CRC_HPP +#define OSMIUM_OSM_CRC_HPP + +/* + +This file is part of Osmium (http://osmcode.org/libosmium). + +Copyright 2013-2015 Jochen Topf and others (see README). + +Boost Software License - Version 1.0 - August 17th, 2003 + +Permission is hereby granted, free of charge, to any person or organization +obtaining a copy of the software and accompanying documentation covered by +this license (the "Software") to use, reproduce, display, distribute, +execute, and transmit the Software, and to prepare derivative works of the +Software, and to permit third-parties to whom the Software is furnished to +do so, all subject to the following: + +The copyright notices in the Software and this entire statement, including +the above license grant, this restriction and the following disclaimer, +must be included in all copies of the Software, in whole or in part, and +all derivative works of the Software, unless such copies or derivative +works are solely in the form of machine-executable object code generated by +a source language processor. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT +SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE +FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, +ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. + +*/ + +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace osmium { + + template + class CRC { + + static inline uint16_t byte_swap_16(uint16_t value) noexcept { +# if defined(__GNUC__) || defined(__clang__) + return __builtin_bswap16(value); +# else + return (value >> 8) | (value << 8); +# endif + } + + static inline uint32_t byte_swap_32(uint32_t value) noexcept { +# if defined(__GNUC__) || defined(__clang__) + return __builtin_bswap32(value); +# else + return (value >> 24) | + ((value >> 8) & 0x0000FF00) | + ((value << 8) & 0x00FF0000) | + (value << 24); +# endif + } + + static inline uint64_t byte_swap_64(uint64_t value) noexcept { +# if defined(__GNUC__) || defined(__clang__) + return __builtin_bswap64(value); +# else + uint64_t val1 = byte_swap_32(value & 0xFFFFFFFF); + uint64_t val2 = byte_swap_32(value >> 32); + return (val1 << 32) & val2; +# endif + } + + TCRC m_crc; + + public: + + TCRC& operator()() { + return m_crc; + } + + const TCRC& operator()() const { + return m_crc; + } + + void update_bool(bool value) { + m_crc.process_byte(value); + } + + void update_int8(uint8_t value) { + m_crc.process_byte(value); + } + + void update_int16(uint16_t value) { +#if __BYTE_ORDER == __LITTLE_ENDIAN + m_crc.process_bytes(&value, sizeof(uint16_t)); +#else + uint16_t v = byte_swap_16(value); + m_crc.process_bytes(&v, sizeof(uint16_t)); +#endif + } + + void update_int32(uint32_t value) { +#if __BYTE_ORDER == __LITTLE_ENDIAN + m_crc.process_bytes(&value, sizeof(uint32_t)); +#else + uint32_t v = byte_swap_32(value); + m_crc.process_bytes(&v, sizeof(uint32_t)); +#endif + } + + void update_int64(uint64_t value) { +#if __BYTE_ORDER == __LITTLE_ENDIAN + m_crc.process_bytes(&value, sizeof(uint64_t)); +#else + uint64_t v = byte_swap_64(value); + m_crc.process_bytes(&v, sizeof(uint64_t)); +#endif + } + + void update_string(const char* str) { + while (*str) { + m_crc.process_byte(*str++); + } + } + + void update(const Timestamp& timestamp) { + update_int32(uint32_t(timestamp)); + } + + void update(const osmium::Location& location) { + update_int32(location.x()); + update_int32(location.y()); + } + + void update(const osmium::Box& box) { + update(box.bottom_left()); + update(box.top_right()); + } + + void update(const NodeRef& node_ref) { + update_int64(node_ref.ref()); + } + + void update(const NodeRefList& node_refs) { + for (const NodeRef& node_ref : node_refs) { + update(node_ref); + } + } + + void update(const TagList& tags) { + m_crc.process_bytes(tags.data(), tags.byte_size()); + } + + void update(const osmium::RelationMember& member) { + update_int64(member.ref()); + update_int16(uint16_t(member.type())); + update_string(member.role()); + } + + void update(const osmium::RelationMemberList& members) { + for (const RelationMember& member : members) { + update(member); + } + } + + void update(const osmium::OSMObject& object) { + update_int64(object.id()); + update_bool(object.visible()); + update_int32(object.version()); + update(object.timestamp()); + update_int32(object.uid()); + update_string(object.user()); + update(object.tags()); + } + + void update(const osmium::Node& node) { + update(static_cast(node)); + update(node.location()); + } + + void update(const osmium::Way& way) { + update(static_cast(way)); + update(way.nodes()); + } + + void update(const osmium::Relation& relation) { + update(static_cast(relation)); + update(relation.members()); + } + + void update(const osmium::Area& area) { + update(static_cast(area)); + for (auto it = area.cbegin(); it != area.cend(); ++it) { + if (it->type() == osmium::item_type::outer_ring || + it->type() == osmium::item_type::inner_ring) { + update(static_cast(*it)); + } + } + } + + void update(const osmium::Changeset& changeset) { + update_int64(changeset.id()); + update(changeset.created_at()); + update(changeset.closed_at()); + update(changeset.bounds()); + update_int32(changeset.num_changes()); + update_int32(changeset.uid()); + update_string(changeset.user()); + } + + }; // class CRC + +} // namespace osmium + +#endif // OSMIUM_OSM_CRC diff --git a/include/osmium/osm/diff_object.hpp b/include/osmium/osm/diff_object.hpp index 55a5cef68d6..1e053fdda7e 100644 --- a/include/osmium/osm/diff_object.hpp +++ b/include/osmium/osm/diff_object.hpp @@ -112,8 +112,35 @@ namespace osmium { return m_curr->timestamp(); } + /** + * Return the timestamp when the current version of the object is + * not valid any more, ie the time when the next version of the object + * is valid. If this is the last version of the object, this will + * return a special "end of time" timestamp that is guaranteed to + * be larger than any normal timestamp. + */ const osmium::Timestamp end_time() const noexcept { - return last() ? osmium::Timestamp() : m_next->timestamp(); + return last() ? osmium::end_of_time() : m_next->timestamp(); + } + + /** + * Current object version is valid between time "from" (inclusive) and + * time "to" (not inclusive). + * + * This is a bit more complex than you'd think, because we have to + * handle the case properly where the start_time() == end_time(). + */ + bool is_between(const osmium::Timestamp& from, const osmium::Timestamp& to) const noexcept { + return start_time() < to && + ((start_time() != end_time() && end_time() > from) || + (start_time() == end_time() && end_time() >= from)); + } + + /** + * Current object version is visible at the given timestamp. + */ + bool is_visible_at(const osmium::Timestamp& timestamp) const noexcept { + return start_time() <= timestamp && end_time() > timestamp && m_curr->visible(); } }; // class DiffObject diff --git a/include/osmium/osm/entity.hpp b/include/osmium/osm/entity.hpp index 14861a2d5c8..ce292c8d64b 100644 --- a/include/osmium/osm/entity.hpp +++ b/include/osmium/osm/entity.hpp @@ -35,6 +35,7 @@ DEALINGS IN THE SOFTWARE. #include #include +#include namespace osmium { diff --git a/include/osmium/osm/item_type.hpp b/include/osmium/osm/item_type.hpp index c2187a36aed..54975e326d7 100644 --- a/include/osmium/osm/item_type.hpp +++ b/include/osmium/osm/item_type.hpp @@ -33,6 +33,7 @@ DEALINGS IN THE SOFTWARE. */ +#include #include // IWYU pragma: keep #include #include @@ -56,6 +57,25 @@ namespace osmium { }; // enum class item_type + /** + * Return item_type for index: + * 0 -> node, 1 -> way, 2 -> relation + */ + inline item_type nwr_index_to_item_type(unsigned int i) noexcept { + assert(i <= 2); + return item_type(i+1); + } + + /** + * Return index for item_type: + * node -> 0, way -> 1, relation -> 2 + */ + inline unsigned int item_type_to_nwr_index(item_type type) noexcept { + unsigned int i = static_cast(type); + assert(i >= 1 && i <= 3); + return i - 1; + } + inline item_type char_to_item_type(const char c) noexcept { switch (c) { case 'X': diff --git a/include/osmium/osm/node_ref.hpp b/include/osmium/osm/node_ref.hpp index 76afa75fcbd..72359cd0fcf 100644 --- a/include/osmium/osm/node_ref.hpp +++ b/include/osmium/osm/node_ref.hpp @@ -33,11 +33,11 @@ DEALINGS IN THE SOFTWARE. */ +#include #include #include #include -#include #include #include diff --git a/include/osmium/osm/object.hpp b/include/osmium/osm/object.hpp index d5ae48aabe4..8c745ce9b3f 100644 --- a/include/osmium/osm/object.hpp +++ b/include/osmium/osm/object.hpp @@ -48,6 +48,7 @@ DEALINGS IN THE SOFTWARE. #include #include #include +#include namespace osmium { diff --git a/include/osmium/osm/relation.hpp b/include/osmium/osm/relation.hpp index e5b42fed56b..99a4f4cd169 100644 --- a/include/osmium/osm/relation.hpp +++ b/include/osmium/osm/relation.hpp @@ -120,6 +120,11 @@ namespace osmium { return static_cast(std::abs(m_ref)); } + RelationMember& set_ref(const osmium::object_id_type ref) noexcept { + m_ref = ref; + return *this; + } + item_type type() const noexcept { return m_type; } diff --git a/include/osmium/osm/timestamp.hpp b/include/osmium/osm/timestamp.hpp index 6b6a6e18988..390f0e74522 100644 --- a/include/osmium/osm/timestamp.hpp +++ b/include/osmium/osm/timestamp.hpp @@ -39,9 +39,9 @@ DEALINGS IN THE SOFTWARE. #include #include #include -#include #include +#include // IWYU pragma: keep namespace osmium { @@ -170,6 +170,16 @@ namespace osmium { return out; } + template <> + inline osmium::Timestamp min_op_start_value() { + return end_of_time(); + } + + template <> + inline osmium::Timestamp max_op_start_value() { + return start_of_time(); + } + } // namespace osmium #endif // OSMIUM_OSM_TIMESTAMP_HPP diff --git a/include/osmium/osm/types.hpp b/include/osmium/osm/types.hpp index aea61bd376f..b3414e59404 100644 --- a/include/osmium/osm/types.hpp +++ b/include/osmium/osm/types.hpp @@ -34,7 +34,6 @@ DEALINGS IN THE SOFTWARE. */ #include -#include namespace osmium { @@ -58,26 +57,6 @@ namespace osmium { */ typedef uint16_t string_size_type; - inline object_id_type string_to_object_id(const char* string) { - return std::atoll(string); - } - - inline object_version_type string_to_object_version(const char* string) { - return static_cast(std::atol(string)); - } - - inline changeset_id_type string_to_changeset_id(const char* string) { - return static_cast(std::atol(string)); - } - - inline signed_user_id_type string_to_user_id(const char* string) { - return static_cast(std::atol(string)); - } - - inline num_changes_type string_to_num_changes(const char* string) { - return static_cast(std::atol(string)); - } - } // namespace osmium #endif // OSMIUM_OSM_TYPES_HPP diff --git a/include/osmium/osm/types_from_string.hpp b/include/osmium/osm/types_from_string.hpp new file mode 100644 index 00000000000..b8de14c90f2 --- /dev/null +++ b/include/osmium/osm/types_from_string.hpp @@ -0,0 +1,116 @@ +#ifndef OSMIUM_OSM_TYPES_FROM_STRING_HPP +#define OSMIUM_OSM_TYPES_FROM_STRING_HPP + +/* + +This file is part of Osmium (http://osmcode.org/libosmium). + +Copyright 2013-2015 Jochen Topf and others (see README). + +Boost Software License - Version 1.0 - August 17th, 2003 + +Permission is hereby granted, free of charge, to any person or organization +obtaining a copy of the software and accompanying documentation covered by +this license (the "Software") to use, reproduce, display, distribute, +execute, and transmit the Software, and to prepare derivative works of the +Software, and to permit third-parties to whom the Software is furnished to +do so, all subject to the following: + +The copyright notices in the Software and this entire statement, including +the above license grant, this restriction and the following disclaimer, +must be included in all copies of the Software, in whole or in part, and +all derivative works of the Software, unless such copies or derivative +works are solely in the form of machine-executable object code generated by +a source language processor. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT +SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE +FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, +ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. + +*/ + +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +namespace osmium { + + inline object_id_type string_to_object_id(const char* input) { + assert(input); + if (*input != '\0' && !std::isspace(*input)) { + char* end; + auto id = std::strtoll(input, &end, 10); + if (id != std::numeric_limits::min() && id != std::numeric_limits::max() && *end == '\0') { + return id; + } + } + throw std::range_error(std::string("illegal id: '") + input + "'"); + } + + inline std::pair string_to_object_id(const char* input, osmium::osm_entity_bits::type types) { + assert(input); + assert(types != osmium::osm_entity_bits::nothing); + if (*input != '\0') { + if (std::isdigit(*input)) { + return std::make_pair(osmium::item_type::undefined, string_to_object_id(input)); + } + osmium::item_type t = osmium::char_to_item_type(*input); + if (osmium::osm_entity_bits::from_item_type(t) & types) { + return std::make_pair(t, string_to_object_id(input+1)); + } + } + throw std::range_error(std::string("not a valid id: '") + input + "'"); + } + + namespace detail { + + inline long string_to_ulong(const char* input, const char *name) { + if (*input != '\0' && *input != '-' && !std::isspace(*input)) { + char* end; + auto value = std::strtoul(input, &end, 10); + if (value != std::numeric_limits::max() && *end == '\0') { + return value; + } + } + throw std::range_error(std::string("illegal ") + name + ": '" + input + "'"); + } + + } // namespace detail + + inline object_version_type string_to_object_version(const char* input) { + assert(input); + return static_cast(detail::string_to_ulong(input, "version")); + } + + inline changeset_id_type string_to_changeset_id(const char* input) { + assert(input); + return static_cast(detail::string_to_ulong(input, "changeset")); + } + + inline signed_user_id_type string_to_user_id(const char* input) { + assert(input); + if (input[0] == '-' && input[1] == '1' && input[2] == '\0') { + return -1; + } + return static_cast(detail::string_to_ulong(input, "user id")); + } + + inline num_changes_type string_to_num_changes(const char* input) { + assert(input); + return static_cast(detail::string_to_ulong(input, "value for num changes")); + } + +} // namespace osmium + +#endif // OSMIUM_OSM_TYPES_FROM_STRING_HPP diff --git a/include/osmium/relations/collector.hpp b/include/osmium/relations/collector.hpp index e3c4980870f..40e377393b2 100644 --- a/include/osmium/relations/collector.hpp +++ b/include/osmium/relations/collector.hpp @@ -512,7 +512,7 @@ namespace osmium { double percent = static_cast(size_before - size_after); percent /= size_before; percent *= 100; - std::cerr << "PURGE (size before=" << size_before << " after=" << size_after << " purged=" << (size_before - size_after) << " / " << static_cast(percent) << "%)\n"; +// std::cerr << "PURGE (size before=" << size_before << " after=" << size_after << " purged=" << (size_before - size_after) << " / " << static_cast(percent) << "%)\n"; m_count_complete = 0; } } diff --git a/include/osmium/thread/pool.hpp b/include/osmium/thread/pool.hpp index 87dd1fb933e..391603108ff 100644 --- a/include/osmium/thread/pool.hpp +++ b/include/osmium/thread/pool.hpp @@ -149,6 +149,7 @@ namespace osmium { ~Pool() { m_done = true; + m_work_queue.shutdown(); } size_t queue_size() const { diff --git a/include/osmium/thread/queue.hpp b/include/osmium/thread/queue.hpp index baaf2dc766b..76ad9a02031 100644 --- a/include/osmium/thread/queue.hpp +++ b/include/osmium/thread/queue.hpp @@ -41,9 +41,7 @@ DEALINGS IN THE SOFTWARE. #include #include #include -#include - -#include +#include // IWYU pragma: keep (for std::move) namespace osmium { @@ -71,6 +69,8 @@ namespace osmium { /// Used to signal readers when data is available in the queue. std::condition_variable m_data_available; + std::atomic m_done; + #ifdef OSMIUM_DEBUG_QUEUE_SIZE /// The largest size the queue has been so far. size_t m_largest_size; @@ -94,7 +94,8 @@ namespace osmium { m_name(name), m_mutex(), m_queue(), - m_data_available() + m_data_available(), + m_done(false) #ifdef OSMIUM_DEBUG_QUEUE_SIZE , m_largest_size(0), @@ -104,6 +105,7 @@ namespace osmium { } ~Queue() { + shutdown(); #ifdef OSMIUM_DEBUG_QUEUE_SIZE std::cerr << "queue '" << m_name << "' with max_size=" << m_max_size << " had largest size " << m_largest_size << " and was full " << m_full_counter << " times\n"; #endif @@ -132,24 +134,33 @@ namespace osmium { m_data_available.notify_one(); } + void shutdown() { + m_done = true; + m_data_available.notify_all(); + } + void wait_and_pop(T& value) { std::unique_lock lock(m_mutex); m_data_available.wait(lock, [this] { - return !m_queue.empty(); + return !m_queue.empty() || m_done; }); - value = std::move(m_queue.front()); - m_queue.pop(); + if (!m_queue.empty()) { + value = std::move(m_queue.front()); + m_queue.pop(); + } } void wait_and_pop_with_timeout(T& value) { std::unique_lock lock(m_mutex); if (!m_data_available.wait_for(lock, std::chrono::seconds(1), [this] { - return !m_queue.empty(); + return !m_queue.empty() || m_done; })) { return; } - value = std::move(m_queue.front()); - m_queue.pop(); + if (!m_queue.empty()) { + value = std::move(m_queue.front()); + m_queue.pop(); + } } bool try_pop(T& value) { diff --git a/include/osmium/thread/util.hpp b/include/osmium/thread/util.hpp index 62bb82ab502..ca4f6dd538e 100644 --- a/include/osmium/thread/util.hpp +++ b/include/osmium/thread/util.hpp @@ -58,7 +58,7 @@ namespace osmium { /** * Wait until the given future becomes ready. Will block if the future - * is not ready. Can be called more than once unless future.get(). + * is not ready. Can be called more than once unlike future.get(). */ template inline void wait_until_done(std::future& future) { diff --git a/include/osmium/util/data_file.hpp b/include/osmium/util/data_file.hpp new file mode 100644 index 00000000000..22bf1910a62 --- /dev/null +++ b/include/osmium/util/data_file.hpp @@ -0,0 +1,194 @@ +#ifndef OSMIUM_UTIL_DATA_FILE_HPP +#define OSMIUM_UTIL_DATA_FILE_HPP + +/* + +This file is part of Osmium (http://osmcode.org/libosmium). + +Copyright 2013-2015 Jochen Topf and others (see README). + +Boost Software License - Version 1.0 - August 17th, 2003 + +Permission is hereby granted, free of charge, to any person or organization +obtaining a copy of the software and accompanying documentation covered by +this license (the "Software") to use, reproduce, display, distribute, +execute, and transmit the Software, and to prepare derivative works of the +Software, and to permit third-parties to whom the Software is furnished to +do so, all subject to the following: + +The copyright notices in the Software and this entire statement, including +the above license grant, this restriction and the following disclaimer, +must be included in all copies of the Software, in whole or in part, and +all derivative works of the Software, unless such copies or derivative +works are solely in the form of machine-executable object code generated by +a source language processor. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT +SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE +FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, +ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. + +*/ + +#include +#include +#include +#include +#include +#include + +#ifdef _WIN32 +# include +# include +#endif + +#include + +namespace osmium { + + namespace util { + + /** + * Class wrapper for convenient access to some low-level file + * functions. + */ + class DataFile { + + FILE* m_file; + + public: + + /** + * Create and open a temporary file. It is removed after opening. + * + * @throws std::system_error if something went wrong. + */ + DataFile() : + m_file(::tmpfile()) { + if (!m_file) { + throw std::system_error(errno, std::system_category(), "tmpfile failed"); + } + } + + /** + * Create and open a temporary file with the specified size. It + * is removed after opening. + * + * @throws std::system_error if something went wrong. + */ + explicit DataFile(size_t size) : + DataFile() { + grow(size); + } + + /** + * Create and open a named file. + * + * @param filename the name of the file + * @param writable should the file be writable? + * @throws std::system_error if something went wrong. + */ + DataFile(const char* filename, bool writable) : + m_file(::fopen(filename, writable ? "wb+" : "rb" )) { + if (!m_file) { + throw std::system_error(errno, std::system_category(), "fopen failed"); + } + } + + /** + * Create and open a named file. + * + * @param filename the name of the file + * @param writable should the file be writable? + * @throws std::system_error if something went wrong. + */ + DataFile(const std::string& filename, bool writable) : + DataFile(filename.c_str(), writable) { + } + + /** + * In boolean context the DataFile class returns true if the file + * is open. + */ + operator bool() const noexcept { + return m_file != nullptr; + } + + /** + * Close the file. + * + * Does nothing if the file is already closed. + * + * @throws std::system_error if file could not be closed + */ + void close() { + if (m_file) { + if (::fclose(m_file) != 0) { + throw std::system_error(errno, std::system_category(), "fclose failed"); + } + m_file = nullptr; + } + } + + ~DataFile() noexcept { + try { + close(); + } catch (std::system_error&) { + // ignore + } + } + + /** + * Get file descriptor of underlying file. + * + * @throws std::runtime_errro if file is not open + * @throws std::system_error if fileno(3) call failed + */ + int fd() const { + if (!m_file) { + throw std::runtime_error("no open file"); + } + + int fd = ::fileno(m_file); + + if (fd == -1) { + throw std::system_error(errno, std::system_category(), "fileno failed"); + } + + return fd; + } + + /** + * Ask the operating system for the size of this file. + * + * @throws std::system_error if fstat(2) call failed + */ + size_t size() const { + return osmium::util::file_size(fd()); + } + + /** + * Grow file to given size. + * + * If the file is large enough already, nothing is done. + * The file is never shrunk. + * + * @throws std::system_error if ftruncate(2) call failed + */ + void grow(size_t new_size) const { + if (size() < new_size) { + osmium::util::resize_file(fd(), new_size); + } + } + + }; // class DataFile + + } // namespace util + +} // namespace osmium + + +#endif // OSMIUM_UTIL_DATA_FILE_HPP diff --git a/include/osmium/util/delta.hpp b/include/osmium/util/delta.hpp new file mode 100644 index 00000000000..0c77e52426f --- /dev/null +++ b/include/osmium/util/delta.hpp @@ -0,0 +1,147 @@ +#ifndef OSMIUM_UTIL_DELTA_HPP +#define OSMIUM_UTIL_DELTA_HPP + +/* + +This file is part of Osmium (http://osmcode.org/libosmium). + +Copyright 2013-2015 Jochen Topf and others (see README). + +Boost Software License - Version 1.0 - August 17th, 2003 + +Permission is hereby granted, free of charge, to any person or organization +obtaining a copy of the software and accompanying documentation covered by +this license (the "Software") to use, reproduce, display, distribute, +execute, and transmit the Software, and to prepare derivative works of the +Software, and to permit third-parties to whom the Software is furnished to +do so, all subject to the following: + +The copyright notices in the Software and this entire statement, including +the above license grant, this restriction and the following disclaimer, +must be included in all copies of the Software, in whole or in part, and +all derivative works of the Software, unless such copies or derivative +works are solely in the form of machine-executable object code generated by +a source language processor. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT +SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE +FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, +ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. + +*/ + +#include +#include +#include + +namespace osmium { + + namespace util { + + /** + * Helper class for delta encoding. + */ + template + class DeltaEncode { + + T m_value; + + public: + + DeltaEncode(T value = 0) : + m_value(value) { + } + + void clear() { + m_value = 0; + } + + T update(T new_value) { + using std::swap; + swap(m_value, new_value); + return m_value - new_value; + } + + }; // class DeltaEncode + + /** + * Helper class for delta decoding. + */ + template + class DeltaDecode { + + T m_value; + + public: + + DeltaDecode() : + m_value(0) { + } + + void clear() { + m_value = 0; + } + + T update(T delta) { + m_value += delta; + return m_value; + } + + }; // class DeltaDecode + + template + class DeltaEncodeIterator : public std::iterator { + + typedef TValue value_type; + + TBaseIterator m_it; + TBaseIterator m_end; + value_type m_delta; + DeltaEncode m_value; + TTransform m_trans; + + public: + + DeltaEncodeIterator(TBaseIterator first, TBaseIterator last, TTransform& trans) : + m_it(first), + m_end(last), + m_delta(m_trans(m_it)), + m_value(m_delta), + m_trans(trans) { + } + + DeltaEncodeIterator& operator++() { + if (m_it != m_end) { + m_delta = m_value.update(m_trans(++m_it)); + } + return *this; + } + + DeltaEncodeIterator operator++(int) { + DeltaEncodeIterator tmp(*this); + operator++(); + return tmp; + } + + value_type operator*() { + return m_delta; + } + + bool operator==(const DeltaEncodeIterator& rhs) const { + return m_it == rhs.m_it && m_end == rhs.m_end; + } + + bool operator!=(const DeltaEncodeIterator& rhs) const { + return !(*this == rhs); + } + + }; // class DeltaEncodeIterator + + } // namespace util + +} // namespace osmium + +#endif // OSMIUM_UTIL_DELTA_HPP diff --git a/include/osmium/util/endian.hpp b/include/osmium/util/endian.hpp new file mode 100644 index 00000000000..a5d91543ef7 --- /dev/null +++ b/include/osmium/util/endian.hpp @@ -0,0 +1,45 @@ +#ifndef OSMIUM_UTIL_ENDIAN_HPP +#define OSMIUM_UTIL_ENDIAN_HPP + +/* + +This file is part of Osmium (http://osmcode.org/libosmium). + +Copyright 2013-2015 Jochen Topf and others (see README). + +Boost Software License - Version 1.0 - August 17th, 2003 + +Permission is hereby granted, free of charge, to any person or organization +obtaining a copy of the software and accompanying documentation covered by +this license (the "Software") to use, reproduce, display, distribute, +execute, and transmit the Software, and to prepare derivative works of the +Software, and to permit third-parties to whom the Software is furnished to +do so, all subject to the following: + +The copyright notices in the Software and this entire statement, including +the above license grant, this restriction and the following disclaimer, +must be included in all copies of the Software, in whole or in part, and +all derivative works of the Software, unless such copies or derivative +works are solely in the form of machine-executable object code generated by +a source language processor. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT +SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE +FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, +ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. + +*/ + +// Windows is only available for little endian architectures +// http://stackoverflow.com/questions/6449468/can-i-safely-assume-that-windows-installations-will-always-be-little-endian +#if !defined(_WIN32) && !defined(__APPLE__) +# include +#else +# define __LITTLE_ENDIAN 1234 +# define __BYTE_ORDER __LITTLE_ENDIAN +#endif + +#endif // OSMIUM_UTIL_ENDIAN_HPP diff --git a/include/osmium/util/file.hpp b/include/osmium/util/file.hpp new file mode 100644 index 00000000000..461f4e6429b --- /dev/null +++ b/include/osmium/util/file.hpp @@ -0,0 +1,119 @@ +#ifndef OSMIUM_UTIL_FILE_HPP +#define OSMIUM_UTIL_FILE_HPP + +/* + +This file is part of Osmium (http://osmcode.org/libosmium). + +Copyright 2013-2015 Jochen Topf and others (see README). + +Boost Software License - Version 1.0 - August 17th, 2003 + +Permission is hereby granted, free of charge, to any person or organization +obtaining a copy of the software and accompanying documentation covered by +this license (the "Software") to use, reproduce, display, distribute, +execute, and transmit the Software, and to prepare derivative works of the +Software, and to permit third-parties to whom the Software is furnished to +do so, all subject to the following: + +The copyright notices in the Software and this entire statement, including +the above license grant, this restriction and the following disclaimer, +must be included in all copies of the Software, in whole or in part, and +all derivative works of the Software, unless such copies or derivative +works are solely in the form of machine-executable object code generated by +a source language processor. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT +SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE +FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, +ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. + +*/ + +#include +#include +#include +#include +#include +#include + +#ifdef _WIN32 +# include +# include +#endif + +#ifndef _MSC_VER +# include +#else +// https://msdn.microsoft.com/en-us/library/whx354w1.aspx +# define ftruncate _chsize_s +#endif + +namespace osmium { + + namespace util { + + /** + * Get file size. + * This is a small wrapper around a system call. + * + * @param fd File descriptor + * @returns file size + * @throws std::system_error If system call failed + */ + inline size_t file_size(int fd) { +#ifdef _MSC_VER + // Windows implementation + // https://msdn.microsoft.com/en-us/library/dfbc2kec.aspx + auto size = ::_filelengthi64(fd); + if (size == -1L) { + throw std::system_error(errno, std::system_category(), "_filelengthi64 failed"); + } + return size_t(size); +#else + // Unix implementation + struct stat s; + if (::fstat(fd, &s) != 0) { + throw std::system_error(errno, std::system_category(), "fstat failed"); + } + return size_t(s.st_size); +#endif + } + + /** + * Resize file. + * Small wrapper around ftruncate(2) system call. + * + * @param fd File descriptor + * @param new_size New size + * @throws std::system_error If ftruncate(2) call failed + */ + inline void resize_file(int fd, size_t new_size) { + if (::ftruncate(fd, new_size) != 0) { + throw std::system_error(errno, std::system_category(), "ftruncate failed"); + } + } + + /** + * Get the page size for this system. + */ + inline size_t get_pagesize() { +#ifdef _WIN32 + // Windows implementation + SYSTEM_INFO si; + GetSystemInfo(&si); + return si.dwPageSize; +#else + // Unix implementation + return ::sysconf(_SC_PAGESIZE); +#endif + } + + } // namespace util + +} // namespace osmium + +#endif // OSMIUM_UTIL_FILE_HPP diff --git a/include/osmium/util/memory_mapping.hpp b/include/osmium/util/memory_mapping.hpp new file mode 100644 index 00000000000..e48aff2825f --- /dev/null +++ b/include/osmium/util/memory_mapping.hpp @@ -0,0 +1,750 @@ +#ifndef OSMIUM_UTIL_MEMORY_MAPPING_HPP +#define OSMIUM_UTIL_MEMORY_MAPPING_HPP + +/* + +This file is part of Osmium (http://osmcode.org/libosmium). + +Copyright 2013-2015 Jochen Topf and others (see README). + +Boost Software License - Version 1.0 - August 17th, 2003 + +Permission is hereby granted, free of charge, to any person or organization +obtaining a copy of the software and accompanying documentation covered by +this license (the "Software") to use, reproduce, display, distribute, +execute, and transmit the Software, and to prepare derivative works of the +Software, and to permit third-parties to whom the Software is furnished to +do so, all subject to the following: + +The copyright notices in the Software and this entire statement, including +the above license grant, this restriction and the following disclaimer, +must be included in all copies of the Software, in whole or in part, and +all derivative works of the Software, unless such copies or derivative +works are solely in the form of machine-executable object code generated by +a source language processor. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT +SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE +FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, +ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. + +*/ + +#include +#include +#include +#include + +#include + +#ifndef _WIN32 +# include +#else +# include +# include +# include +#endif + +namespace osmium { + + namespace util { + + /** + * Class for wrapping memory mapping system calls. + * + * Usage for anonymous mapping: + * @code + * MemoryMapping mapping(1024); // create anonymous mapping with size + * auto ptr = mapping.get_addr(); // get pointer to memory + * mapping.unmap(); // release mapping by calling unmap() (or at end of scope) + * @endcode + * + * Or for file-backed mapping: + * @code + * int fd = ::open(...); + * { + * MemoryMapping mapping(1024, MemoryMapping::mapping_mode::write_shared, fd, offset); + * // use mapping + * } + * ::close(fd); + * @endcode + * + * If the file backing a file-backed mapping is not large enough, it + * will be resized. This works, of course, only for writable files, + * so for read-only files you have to make sure they are large enough + * for any mapping you want. + * + * If you ask for a zero-sized mapping, a mapping of the systems page + * size will be created instead. For file-backed mapping this will only + * work if the file is writable. + * + * There are different implementations for Unix and Windows systems. + * On Unix systems this wraps the mmap(), munmap(), and the mremap() + * system calls. On Windows it wraps the CreateFileMapping(), + * CloseHandle(), MapViewOfFile(), and UnmapViewOfFile() functions. + */ + class MemoryMapping { + +public: + enum class mapping_mode { + readonly = 0, + write_private = 1, + write_shared = 2 + }; + +private: + + /// The size of the mapping + size_t m_size; + + /// Offset into the file + off_t m_offset; + + /// File handle we got the mapping from + int m_fd; + + /// Mapping mode + mapping_mode m_mapping_mode; + +#ifdef _WIN32 + HANDLE m_handle; +#endif + + /// The address where the memory is mapped + void* m_addr; + + bool is_valid() const noexcept; + + void make_invalid() noexcept; + +#ifdef _WIN32 + typedef DWORD flag_type; +#else + typedef int flag_type; +#endif + + flag_type get_protection() const noexcept; + + flag_type get_flags() const noexcept; + + // A zero-sized mapping is not allowed by the operating system. + // So if the user asks for a mapping of size 0, we map a full + // page instead. This way we don't have a special case in the rest + // of the code. + static size_t initial_size(size_t size) { + if (size == 0) { + return osmium::util::get_pagesize(); + } + return size; + } + +#ifdef _WIN32 + HANDLE get_handle() const noexcept; + HANDLE osmium::util::MemoryMapping::create_file_mapping() const noexcept; + void* osmium::util::MemoryMapping::map_view_of_file() const noexcept; +#endif + + int resize_fd(int fd) { + // Anonymous mapping doesn't need resizing. + if (fd == -1) { + return -1; + } + + // Make sure the file backing this mapping is large enough. + if (osmium::util::file_size(fd) < m_size + m_offset) { + osmium::util::resize_file(fd, m_size + m_offset); + } + return fd; + } + + public: + + /** + * Create memory mapping of given size. + * + * If fd is not set (or fd == -1), an anonymous mapping will be + * created, otherwise a mapping based on the file descriptor will + * be created. + * + * @pre size > 0 or mode == write_shared oder write_private + * + * @param size Size of the mapping in bytes + * @param mode Mapping mode: readonly, or writable (shared or private) + * @param fd Open file descriptor of a file we want to map + * @param offset Offset into the file where the mapping should start + * @throws std::system_error if the mapping fails + */ + MemoryMapping(size_t size, mapping_mode mode, int fd=-1, off_t offset=0); + + /// DEPRECATED: For backwards compatibility + MemoryMapping(size_t size, bool writable=true, int fd=-1, off_t offset=0) : + MemoryMapping(size, writable ? mapping_mode::write_shared : mapping_mode::readonly, fd, offset) { + } + + /// You can not copy construct a MemoryMapping. + MemoryMapping(const MemoryMapping&) = delete; + + /// You can not copy a MemoryMapping. + MemoryMapping& operator=(const MemoryMapping&) = delete; + + /** + * Move construct a mapping from another one. The other mapping + * will be marked as invalid. + */ + MemoryMapping(MemoryMapping&& other); + + /** + * Move a mapping. The other mapping will be marked as invalid. + */ + MemoryMapping& operator=(MemoryMapping&& other); + + /** + * Releases the mapping by calling unmap(). Will never throw. + * Call unmap() instead if you want to be notified of any error. + */ + ~MemoryMapping() noexcept { + try { + unmap(); + } catch (std::system_error&) { + // ignore + } + } + + /** + * Unmap a mapping. If the mapping is not valid, it will do + * nothing. + * + * @throws std::system_error if the unmapping fails + */ + void unmap(); + + /** + * Resize a mapping to the given new size. + * + * On Linux systems this will use the mremap() function. On other + * systems it will unmap and remap the memory. This can only be + * done for file-based mappings, not anonymous mappings! + * + * @param new_size Number of bytes to resize to + * @throws std::system_error if the remapping fails + */ + void resize(size_t new_size); + + /** + * In a boolean context a MemoryMapping is true when it is a valid + * existing mapping. + */ + operator bool() const noexcept { + return is_valid(); + } + + /** + * The number of bytes mapped. This is the same size you created + * the mapping with. The actual mapping will probably be larger + * because the system will round it to the page size. + */ + size_t size() const noexcept { + return m_size; + } + + /** + * The file descriptor this mapping was created from. + * + * @returns file descriptor, -1 for anonymous mappings + */ + int fd() const noexcept { + return m_fd; + } + + /** + * Was this mapping created as a writable mapping? + */ + bool writable() const noexcept { + return m_mapping_mode != mapping_mode::readonly; + } + + /** + * Get the address of the mapping as any pointer type you like. + * + * @throws std::runtime_error if the mapping is invalid + */ + template + T* get_addr() const { + if (is_valid()) { + return reinterpret_cast(m_addr); + } + throw std::runtime_error("invalid memory mapping"); + } + + }; // class MemoryMapping + + /** + * Anonymous memory mapping. + * + * Usage for anonymous mapping: + * @code + * AnonymousMemoryMapping mapping(1024); // create anonymous mapping with size + * auto ptr = mapping.get_addr(); // get pointer to memory + * mapping.unmap(); // release mapping by calling unmap() (or at end of scope) + * @endcode + */ + class AnonymousMemoryMapping : public MemoryMapping { + + public: + + AnonymousMemoryMapping(size_t size) : + MemoryMapping(size, mapping_mode::write_private) { + } + +#ifndef __linux__ + /** + * On systems other than Linux anonymous mappings can not be + * resized! + */ + void resize(size_t) = delete; +#endif + + }; // class AnonymousMemoryMapping + + /** + * A thin wrapper around the MemoryMapping class used when all the + * data in the mapped memory is of the same type. Instead of thinking + * about the number of bytes mapped, this counts sizes in the number + * of objects of that type. + * + * Note that no effort is made to actually initialize the objects in + * this memory. This has to be done by the caller! + */ + template + class TypedMemoryMapping { + + MemoryMapping m_mapping; + + public: + + /** + * Create anonymous typed memory mapping of given size. + * + * @param size Number of objects of type T to be mapped + * @throws std::system_error if the mapping fails + */ + TypedMemoryMapping(size_t size) : + m_mapping(sizeof(T) * size, MemoryMapping::mapping_mode::write_private) { + } + + /** + * Create file-backed memory mapping of given size. The file must + * contain at least `sizeof(T) * size` bytes! + * + * @param size Number of objects of type T to be mapped + * @param mode Mapping mode: readonly, or writable (shared or private) + * @param fd Open file descriptor of a file we want to map + * @param offset Offset into the file where the mapping should start + * @throws std::system_error if the mapping fails + */ + TypedMemoryMapping(size_t size, MemoryMapping::mapping_mode mode, int fd, off_t offset = 0) : + m_mapping(sizeof(T) * size, mode, fd, sizeof(T) * offset) { + } + + /// DEPRECATED: For backwards compatibility + TypedMemoryMapping(size_t size, bool writable, int fd, off_t offset = 0) : + m_mapping(sizeof(T) * size, writable ? MemoryMapping::mapping_mode::write_shared : MemoryMapping::mapping_mode::readonly, fd, sizeof(T) * offset) { + } + + /// You can not copy construct a TypedMemoryMapping. + TypedMemoryMapping(const TypedMemoryMapping&) = delete; + + /// You can not copy a MemoryMapping. + TypedMemoryMapping& operator=(const TypedMemoryMapping&) = delete; + + /** + * Move construct a mapping from another one. The other mapping + * will be marked as invalid. + */ + TypedMemoryMapping(TypedMemoryMapping&& other) = default; + + /** + * Move a mapping. The other mapping will be marked as invalid. + */ + TypedMemoryMapping& operator=(TypedMemoryMapping&& other) = default; + + /** + * Releases the mapping by calling unmap(). Will never throw. + * Call unmap() instead if you want to be notified of any error. + */ + ~TypedMemoryMapping() = default; + + /** + * Unmap a mapping. If the mapping is not valid, it will do + * nothing. + * + * @throws std::system_error if the unmapping fails + */ + void unmap() { + m_mapping.unmap(); + } + + /** + * Resize a mapping to the given new size. + * + * On Linux systems this will use the mremap() function. On other + * systems it will unmap and remap the memory. This can only be + * done for file-based mappings, not anonymous mappings! + * + * @param new_size Number of objects of type T to resize to + * @throws std::system_error if the remapping fails + */ + void resize(size_t new_size) { + m_mapping.resize(sizeof(T) * new_size); + } + + /** + * In a boolean context a TypedMemoryMapping is true when it is + * a valid existing mapping. + */ + operator bool() const noexcept { + return !!m_mapping; + } + + /** + * The number of objects of class T mapped. This is the same size + * you created the mapping with. The actual mapping will probably + * be larger because the system will round it to the page size. + */ + size_t size() const noexcept { + assert(m_mapping.size() % sizeof(T) == 0); + return m_mapping.size() / sizeof(T); + } + + /** + * The file descriptor this mapping was created from. + * + * @returns file descriptor, -1 for anonymous mappings + */ + int fd() const noexcept { + return m_mapping.fd(); + } + + /** + * Was this mapping created as a writable mapping? + */ + bool writable() const noexcept { + return m_mapping.writable(); + } + + /** + * Get the address of the beginning of the mapping. + * + * @throws std::runtime_error if the mapping is invalid + */ + T* begin() { + return m_mapping.get_addr(); + } + + /** + * Get the address one past the end of the mapping. + * + * @throws std::runtime_error if the mapping is invalid + */ + T* end() { + return m_mapping.get_addr() + size(); + } + + const T* cbegin() const { + return m_mapping.get_addr(); + } + + const T* cend() const { + return m_mapping.get_addr() + size(); + } + + const T* begin() const { + return m_mapping.get_addr(); + } + + const T* end() const { + return m_mapping.get_addr() + size(); + } + + }; // class TypedMemoryMapping + + template + class AnonymousTypedMemoryMapping : public TypedMemoryMapping { + + public: + + AnonymousTypedMemoryMapping(size_t size) : + TypedMemoryMapping(size) { + } + +#ifndef __linux__ + /** + * On systems other than Linux anonymous mappings can not be + * resized! + */ + void resize(size_t) = delete; +#endif + + }; // class AnonymousTypedMemoryMapping + + } // namespace util + +} // namespace osmium + +#ifndef _WIN32 + +// =========== Unix implementation ============= + +// MAP_FAILED is often a macro containing an old style cast +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wold-style-cast" + +inline bool osmium::util::MemoryMapping::is_valid() const noexcept { + return m_addr != MAP_FAILED; +} + +inline void osmium::util::MemoryMapping::make_invalid() noexcept { + m_addr = MAP_FAILED; +} + +#pragma GCC diagnostic pop + +// for BSD systems +#ifndef MAP_ANONYMOUS +# define MAP_ANONYMOUS MAP_ANON +#endif + +inline int osmium::util::MemoryMapping::get_protection() const noexcept { + if (m_mapping_mode == mapping_mode::readonly) { + return PROT_READ; + } + return PROT_READ | PROT_WRITE; +} + +inline int osmium::util::MemoryMapping::get_flags() const noexcept { + if (m_fd == -1) { + return MAP_PRIVATE | MAP_ANONYMOUS; + } + if (m_mapping_mode == mapping_mode::write_shared) { + return MAP_SHARED; + } + return MAP_PRIVATE; +} + +inline osmium::util::MemoryMapping::MemoryMapping(size_t size, mapping_mode mode, int fd, off_t offset) : + m_size(initial_size(size)), + m_offset(offset), + m_fd(resize_fd(fd)), + m_mapping_mode(mode), + m_addr(::mmap(nullptr, m_size, get_protection(), get_flags(), m_fd, m_offset)) { + assert(!(fd == -1 && mode == mapping_mode::readonly)); + if (!is_valid()) { + throw std::system_error(errno, std::system_category(), "mmap failed"); + } +} + +inline osmium::util::MemoryMapping::MemoryMapping(MemoryMapping&& other) : + m_size(other.m_size), + m_offset(other.m_offset), + m_fd(other.m_fd), + m_mapping_mode(other.m_mapping_mode), + m_addr(other.m_addr) { + other.make_invalid(); +} + +inline osmium::util::MemoryMapping& osmium::util::MemoryMapping::operator=(osmium::util::MemoryMapping&& other) { + unmap(); + m_size = other.m_size; + m_offset = other.m_offset; + m_fd = other.m_fd; + m_mapping_mode = other.m_mapping_mode; + m_addr = other.m_addr; + other.make_invalid(); + return *this; +} + +inline void osmium::util::MemoryMapping::unmap() { + if (is_valid()) { + if (::munmap(m_addr, m_size) != 0) { + throw std::system_error(errno, std::system_category(), "munmap failed"); + } + make_invalid(); + } +} + +inline void osmium::util::MemoryMapping::resize(size_t new_size) { + assert(new_size > 0 && "can not resize to zero size"); + if (m_fd == -1) { // anonymous mapping +#ifdef __linux__ + m_addr = ::mremap(m_addr, m_size, new_size, MREMAP_MAYMOVE); + if (!is_valid()) { + throw std::system_error(errno, std::system_category(), "mremap failed"); + } + m_size = new_size; +#else + assert(false && "can't resize anonymous mappings on non-linux systems"); +#endif + } else { // file-based mapping + unmap(); + m_size = new_size; + resize_fd(m_fd); + m_addr = ::mmap(nullptr, new_size, get_protection(), get_flags(), m_fd, m_offset); + if (!is_valid()) { + throw std::system_error(errno, std::system_category(), "mmap (remap) failed"); + } + } +} + +#else + +// =========== Windows implementation ============= + +/* References: + * CreateFileMapping: http://msdn.microsoft.com/en-us/library/aa366537(VS.85).aspx + * CloseHandle: http://msdn.microsoft.com/en-us/library/ms724211(VS.85).aspx + * MapViewOfFile: http://msdn.microsoft.com/en-us/library/aa366761(VS.85).aspx + * UnmapViewOfFile: http://msdn.microsoft.com/en-us/library/aa366882(VS.85).aspx + */ + +namespace osmium { + + namespace util { + + inline DWORD dword_hi(uint64_t x) { + return static_cast(x >> 32); + } + + inline DWORD dword_lo(uint64_t x) { + return static_cast(x & 0xffffffff); + } + + } // namespace util + +} // namespace osmium + +inline DWORD osmium::util::MemoryMapping::get_protection() const noexcept { + switch (m_mapping_mode) { + case mapping_mode::readonly: + return PAGE_READONLY; + case mapping_mode::write_private: + return PAGE_WRITECOPY; + case mapping_mode::write_shared: + return PAGE_READWRITE; + } +} + +inline DWORD osmium::util::MemoryMapping::get_flags() const noexcept { + switch (m_mapping_mode) { + case mapping_mode::readonly: + return FILE_MAP_READ; + case mapping_mode::write_private: + return FILE_MAP_COPY; + case mapping_mode::write_shared: + return FILE_MAP_WRITE; + } +} + +inline HANDLE osmium::util::MemoryMapping::get_handle() const noexcept { + if (m_fd == -1) { + return INVALID_HANDLE_VALUE; + } + return reinterpret_cast(_get_osfhandle(m_fd)); +} + +inline HANDLE osmium::util::MemoryMapping::create_file_mapping() const noexcept { + return CreateFileMapping(get_handle(), nullptr, get_protection(), osmium::util::dword_hi(static_cast(m_size) + m_offset), osmium::util::dword_lo(static_cast(m_size) + m_offset), nullptr); +} + +inline void* osmium::util::MemoryMapping::map_view_of_file() const noexcept { + return MapViewOfFile(m_handle, get_flags(), osmium::util::dword_hi(m_offset), osmium::util::dword_lo(m_offset), m_size); +} + +inline bool osmium::util::MemoryMapping::is_valid() const noexcept { + return m_addr != nullptr; +} + +inline void osmium::util::MemoryMapping::make_invalid() noexcept { + m_addr = nullptr; +} + +inline osmium::util::MemoryMapping::MemoryMapping(size_t size, MemoryMapping::mapping_mode mode, int fd, off_t offset) : + m_size(initial_size(size)), + m_offset(offset), + m_fd(resize_fd(fd)), + m_mapping_mode(mode), + m_handle(create_file_mapping()), + m_addr(nullptr) { + + if (!m_handle) { + throw std::system_error(GetLastError(), std::system_category(), "CreateFileMapping failed"); + } + + m_addr = map_view_of_file(); + if (!is_valid()) { + throw std::system_error(GetLastError(), std::system_category(), "MapViewOfFile failed"); + } +} + +inline osmium::util::MemoryMapping::MemoryMapping(MemoryMapping&& other) : + m_size(other.m_size), + m_offset(other.m_offset), + m_fd(other.m_fd), + m_mapping_mode(other.m_mapping_mode), + m_handle(std::move(other.m_handle)), + m_addr(other.m_addr) { + other.make_invalid(); + other.m_handle = nullptr; +} + +inline osmium::util::MemoryMapping& osmium::util::MemoryMapping::operator=(osmium::util::MemoryMapping&& other) { + unmap(); + m_size = other.m_size; + m_offset = other.m_offset; + m_fd = other.m_fd; + m_mapping_mode = other.m_mapping_mode; + m_handle = std::move(other.m_handle); + m_addr = other.m_addr; + other.make_invalid(); + other.m_handle = nullptr; + return *this; +} + +inline void osmium::util::MemoryMapping::unmap() { + if (is_valid()) { + if (! UnmapViewOfFile(m_addr)) { + throw std::system_error(GetLastError(), std::system_category(), "UnmapViewOfFile failed"); + } + make_invalid(); + } + + if (m_handle) { + if (! CloseHandle(m_handle)) { + throw std::system_error(GetLastError(), std::system_category(), "CloseHandle failed"); + } + m_handle = nullptr; + } +} + +inline void osmium::util::MemoryMapping::resize(size_t new_size) { + unmap(); + + m_size = new_size; + resize_fd(m_fd); + + m_handle = create_file_mapping(); + if (!m_handle) { + throw std::system_error(GetLastError(), std::system_category(), "CreateFileMapping failed"); + } + + m_addr = map_view_of_file(); + if (!is_valid()) { + throw std::system_error(GetLastError(), std::system_category(), "MapViewOfFile failed"); + } +} + +#endif + +#endif // OSMIUM_UTIL_MEMORY_MAPPING_HPP diff --git a/include/osmium/util/minmax.hpp b/include/osmium/util/minmax.hpp new file mode 100644 index 00000000000..2eb601a245b --- /dev/null +++ b/include/osmium/util/minmax.hpp @@ -0,0 +1,120 @@ +#ifndef OSMIUM_UTIL_MINMAX_HPP +#define OSMIUM_UTIL_MINMAX_HPP + +/* + +This file is part of Osmium (http://osmcode.org/libosmium). + +Copyright 2013-2015 Jochen Topf and others (see README). + +Boost Software License - Version 1.0 - August 17th, 2003 + +Permission is hereby granted, free of charge, to any person or organization +obtaining a copy of the software and accompanying documentation covered by +this license (the "Software") to use, reproduce, display, distribute, +execute, and transmit the Software, and to prepare derivative works of the +Software, and to permit third-parties to whom the Software is furnished to +do so, all subject to the following: + +The copyright notices in the Software and this entire statement, including +the above license grant, this restriction and the following disclaimer, +must be included in all copies of the Software, in whole or in part, and +all derivative works of the Software, unless such copies or derivative +works are solely in the form of machine-executable object code generated by +a source language processor. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT +SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE +FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, +ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. + +*/ + +#include + +namespace osmium { + + template + inline T min_op_start_value() { + return std::numeric_limits::max(); + } + + /** + * Class for calculating the minimum of a bunch of values. + * Works with any numeric type. + * + * Usage: + * + * min_op x; + * x.update(27); + * x.update(12); + * auto min = x.get(); // 12 + */ + template + class min_op { + + T m_value; + + public: + + explicit min_op(T start_value = min_op_start_value()) : + m_value(start_value) { + } + + void update(T value) noexcept { + if (value < m_value) { + m_value = value; + } + } + + T operator()() const noexcept { + return m_value; + } + + }; + + template + inline T max_op_start_value() { + return std::numeric_limits::min(); + } + + /** + * Class for calculating the maximum of a bunch of values. + * Works with any numeric type. + * + * Usage: + * + * max_op x; + * x.update(27); + * x.update(12); + * auto max = x.get(); // 27 + */ + template + class max_op { + + T m_value; + + public: + + explicit max_op(T start_value = max_op_start_value()) : + m_value(start_value) { + } + + void update(T value) noexcept { + if (value > m_value) { + m_value = value; + } + } + + T operator()() const noexcept { + return m_value; + } + + }; + +} // namespace osmium + +#endif // OSMIUM_UTIL_MINMAX_HPP diff --git a/include/osmium/util/string.hpp b/include/osmium/util/string.hpp index 54eb3616b3b..55bfc6cd5e7 100644 --- a/include/osmium/util/string.hpp +++ b/include/osmium/util/string.hpp @@ -43,21 +43,55 @@ namespace osmium { * Split string on the separator character. * * @param str The string to be split. - * @param sep The separastor character. + * @param sep The separator character. + * @param compact Set this to true to remove empty strings from result * @returns Vector with the parts of the string split up. */ - inline std::vector split_string(const std::string& str, const char sep) { + inline std::vector split_string(const std::string& str, const char sep, bool compact = false) { std::vector tokens; if (!str.empty()) { size_t pos = 0; size_t nextpos = str.find_first_of(sep); while (nextpos != std::string::npos) { - tokens.push_back(str.substr(pos, nextpos-pos)); + if (!compact || (nextpos - pos != 0)) { + tokens.push_back(str.substr(pos, nextpos-pos)); + } pos = nextpos + 1; nextpos = str.find_first_of(sep, pos); } - tokens.push_back(str.substr(pos)); + if (!compact || pos != str.size()) { + tokens.push_back(str.substr(pos)); + } + } + + return tokens; + } + + /** + * Split string on the separator character(s). + * + * @param str The string to be split. + * @param sep The separator character(s). + * @param compact Set this to true to remove empty strings from result + * @returns Vector with the parts of the string split up. + */ + inline std::vector split_string(const std::string& str, const char* sep, bool compact = false) { + std::vector tokens; + + if (!str.empty()) { + size_t pos = 0; + size_t nextpos = str.find_first_of(sep); + while (nextpos != std::string::npos) { + if (!compact || (nextpos - pos != 0)) { + tokens.push_back(str.substr(pos, nextpos-pos)); + } + pos = nextpos + 1; + nextpos = str.find_first_of(sep, pos); + } + if (!compact || pos != str.size()) { + tokens.push_back(str.substr(pos)); + } } return tokens; diff --git a/include/protozero/byteswap.hpp b/include/protozero/byteswap.hpp new file mode 100644 index 00000000000..d019c28c5cb --- /dev/null +++ b/include/protozero/byteswap.hpp @@ -0,0 +1,49 @@ +#ifndef PROTOZERO_BYTESWAP_HPP +#define PROTOZERO_BYTESWAP_HPP + +/***************************************************************************** + +protozero - Minimalistic protocol buffer decoder and encoder in C++. + +This file is from https://github.com/mapbox/protozero where you can find more +documentation. + +*****************************************************************************/ + +#include + +namespace protozero { + +template +inline void byteswap(const char* /*data*/, char* /*result*/) { + assert(false); +} + +template <> +inline void byteswap<1>(const char* data, char* result) { + result[0] = data[0]; +} + +template <> +inline void byteswap<4>(const char* data, char* result) { + result[3] = data[0]; + result[2] = data[1]; + result[1] = data[2]; + result[0] = data[3]; +} + +template <> +inline void byteswap<8>(const char* data, char* result) { + result[7] = data[0]; + result[6] = data[1]; + result[5] = data[2]; + result[4] = data[3]; + result[3] = data[4]; + result[2] = data[5]; + result[1] = data[6]; + result[0] = data[7]; +} + +} // end namespace protozero + +#endif // PROTOZERO_BYTESWAP_HPP diff --git a/include/protozero/exception.hpp b/include/protozero/exception.hpp new file mode 100644 index 00000000000..1229f7dcbb6 --- /dev/null +++ b/include/protozero/exception.hpp @@ -0,0 +1,68 @@ +#ifndef PROTOZERO_EXCEPTION_HPP +#define PROTOZERO_EXCEPTION_HPP + +/***************************************************************************** + +protozero - Minimalistic protocol buffer decoder and encoder in C++. + +This file is from https://github.com/mapbox/protozero where you can find more +documentation. + +*****************************************************************************/ + +/** + * @file exception.hpp + * + * @brief Contains the exceptions used in the protozero library. + */ + +#include + +/** + * @brief All parts of the protozero header-only library are in this namespace. + */ +namespace protozero { + +/** + * All exceptions explicitly thrown by the functions of the protozero library + * derive from this exception. + */ +struct exception : std::exception { + /// Returns the explanatory string. + const char *what() const noexcept { return "pbf exception"; } +}; + +/** + * This exception is thrown when parsing a varint thats larger than allowed. + * This should never happen unless the data is corrupted. + */ +struct varint_too_long_exception : exception { + /// Returns the explanatory string. + const char *what() const noexcept { return "varint too long exception"; } +}; + +/** + * This exception is thrown when the wire type of a pdf field is unknown. + * This should never happen unless the data is corrupted. + */ +struct unknown_pbf_wire_type_exception : exception { + /// Returns the explanatory string. + const char *what() const noexcept { return "unknown pbf field type exception"; } +}; + +/** + * This exception is thrown when we are trying to read a field and there + * are not enough bytes left in the buffer to read it. Almost all functions + * of the pbf_reader class can throw this exception. + * + * This should never happen unless the data is corrupted or you have + * initialized the pbf_reader object with incomplete data. + */ +struct end_of_buffer_exception : exception { + /// Returns the explanatory string. + const char *what() const noexcept { return "end of buffer exception"; } +}; + +} // end namespace protozero + +#endif // PROTOZERO_EXCEPTION_HPP diff --git a/include/protozero/pbf_builder.hpp b/include/protozero/pbf_builder.hpp new file mode 100644 index 00000000000..d49a7ba591c --- /dev/null +++ b/include/protozero/pbf_builder.hpp @@ -0,0 +1,111 @@ +#ifndef PROTOZERO_PBF_BUILDER_HPP +#define PROTOZERO_PBF_BUILDER_HPP + +/***************************************************************************** + +protozero - Minimalistic protocol buffer decoder and encoder in C++. + +This file is from https://github.com/mapbox/protozero where you can find more +documentation. + +*****************************************************************************/ + +#include + +#include +#include + +namespace protozero { + +template +class pbf_builder : public pbf_writer { + + static_assert(std::is_same::type>::value, "T must be enum with underlying type protozero::pbf_tag_type"); + +public: + + using enum_type = T; + + pbf_builder(std::string& data) noexcept : + pbf_writer(data) { + } + + template + pbf_builder(pbf_writer& parent_writer, P tag) noexcept : + pbf_writer(parent_writer, pbf_tag_type(tag)) { + } + +#define PROTOZERO_WRITER_WRAP_ADD_SCALAR(name, type) \ + inline void add_##name(T tag, type value) { \ + pbf_writer::add_##name(pbf_tag_type(tag), value); \ + } + + PROTOZERO_WRITER_WRAP_ADD_SCALAR(bool, bool) + PROTOZERO_WRITER_WRAP_ADD_SCALAR(enum, int32_t) + PROTOZERO_WRITER_WRAP_ADD_SCALAR(int32, int32_t) + PROTOZERO_WRITER_WRAP_ADD_SCALAR(sint32, int32_t) + PROTOZERO_WRITER_WRAP_ADD_SCALAR(uint32, uint32_t) + PROTOZERO_WRITER_WRAP_ADD_SCALAR(int64, int64_t) + PROTOZERO_WRITER_WRAP_ADD_SCALAR(sint64, int64_t) + PROTOZERO_WRITER_WRAP_ADD_SCALAR(uint64, uint64_t) + PROTOZERO_WRITER_WRAP_ADD_SCALAR(fixed32, uint32_t) + PROTOZERO_WRITER_WRAP_ADD_SCALAR(sfixed32, int32_t) + PROTOZERO_WRITER_WRAP_ADD_SCALAR(fixed64, uint64_t) + PROTOZERO_WRITER_WRAP_ADD_SCALAR(sfixed64, int64_t) + PROTOZERO_WRITER_WRAP_ADD_SCALAR(float, float) + PROTOZERO_WRITER_WRAP_ADD_SCALAR(double, double) + + inline void add_bytes(T tag, const char* value, size_t size) { + pbf_writer::add_bytes(pbf_tag_type(tag), value, size); + } + + inline void add_bytes(T tag, const std::string& value) { + pbf_writer::add_bytes(pbf_tag_type(tag), value); + } + + inline void add_string(T tag, const char* value, size_t size) { + pbf_writer::add_string(pbf_tag_type(tag), value, size); + } + + inline void add_string(T tag, const std::string& value) { + pbf_writer::add_string(pbf_tag_type(tag), value); + } + + inline void add_string(T tag, const char* value) { + pbf_writer::add_string(pbf_tag_type(tag), value); + } + + inline void add_message(T tag, const char* value, size_t size) { + pbf_writer::add_message(pbf_tag_type(tag), value, size); + } + + inline void add_message(T tag, const std::string& value) { + pbf_writer::add_message(pbf_tag_type(tag), value); + } + +#define PROTOZERO_WRITER_WRAP_ADD_PACKED(name) \ + template \ + inline void add_packed_##name(T tag, InputIterator first, InputIterator last) { \ + pbf_writer::add_packed_##name(pbf_tag_type(tag), first, last); \ + } + + PROTOZERO_WRITER_WRAP_ADD_PACKED(bool) + PROTOZERO_WRITER_WRAP_ADD_PACKED(enum) + PROTOZERO_WRITER_WRAP_ADD_PACKED(int32) + PROTOZERO_WRITER_WRAP_ADD_PACKED(sint32) + PROTOZERO_WRITER_WRAP_ADD_PACKED(uint32) + PROTOZERO_WRITER_WRAP_ADD_PACKED(int64) + PROTOZERO_WRITER_WRAP_ADD_PACKED(sint64) + PROTOZERO_WRITER_WRAP_ADD_PACKED(uint64) + PROTOZERO_WRITER_WRAP_ADD_PACKED(fixed32) + PROTOZERO_WRITER_WRAP_ADD_PACKED(sfixed32) + PROTOZERO_WRITER_WRAP_ADD_PACKED(fixed64) + PROTOZERO_WRITER_WRAP_ADD_PACKED(sfixed64) + PROTOZERO_WRITER_WRAP_ADD_PACKED(float) + PROTOZERO_WRITER_WRAP_ADD_PACKED(double) + +}; + +} // end namespace protozero + +#endif // PROTOZERO_PBF_BUILDER_HPP diff --git a/include/protozero/pbf_message.hpp b/include/protozero/pbf_message.hpp new file mode 100644 index 00000000000..af29a00f443 --- /dev/null +++ b/include/protozero/pbf_message.hpp @@ -0,0 +1,50 @@ +#ifndef PROTOZERO_PBF_MESSAGE_HPP +#define PROTOZERO_PBF_MESSAGE_HPP + +/***************************************************************************** + +protozero - Minimalistic protocol buffer decoder and encoder in C++. + +This file is from https://github.com/mapbox/protozero where you can find more +documentation. + +*****************************************************************************/ + +#include + +#include +#include + +namespace protozero { + +template +class pbf_message : public pbf_reader { + + static_assert(std::is_same::type>::value, "T must be enum with underlying type protozero::pbf_tag_type"); + +public: + + using enum_type = T; + + template + pbf_message(Args&&... args) noexcept : + pbf_reader(std::forward(args)...) { + } + + inline bool next() { + return pbf_reader::next(); + } + + inline bool next(T tag) { + return pbf_reader::next(pbf_tag_type(tag)); + } + + inline T tag() const noexcept { + return T(pbf_reader::tag()); + } + +}; + +} // end namespace protozero + +#endif // PROTOZERO_PBF_MESSAGE_HPP diff --git a/include/protozero/pbf_reader.hpp b/include/protozero/pbf_reader.hpp new file mode 100644 index 00000000000..1c5ed0d7024 --- /dev/null +++ b/include/protozero/pbf_reader.hpp @@ -0,0 +1,1059 @@ +#ifndef PROTOZERO_PBF_READER_HPP +#define PROTOZERO_PBF_READER_HPP + +/***************************************************************************** + +protozero - Minimalistic protocol buffer decoder and encoder in C++. + +This file is from https://github.com/mapbox/protozero where you can find more +documentation. + +*****************************************************************************/ + +/** + * @file pbf_reader.hpp + * + * @brief Contains the pbf_reader class. + */ + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#if __BYTE_ORDER != __LITTLE_ENDIAN +# include +#endif + +/// Wrapper for assert() used for testing +#ifndef protozero_assert +# define protozero_assert(x) assert(x) +#endif + +namespace protozero { + +/** + * This class represents a protobuf message. Either a top-level message or + * a nested sub-message. Top-level messages can be created from any buffer + * with a pointer and length: + * + * @code + * std::string buffer; + * // fill buffer... + * pbf_reader message(buffer.data(), buffer.size()); + * @endcode + * + * Sub-messages are created using get_message(): + * + * @code + * pbf_reader message(...); + * message.next(); + * pbf_reader submessage = message.get_message(); + * @endcode + * + * All methods of the pbf_reader class except get_bytes() and get_string() + * provide the strong exception guarantee, ie they either succeed or do not + * change the pbf_reader object they are called on. Use the get_data() method + * instead of get_bytes() or get_string(), if you need this guarantee. + */ +class pbf_reader { + + // A pointer to the next unread data. + const char *m_data = nullptr; + + // A pointer to one past the end of data. + const char *m_end = nullptr; + + // The wire type of the current field. + pbf_wire_type m_wire_type = pbf_wire_type::unknown; + + // The tag of the current field. + pbf_tag_type m_tag = 0; + + template + inline T get_fixed() { + T result; + skip_bytes(sizeof(T)); +#if __BYTE_ORDER == __LITTLE_ENDIAN + memcpy(&result, m_data - sizeof(T), sizeof(T)); +#else + byteswap(m_data - sizeof(T), reinterpret_cast(&result)); +#endif + return result; + } + +#if __BYTE_ORDER == __LITTLE_ENDIAN + template + inline std::pair packed_fixed() { + protozero_assert(tag() != 0 && "call next() before accessing field value"); + auto len = get_len_and_skip(); + protozero_assert(len % sizeof(T) == 0); + return std::make_pair(reinterpret_cast(m_data-len), reinterpret_cast(m_data)); + } + +#else + + template + class const_fixed_iterator : public std::iterator { + + const char* m_data; + const char* m_end; + + public: + + const_fixed_iterator() noexcept : + m_data(nullptr), + m_end(nullptr) { + } + + const_fixed_iterator(const char *data, const char* end) noexcept : + m_data(data), + m_end(end) { + } + + const_fixed_iterator(const const_fixed_iterator&) noexcept = default; + const_fixed_iterator(const_fixed_iterator&&) noexcept = default; + + const_fixed_iterator& operator=(const const_fixed_iterator&) noexcept = default; + const_fixed_iterator& operator=(const_fixed_iterator&&) noexcept = default; + + ~const_fixed_iterator() noexcept = default; + + T operator*() { + T result; + byteswap(m_data, reinterpret_cast(&result)); + return result; + } + + const_fixed_iterator& operator++() { + m_data += sizeof(T); + return *this; + } + + const_fixed_iterator operator++(int) { + const const_fixed_iterator tmp(*this); + ++(*this); + return tmp; + } + + bool operator==(const const_fixed_iterator& rhs) const noexcept { + return m_data == rhs.m_data && m_end == rhs.m_end; + } + + bool operator!=(const const_fixed_iterator& rhs) const noexcept { + return !(*this == rhs); + } + + }; // class const_fixed_iterator + + template + inline std::pair, const_fixed_iterator> packed_fixed() { + protozero_assert(tag() != 0 && "call next() before accessing field value"); + auto len = get_len_and_skip(); + protozero_assert(len % sizeof(T) == 0); + return std::make_pair(const_fixed_iterator(m_data-len, m_data), + const_fixed_iterator(m_data, m_data)); + } +#endif + + template inline T get_varint(); + template inline T get_svarint(); + + inline pbf_length_type get_length() { return get_varint(); } + + inline void skip_bytes(pbf_length_type len); + + inline pbf_length_type get_len_and_skip(); + +public: + + /** + * Construct a pbf_reader message from a data pointer and a length. The pointer + * will be stored inside the pbf_reader object, no data is copied. So you must + * make sure the buffer stays valid as long as the pbf_reader object is used. + * + * The buffer must contain a complete protobuf message. + * + * @post There is no current field. + */ + inline pbf_reader(const char *data, size_t length) noexcept; + + /** + * Construct a pbf_reader message from a data pointer and a length. The pointer + * will be stored inside the pbf_reader object, no data is copied. So you must + * make sure the buffer stays valid as long as the pbf_reader object is used. + * + * The buffer must contain a complete protobuf message. + * + * @post There is no current field. + */ + inline pbf_reader(std::pair data) noexcept; + + /** + * Construct a pbf_reader message from a std::string. A pointer to the string + * internals will be stored inside the pbf_reader object, no data is copied. + * So you must make sure the string is unchanged as long as the pbf_reader + * object is used. + * + * The string must contain a complete protobuf message. + * + * @post There is no current field. + */ + inline pbf_reader(const std::string& data) noexcept; + + /** + * pbf_reader can be default constructed and behaves like it has an empty + * buffer. + */ + inline pbf_reader() noexcept = default; + + /// pbf_reader messages can be copied trivially. + inline pbf_reader(const pbf_reader&) noexcept = default; + + /// pbf_reader messages can be moved trivially. + inline pbf_reader(pbf_reader&&) noexcept = default; + + /// pbf_reader messages can be copied trivially. + inline pbf_reader& operator=(const pbf_reader& other) noexcept = default; + + /// pbf_reader messages can be moved trivially. + inline pbf_reader& operator=(pbf_reader&& other) noexcept = default; + + inline ~pbf_reader() = default; + + /** + * In a boolean context the pbf_reader class evaluates to `true` if there are + * still fields available and to `false` if the last field has been read. + */ + inline operator bool() const noexcept; + + /** + * Return the length in bytes of the current message. If you have + * already called next() and/or any of the get_*() functions, this will + * return the remaining length. + * + * This can, for instance, be used to estimate the space needed for a + * buffer. Of course you have to know reasonably well what data to expect + * and how it is encoded for this number to have any meaning. + */ + size_t length() const noexcept { + return size_t(m_end - m_data); + } + + /** + * Set next field in the message as the current field. This is usually + * called in a while loop: + * + * @code + * pbf_reader message(...); + * while (message.next()) { + * // handle field + * } + * @endcode + * + * @returns `true` if there is a next field, `false` if not. + * @pre There must be no current field. + * @post If it returns `true` there is a current field now. + */ + inline bool next(); + + /** + * Set next field with given tag in the message as the current field. + * Fields with other tags are skipped. This is usually called in a while + * loop for repeated fields: + * + * @code + * pbf_reader message(...); + * while (message.next(17)) { + * // handle field + * } + * @endcode + * + * or you can call it just once to get the one field with this tag: + * + * @code + * pbf_reader message(...); + * if (message.next(17)) { + * // handle field + * } + * @endcode + * + * @returns `true` if there is a next field with this tag. + * @pre There must be no current field. + * @post If it returns `true` there is a current field now with the given tag. + */ + inline bool next(pbf_tag_type tag); + + /** + * The tag of the current field. The tag is the field number from the + * description in the .proto file. + * + * Call next() before calling this function to set the current field. + * + * @returns tag of the current field. + * @pre There must be a current field (ie. next() must have returned `true`). + */ + inline pbf_tag_type tag() const noexcept; + + /** + * Get the wire type of the current field. The wire types are: + * + * * 0 - varint + * * 1 - 64 bit + * * 2 - length-delimited + * * 5 - 32 bit + * + * All other types are illegal. + * + * Call next() before calling this function to set the current field. + * + * @returns wire type of the current field. + * @pre There must be a current field (ie. next() must have returned `true`). + */ + inline pbf_wire_type wire_type() const noexcept; + + /** + * Check the wire type of the current field. + * + * @returns `true` if the current field has the given wire type. + * @pre There must be a current field (ie. next() must have returned `true`). + */ + inline bool has_wire_type(pbf_wire_type type) const noexcept; + + /** + * Consume the current field. + * + * @pre There must be a current field (ie. next() must have returned `true`). + * @post The current field was consumed and there is no current field now. + */ + inline void skip(); + + ///@{ + /** + * @name Scalar field accessor functions + */ + + /** + * Consume and return value of current "bool" field. + * + * @pre There must be a current field (ie. next() must have returned `true`). + * @pre The current field must be of type "bool". + * @post The current field was consumed and there is no current field now. + */ + inline bool get_bool(); + + /** + * Consume and return value of current "enum" field. + * + * @pre There must be a current field (ie. next() must have returned `true`). + * @pre The current field must be of type "enum". + * @post The current field was consumed and there is no current field now. + */ + inline int32_t get_enum() { + protozero_assert(has_wire_type(pbf_wire_type::varint) && "not a varint"); + return get_varint(); + } + + /** + * Consume and return value of current "int32" varint field. + * + * @pre There must be a current field (ie. next() must have returned `true`). + * @pre The current field must be of type "int32". + * @post The current field was consumed and there is no current field now. + */ + inline int32_t get_int32() { + protozero_assert(has_wire_type(pbf_wire_type::varint) && "not a varint"); + return get_varint(); + } + + /** + * Consume and return value of current "sint32" varint field. + * + * @pre There must be a current field (ie. next() must have returned `true`). + * @pre The current field must be of type "sint32". + * @post The current field was consumed and there is no current field now. + */ + inline int32_t get_sint32() { + protozero_assert(has_wire_type(pbf_wire_type::varint) && "not a varint"); + return get_svarint(); + } + + /** + * Consume and return value of current "uint32" varint field. + * + * @pre There must be a current field (ie. next() must have returned `true`). + * @pre The current field must be of type "uint32". + * @post The current field was consumed and there is no current field now. + */ + inline uint32_t get_uint32() { + protozero_assert(has_wire_type(pbf_wire_type::varint) && "not a varint"); + return get_varint(); + } + + /** + * Consume and return value of current "int64" varint field. + * + * @pre There must be a current field (ie. next() must have returned `true`). + * @pre The current field must be of type "int64". + * @post The current field was consumed and there is no current field now. + */ + inline int64_t get_int64() { + protozero_assert(has_wire_type(pbf_wire_type::varint) && "not a varint"); + return get_varint(); + } + + /** + * Consume and return value of current "sint64" varint field. + * + * @pre There must be a current field (ie. next() must have returned `true`). + * @pre The current field must be of type "sint64". + * @post The current field was consumed and there is no current field now. + */ + inline int64_t get_sint64() { + protozero_assert(has_wire_type(pbf_wire_type::varint) && "not a varint"); + return get_svarint(); + } + + /** + * Consume and return value of current "uint64" varint field. + * + * @pre There must be a current field (ie. next() must have returned `true`). + * @pre The current field must be of type "uint64". + * @post The current field was consumed and there is no current field now. + */ + inline uint64_t get_uint64() { + protozero_assert(has_wire_type(pbf_wire_type::varint) && "not a varint"); + return get_varint(); + } + + /** + * Consume and return value of current "fixed32" field. + * + * @pre There must be a current field (ie. next() must have returned `true`). + * @pre The current field must be of type "fixed32". + * @post The current field was consumed and there is no current field now. + */ + inline uint32_t get_fixed32(); + + /** + * Consume and return value of current "sfixed32" field. + * + * @pre There must be a current field (ie. next() must have returned `true`). + * @pre The current field must be of type "sfixed32". + * @post The current field was consumed and there is no current field now. + */ + inline int32_t get_sfixed32(); + + /** + * Consume and return value of current "fixed64" field. + * + * @pre There must be a current field (ie. next() must have returned `true`). + * @pre The current field must be of type "fixed64". + * @post The current field was consumed and there is no current field now. + */ + inline uint64_t get_fixed64(); + + /** + * Consume and return value of current "sfixed64" field. + * + * @pre There must be a current field (ie. next() must have returned `true`). + * @pre The current field must be of type "sfixed64". + * @post The current field was consumed and there is no current field now. + */ + inline int64_t get_sfixed64(); + + /** + * Consume and return value of current "float" field. + * + * @pre There must be a current field (ie. next() must have returned `true`). + * @pre The current field must be of type "float". + * @post The current field was consumed and there is no current field now. + */ + inline float get_float(); + + /** + * Consume and return value of current "double" field. + * + * @pre There must be a current field (ie. next() must have returned `true`). + * @pre The current field must be of type "double". + * @post The current field was consumed and there is no current field now. + */ + inline double get_double(); + + /** + * Consume and return value of current "bytes" or "string" field. + * + * @returns A pair with a pointer to the data and the length of the data. + * @pre There must be a current field (ie. next() must have returned `true`). + * @pre The current field must be of type "bytes" or "string". + * @post The current field was consumed and there is no current field now. + */ + inline std::pair get_data(); + + /** + * Consume and return value of current "bytes" field. + * + * @pre There must be a current field (ie. next() must have returned `true`). + * @pre The current field must be of type "bytes". + * @post The current field was consumed and there is no current field now. + */ + inline std::string get_bytes(); + + /** + * Consume and return value of current "string" field. + * + * @pre There must be a current field (ie. next() must have returned `true`). + * @pre The current field must be of type "string". + * @post The current field was consumed and there is no current field now. + */ + inline std::string get_string(); + + /** + * Consume and return value of current "message" field. + * + * @pre There must be a current field (ie. next() must have returned `true`). + * @pre The current field must be of type "message". + * @post The current field was consumed and there is no current field now. + */ + inline pbf_reader get_message() { + return pbf_reader(get_data()); + } + + ///@} + +private: + + template + class const_varint_iterator : public std::iterator { + + protected: + + const char* m_data; + const char* m_end; + + public: + + const_varint_iterator() noexcept : + m_data(nullptr), + m_end(nullptr) { + } + + const_varint_iterator(const char *data, const char* end) noexcept : + m_data(data), + m_end(end) { + } + + const_varint_iterator(const const_varint_iterator&) noexcept = default; + const_varint_iterator(const_varint_iterator&&) noexcept = default; + + const_varint_iterator& operator=(const const_varint_iterator&) noexcept = default; + const_varint_iterator& operator=(const_varint_iterator&&) noexcept = default; + + ~const_varint_iterator() noexcept = default; + + T operator*() { + const char* d = m_data; // will be thrown away + return static_cast(decode_varint(&d, m_end)); + } + + const_varint_iterator& operator++() { + // Ignore the result, we call decode_varint() just for the + // side-effect of updating m_data. + decode_varint(&m_data, m_end); + return *this; + } + + const_varint_iterator operator++(int) { + const const_varint_iterator tmp(*this); + ++(*this); + return tmp; + } + + bool operator==(const const_varint_iterator& rhs) const noexcept { + return m_data == rhs.m_data && m_end == rhs.m_end; + } + + bool operator!=(const const_varint_iterator& rhs) const noexcept { + return !(*this == rhs); + } + + }; // class const_varint_iterator + + template + class const_svarint_iterator : public const_varint_iterator { + + public: + + const_svarint_iterator() noexcept : + const_varint_iterator() { + } + + const_svarint_iterator(const char *data, const char* end) noexcept : + const_varint_iterator(data, end) { + } + + const_svarint_iterator(const const_svarint_iterator&) = default; + const_svarint_iterator(const_svarint_iterator&&) = default; + + const_svarint_iterator& operator=(const const_svarint_iterator&) = default; + const_svarint_iterator& operator=(const_svarint_iterator&&) = default; + + ~const_svarint_iterator() = default; + + T operator*() { + const char* d = this->m_data; // will be thrown away + return static_cast(decode_zigzag64(decode_varint(&d, this->m_end))); + } + + const_svarint_iterator& operator++() { + // Ignore the result, we call decode_varint() just for the + // side-effect of updating m_data. + decode_varint(&this->m_data, this->m_end); + return *this; + } + + const_svarint_iterator operator++(int) { + const const_svarint_iterator tmp(*this); + ++(*this); + return tmp; + } + + }; // class const_svarint_iterator + +public: + + /// Forward iterator for iterating over bool (int32 varint) values. + typedef const_varint_iterator< int32_t> const_bool_iterator; + + /// Forward iterator for iterating over enum (int32 varint) values. + typedef const_varint_iterator< int32_t> const_enum_iterator; + + /// Forward iterator for iterating over int32 (varint) values. + typedef const_varint_iterator< int32_t> const_int32_iterator; + + /// Forward iterator for iterating over sint32 (varint) values. + typedef const_svarint_iterator const_sint32_iterator; + + /// Forward iterator for iterating over uint32 (varint) values. + typedef const_varint_iterator const_uint32_iterator; + + /// Forward iterator for iterating over int64 (varint) values. + typedef const_varint_iterator< int64_t> const_int64_iterator; + + /// Forward iterator for iterating over sint64 (varint) values. + typedef const_svarint_iterator const_sint64_iterator; + + /// Forward iterator for iterating over uint64 (varint) values. + typedef const_varint_iterator const_uint64_iterator; + + ///@{ + /** + * @name Repeated packed field accessor functions + */ + + /** + * Consume current "repeated packed bool" field. + * + * @returns a pair of iterators to the beginning and one past the end of + * the data. + * @pre There must be a current field (ie. next() must have returned `true`). + * @pre The current field must be of type "repeated packed bool". + * @post The current field was consumed and there is no current field now. + */ + inline std::pair get_packed_bool(); + + /** + * Consume current "repeated packed enum" field. + * + * @returns a pair of iterators to the beginning and one past the end of + * the data. + * @pre There must be a current field (ie. next() must have returned `true`). + * @pre The current field must be of type "repeated packed enum". + * @post The current field was consumed and there is no current field now. + */ + inline std::pair get_packed_enum(); + + /** + * Consume current "repeated packed int32" field. + * + * @returns a pair of iterators to the beginning and one past the end of + * the data. + * @pre There must be a current field (ie. next() must have returned `true`). + * @pre The current field must be of type "repeated packed int32". + * @post The current field was consumed and there is no current field now. + */ + inline std::pair get_packed_int32(); + + /** + * Consume current "repeated packed sint32" field. + * + * @returns a pair of iterators to the beginning and one past the end of + * the data. + * @pre There must be a current field (ie. next() must have returned `true`). + * @pre The current field must be of type "repeated packed sint32". + * @post The current field was consumed and there is no current field now. + */ + inline std::pair get_packed_sint32(); + + /** + * Consume current "repeated packed uint32" field. + * + * @returns a pair of iterators to the beginning and one past the end of + * the data. + * @pre There must be a current field (ie. next() must have returned `true`). + * @pre The current field must be of type "repeated packed uint32". + * @post The current field was consumed and there is no current field now. + */ + inline std::pair get_packed_uint32(); + + /** + * Consume current "repeated packed int64" field. + * + * @returns a pair of iterators to the beginning and one past the end of + * the data. + * @pre There must be a current field (ie. next() must have returned `true`). + * @pre The current field must be of type "repeated packed int64". + * @post The current field was consumed and there is no current field now. + */ + inline std::pair get_packed_int64(); + + /** + * Consume current "repeated packed sint64" field. + * + * @returns a pair of iterators to the beginning and one past the end of + * the data. + * @pre There must be a current field (ie. next() must have returned `true`). + * @pre The current field must be of type "repeated packed sint64". + * @post The current field was consumed and there is no current field now. + */ + inline std::pair get_packed_sint64(); + + /** + * Consume current "repeated packed uint64" field. + * + * @returns a pair of iterators to the beginning and one past the end of + * the data. + * @pre There must be a current field (ie. next() must have returned `true`). + * @pre The current field must be of type "repeated packed uint64". + * @post The current field was consumed and there is no current field now. + */ + inline std::pair get_packed_uint64(); + + /** + * Consume current "repeated packed fixed32" field. + * + * @returns a pair of iterators to the beginning and one past the end of + * the data. + * @pre There must be a current field (ie. next() must have returned `true`). + * @pre The current field must be of type "repeated packed fixed32". + * @post The current field was consumed and there is no current field now. + */ + inline auto get_packed_fixed32() -> decltype(packed_fixed()) { + return packed_fixed(); + } + + /** + * Consume current "repeated packed sfixed32" field. + * + * @returns a pair of iterators to the beginning and one past the end of + * the data. + * @pre There must be a current field (ie. next() must have returned `true`). + * @pre The current field must be of type "repeated packed sfixed32". + * @post The current field was consumed and there is no current field now. + */ + inline auto get_packed_sfixed32() -> decltype(packed_fixed()) { + return packed_fixed(); + } + + /** + * Consume current "repeated packed fixed64" field. + * + * @returns a pair of iterators to the beginning and one past the end of + * the data. + * @pre There must be a current field (ie. next() must have returned `true`). + * @pre The current field must be of type "repeated packed fixed64". + * @post The current field was consumed and there is no current field now. + */ + inline auto get_packed_fixed64() -> decltype(packed_fixed()) { + return packed_fixed(); + } + + /** + * Consume current "repeated packed sfixed64" field. + * + * @returns a pair of iterators to the beginning and one past the end of + * the data. + * @pre There must be a current field (ie. next() must have returned `true`). + * @pre The current field must be of type "repeated packed sfixed64". + * @post The current field was consumed and there is no current field now. + */ + inline auto get_packed_sfixed64() -> decltype(packed_fixed()) { + return packed_fixed(); + } + + /** + * Consume current "repeated packed float" field. + * + * @returns a pair of iterators to the beginning and one past the end of + * the data. + * @pre There must be a current field (ie. next() must have returned `true`). + * @pre The current field must be of type "repeated packed float". + * @post The current field was consumed and there is no current field now. + */ + inline auto get_packed_float() -> decltype(packed_fixed()) { + return packed_fixed(); + } + + /** + * Consume current "repeated packed double" field. + * + * @returns a pair of iterators to the beginning and one past the end of + * the data. + * @pre There must be a current field (ie. next() must have returned `true`). + * @pre The current field must be of type "repeated packed double". + * @post The current field was consumed and there is no current field now. + */ + inline auto get_packed_double() -> decltype(packed_fixed()) { + return packed_fixed(); + } + + ///@} + +}; // class pbf_reader + +pbf_reader::pbf_reader(const char *data, size_t length) noexcept + : m_data(data), + m_end(data + length), + m_wire_type(pbf_wire_type::unknown), + m_tag(0) { +} + +pbf_reader::pbf_reader(std::pair data) noexcept + : m_data(data.first), + m_end(data.first + data.second), + m_wire_type(pbf_wire_type::unknown), + m_tag(0) { +} + +pbf_reader::pbf_reader(const std::string& data) noexcept + : m_data(data.data()), + m_end(data.data() + data.size()), + m_wire_type(pbf_wire_type::unknown), + m_tag(0) { +} + +pbf_reader::operator bool() const noexcept { + return m_data < m_end; +} + +bool pbf_reader::next() { + if (m_data == m_end) { + return false; + } + + auto value = get_varint(); + m_tag = value >> 3; + + // tags 0 and 19000 to 19999 are not allowed as per + // https://developers.google.com/protocol-buffers/docs/proto + protozero_assert(((m_tag > 0 && m_tag < 19000) || (m_tag > 19999 && m_tag <= ((1 << 29) - 1))) && "tag out of range"); + + m_wire_type = pbf_wire_type(value & 0x07); +// XXX do we want this check? or should it throw an exception? +// protozero_assert((m_wire_type <=2 || m_wire_type == 5) && "illegal wire type"); + return true; +} + +bool pbf_reader::next(pbf_tag_type requested_tag) { + while (next()) { + if (m_tag == requested_tag) { + return true; + } else { + skip(); + } + } + return false; +} + +pbf_tag_type pbf_reader::tag() const noexcept { + return m_tag; +} + +pbf_wire_type pbf_reader::wire_type() const noexcept { + return m_wire_type; +} + +bool pbf_reader::has_wire_type(pbf_wire_type type) const noexcept { + return wire_type() == type; +} + +void pbf_reader::skip_bytes(pbf_length_type len) { + if (m_data + len > m_end) { + throw end_of_buffer_exception(); + } + m_data += len; + +// In debug builds reset the tag to zero so that we can detect (some) +// wrong code. +#ifndef NDEBUG + m_tag = 0; +#endif +} + +void pbf_reader::skip() { + protozero_assert(tag() != 0 && "call next() before calling skip()"); + switch (wire_type()) { + case pbf_wire_type::varint: + (void)get_uint32(); // called for the side-effect of skipping value + break; + case pbf_wire_type::fixed64: + skip_bytes(8); + break; + case pbf_wire_type::length_delimited: + skip_bytes(get_length()); + break; + case pbf_wire_type::fixed32: + skip_bytes(4); + break; + default: + throw unknown_pbf_wire_type_exception(); + } +} + +pbf_length_type pbf_reader::get_len_and_skip() { + auto len = get_length(); + skip_bytes(len); + return len; +} + +template +T pbf_reader::get_varint() { + return static_cast(decode_varint(&m_data, m_end)); +} + +template +T pbf_reader::get_svarint() { + protozero_assert((has_wire_type(pbf_wire_type::varint) || has_wire_type(pbf_wire_type::length_delimited)) && "not a varint"); + return static_cast(decode_zigzag64(decode_varint(&m_data, m_end))); +} + +uint32_t pbf_reader::get_fixed32() { + protozero_assert(tag() != 0 && "call next() before accessing field value"); + protozero_assert(has_wire_type(pbf_wire_type::fixed32) && "not a 32-bit fixed"); + return get_fixed(); +} + +int32_t pbf_reader::get_sfixed32() { + protozero_assert(tag() != 0 && "call next() before accessing field value"); + protozero_assert(has_wire_type(pbf_wire_type::fixed32) && "not a 32-bit fixed"); + return get_fixed(); +} + +uint64_t pbf_reader::get_fixed64() { + protozero_assert(tag() != 0 && "call next() before accessing field value"); + protozero_assert(has_wire_type(pbf_wire_type::fixed64) && "not a 64-bit fixed"); + return get_fixed(); +} + +int64_t pbf_reader::get_sfixed64() { + protozero_assert(tag() != 0 && "call next() before accessing field value"); + protozero_assert(has_wire_type(pbf_wire_type::fixed64) && "not a 64-bit fixed"); + return get_fixed(); +} + +float pbf_reader::get_float() { + protozero_assert(tag() != 0 && "call next() before accessing field value"); + protozero_assert(has_wire_type(pbf_wire_type::fixed32) && "not a 32-bit fixed"); + return get_fixed(); +} + +double pbf_reader::get_double() { + protozero_assert(tag() != 0 && "call next() before accessing field value"); + protozero_assert(has_wire_type(pbf_wire_type::fixed64) && "not a 64-bit fixed"); + return get_fixed(); +} + +bool pbf_reader::get_bool() { + protozero_assert(tag() != 0 && "call next() before accessing field value"); + protozero_assert(has_wire_type(pbf_wire_type::varint) && "not a varint"); + protozero_assert((*m_data & 0x80) == 0 && "not a 1 byte varint"); + skip_bytes(1); + return m_data[-1] != 0; // -1 okay because we incremented m_data the line before +} + +std::pair pbf_reader::get_data() { + protozero_assert(tag() != 0 && "call next() before accessing field value"); + protozero_assert(has_wire_type(pbf_wire_type::length_delimited) && "not of type string, bytes or message"); + auto len = get_len_and_skip(); + return std::make_pair(m_data-len, len); +} + +std::string pbf_reader::get_bytes() { + auto d = get_data(); + return std::string(d.first, d.second); +} + +std::string pbf_reader::get_string() { + return get_bytes(); +} + +std::pair pbf_reader::get_packed_bool() { + return get_packed_int32(); +} + +std::pair pbf_reader::get_packed_enum() { + return get_packed_int32(); +} + +std::pair pbf_reader::get_packed_int32() { + protozero_assert(tag() != 0 && "call next() before accessing field value"); + auto len = get_len_and_skip(); + return std::make_pair(pbf_reader::const_int32_iterator(m_data-len, m_data), + pbf_reader::const_int32_iterator(m_data, m_data)); +} + +std::pair pbf_reader::get_packed_uint32() { + protozero_assert(tag() != 0 && "call next() before accessing field value"); + auto len = get_len_and_skip(); + return std::make_pair(pbf_reader::const_uint32_iterator(m_data-len, m_data), + pbf_reader::const_uint32_iterator(m_data, m_data)); +} + +std::pair pbf_reader::get_packed_sint32() { + protozero_assert(tag() != 0 && "call next() before accessing field value"); + auto len = get_len_and_skip(); + return std::make_pair(pbf_reader::const_sint32_iterator(m_data-len, m_data), + pbf_reader::const_sint32_iterator(m_data, m_data)); +} + +std::pair pbf_reader::get_packed_int64() { + protozero_assert(tag() != 0 && "call next() before accessing field value"); + auto len = get_len_and_skip(); + return std::make_pair(pbf_reader::const_int64_iterator(m_data-len, m_data), + pbf_reader::const_int64_iterator(m_data, m_data)); +} + +std::pair pbf_reader::get_packed_uint64() { + protozero_assert(tag() != 0 && "call next() before accessing field value"); + auto len = get_len_and_skip(); + return std::make_pair(pbf_reader::const_uint64_iterator(m_data-len, m_data), + pbf_reader::const_uint64_iterator(m_data, m_data)); +} + +std::pair pbf_reader::get_packed_sint64() { + protozero_assert(tag() != 0 && "call next() before accessing field value"); + auto len = get_len_and_skip(); + return std::make_pair(pbf_reader::const_sint64_iterator(m_data-len, m_data), + pbf_reader::const_sint64_iterator(m_data, m_data)); +} + +} // end namespace protozero + +#endif // PROTOZERO_PBF_READER_HPP diff --git a/include/protozero/pbf_types.hpp b/include/protozero/pbf_types.hpp new file mode 100644 index 00000000000..9f38584f439 --- /dev/null +++ b/include/protozero/pbf_types.hpp @@ -0,0 +1,49 @@ +#ifndef PROTOZERO_PBF_TYPES_HPP +#define PROTOZERO_PBF_TYPES_HPP + +/***************************************************************************** + +protozero - Minimalistic protocol buffer decoder and encoder in C++. + +This file is from https://github.com/mapbox/protozero where you can find more +documentation. + +*****************************************************************************/ + +/** + * @file pbf_types.hpp + * + * @brief Contains the declaration of low-level types used in the pbf format. + */ + +#include + +namespace protozero { + + /** + * The type used for field tags (field numbers). + */ + typedef uint32_t pbf_tag_type; + + /** + * The type used to encode type information. + * See the table on + * https://developers.google.com/protocol-buffers/docs/encoding + */ + enum class pbf_wire_type : uint32_t { + varint = 0, // int32/64, uint32/64, sint32/64, bool, enum + fixed64 = 1, // fixed64, sfixed64, double + length_delimited = 2, // string, bytes, embedded messages, + // packed repeated fields + fixed32 = 5, // fixed32, sfixed32, float + unknown = 99 // used for default setting in this library + }; + + /** + * The type used for length values, such as the length of a field. + */ + typedef uint32_t pbf_length_type; + +} // end namespace protozero + +#endif // PROTOZERO_PBF_TYPES_HPP diff --git a/include/protozero/pbf_writer.hpp b/include/protozero/pbf_writer.hpp new file mode 100644 index 00000000000..53cbfdf1c49 --- /dev/null +++ b/include/protozero/pbf_writer.hpp @@ -0,0 +1,664 @@ +#ifndef PROTOZERO_PBF_WRITER_HPP +#define PROTOZERO_PBF_WRITER_HPP + +/***************************************************************************** + +protozero - Minimalistic protocol buffer decoder and encoder in C++. + +This file is from https://github.com/mapbox/protozero where you can find more +documentation. + +*****************************************************************************/ + +/** + * @file pbf_writer.hpp + * + * @brief Contains the pbf_writer class. + */ + +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#if __BYTE_ORDER != __LITTLE_ENDIAN +# include +#endif + +/// Wrapper for assert() used for testing +#ifndef protozero_assert +# define protozero_assert(x) assert(x) +#endif + +namespace protozero { + +/** + * The pbf_writer is used to write PBF formatted messages into a buffer. + * + * Almost all methods in this class can throw an std::bad_alloc exception if + * the std::string used as a buffer wants to resize. + */ +class pbf_writer { + + std::string* m_data; + pbf_writer* m_parent_writer; + size_t m_pos = 0; + + inline void add_varint(uint64_t value) { + protozero_assert(m_pos == 0 && "you can't add fields to a parent pbf_writer if there is an existing pbf_writer for a submessage"); + protozero_assert(m_data); + write_varint(std::back_inserter(*m_data), value); + } + + inline void add_field(pbf_tag_type tag, pbf_wire_type type) { + protozero_assert(((tag > 0 && tag < 19000) || (tag > 19999 && tag <= ((1 << 29) - 1))) && "tag out of range"); + uint32_t b = (tag << 3) | uint32_t(type); + add_varint(b); + } + + inline void add_tagged_varint(pbf_tag_type tag, uint64_t value) { + add_field(tag, pbf_wire_type::varint); + add_varint(value); + } + + template + inline void add_fixed(T value) { + protozero_assert(m_pos == 0 && "you can't add fields to a parent pbf_writer if there is an existing pbf_writer for a submessage"); + protozero_assert(m_data); +#if __BYTE_ORDER == __LITTLE_ENDIAN + m_data->append(reinterpret_cast(&value), sizeof(T)); +#else + auto size = m_data->size(); + m_data->resize(size + sizeof(T)); + byteswap(reinterpret_cast(&value), const_cast(m_data->data() + size)); +#endif + } + + template + inline void add_packed_fixed(pbf_tag_type tag, It first, It last, std::input_iterator_tag) { + if (first == last) { + return; + } + + pbf_writer sw(*this, tag); + + while (first != last) { + sw.add_fixed(*first++); + } + } + + template + inline void add_packed_fixed(pbf_tag_type tag, It first, It last, std::forward_iterator_tag) { + if (first == last) { + return; + } + + add_length_varint(tag, sizeof(T) * pbf_length_type(std::distance(first, last))); + + while (first != last) { + add_fixed(*first++); + } + } + + template + inline void add_packed_varint(pbf_tag_type tag, It first, It last) { + if (first == last) { + return; + } + + pbf_writer sw(*this, tag); + + while (first != last) { + sw.add_varint(uint64_t(*first++)); + } + } + + template + inline void add_packed_svarint(pbf_tag_type tag, It first, It last) { + if (first == last) { + return; + } + + pbf_writer sw(*this, tag); + + while (first != last) { + sw.add_varint(encode_zigzag64(*first++)); + } + } + + // The number of bytes to reserve for the varint holding the length of + // a length-delimited field. The length has to fit into pbf_length_type, + // and a varint needs 8 bit for every 7 bit. + static const int reserve_bytes = sizeof(pbf_length_type) * 8 / 7 + 1; + + inline void open_submessage(pbf_tag_type tag) { + protozero_assert(m_pos == 0); + protozero_assert(m_data); + add_field(tag, pbf_wire_type::length_delimited); + m_data->append(size_t(reserve_bytes), '\0'); + m_pos = m_data->size(); + } + + inline void close_submessage() { + protozero_assert(m_pos != 0); + protozero_assert(m_data); + auto length = pbf_length_type(m_data->size() - m_pos); + + protozero_assert(m_data->size() >= m_pos - reserve_bytes); + auto n = write_varint(m_data->begin() + long(m_pos) - reserve_bytes, length); + + m_data->erase(m_data->begin() + long(m_pos) - reserve_bytes + n, m_data->begin() + long(m_pos)); + m_pos = 0; + } + + inline void add_length_varint(pbf_tag_type tag, pbf_length_type length) { + add_field(tag, pbf_wire_type::length_delimited); + add_varint(length); + } + +public: + + /** + * Create a writer using the given string as a data store. The pbf_writer + * stores a reference to that string and adds all data to it. + */ + inline explicit pbf_writer(std::string& data) noexcept : + m_data(&data), + m_parent_writer(nullptr), + m_pos(0) { + } + + /** + * Create a writer without a data store. In this form the writer can not + * be used! + */ + inline pbf_writer() noexcept : + m_data(nullptr), + m_parent_writer(nullptr), + m_pos(0) { + } + + /** + * Construct a pbf_writer for a submessage from the pbf_writer of the + * parent message. + * + * @param parent_writer The pbf_writer + * @param tag Tag (field number) of the field that will be written + */ + inline pbf_writer(pbf_writer& parent_writer, pbf_tag_type tag) : + m_data(parent_writer.m_data), + m_parent_writer(&parent_writer), + m_pos(0) { + m_parent_writer->open_submessage(tag); + } + + /// A pbf_writer object can be copied + pbf_writer(const pbf_writer&) noexcept = default; + + /// A pbf_writer object can be copied + pbf_writer& operator=(const pbf_writer&) noexcept = default; + + /// A pbf_writer object can be moved + inline pbf_writer(pbf_writer&&) noexcept = default; + + /// A pbf_writer object can be moved + inline pbf_writer& operator=(pbf_writer&&) noexcept = default; + + inline ~pbf_writer() { + if (m_parent_writer) { + m_parent_writer->close_submessage(); + } + } + + ///@{ + /** + * @name Scalar field writer functions + */ + + /** + * Add "bool" field to data. + * + * @param tag Tag (field number) of the field + * @param value Value to be written + */ + inline void add_bool(pbf_tag_type tag, bool value) { + add_field(tag, pbf_wire_type::varint); + add_fixed(value); + } + + /** + * Add "enum" field to data. + * + * @param tag Tag (field number) of the field + * @param value Value to be written + */ + inline void add_enum(pbf_tag_type tag, int32_t value) { + add_tagged_varint(tag, uint64_t(value)); + } + + /** + * Add "int32" field to data. + * + * @param tag Tag (field number) of the field + * @param value Value to be written + */ + inline void add_int32(pbf_tag_type tag, int32_t value) { + add_tagged_varint(tag, uint64_t(value)); + } + + /** + * Add "sint32" field to data. + * + * @param tag Tag (field number) of the field + * @param value Value to be written + */ + inline void add_sint32(pbf_tag_type tag, int32_t value) { + add_tagged_varint(tag, encode_zigzag32(value)); + } + + /** + * Add "uint32" field to data. + * + * @param tag Tag (field number) of the field + * @param value Value to be written + */ + inline void add_uint32(pbf_tag_type tag, uint32_t value) { + add_tagged_varint(tag, value); + } + + /** + * Add "int64" field to data. + * + * @param tag Tag (field number) of the field + * @param value Value to be written + */ + inline void add_int64(pbf_tag_type tag, int64_t value) { + add_tagged_varint(tag, uint64_t(value)); + } + + /** + * Add "sint64" field to data. + * + * @param tag Tag (field number) of the field + * @param value Value to be written + */ + inline void add_sint64(pbf_tag_type tag, int64_t value) { + add_tagged_varint(tag, encode_zigzag64(value)); + } + + /** + * Add "uint64" field to data. + * + * @param tag Tag (field number) of the field + * @param value Value to be written + */ + inline void add_uint64(pbf_tag_type tag, uint64_t value) { + add_tagged_varint(tag, value); + } + + /** + * Add "fixed32" field to data. + * + * @param tag Tag (field number) of the field + * @param value Value to be written + */ + inline void add_fixed32(pbf_tag_type tag, uint32_t value) { + add_field(tag, pbf_wire_type::fixed32); + add_fixed(value); + } + + /** + * Add "sfixed32" field to data. + * + * @param tag Tag (field number) of the field + * @param value Value to be written + */ + inline void add_sfixed32(pbf_tag_type tag, int32_t value) { + add_field(tag, pbf_wire_type::fixed32); + add_fixed(value); + } + + /** + * Add "fixed64" field to data. + * + * @param tag Tag (field number) of the field + * @param value Value to be written + */ + inline void add_fixed64(pbf_tag_type tag, uint64_t value) { + add_field(tag, pbf_wire_type::fixed64); + add_fixed(value); + } + + /** + * Add "sfixed64" field to data. + * + * @param tag Tag (field number) of the field + * @param value Value to be written + */ + inline void add_sfixed64(pbf_tag_type tag, int64_t value) { + add_field(tag, pbf_wire_type::fixed64); + add_fixed(value); + } + + /** + * Add "float" field to data. + * + * @param tag Tag (field number) of the field + * @param value Value to be written + */ + inline void add_float(pbf_tag_type tag, float value) { + add_field(tag, pbf_wire_type::fixed32); + add_fixed(value); + } + + /** + * Add "double" field to data. + * + * @param tag Tag (field number) of the field + * @param value Value to be written + */ + inline void add_double(pbf_tag_type tag, double value) { + add_field(tag, pbf_wire_type::fixed64); + add_fixed(value); + } + + /** + * Add "bytes" field to data. + * + * @param tag Tag (field number) of the field + * @param value Pointer to value to be written + * @param size Number of bytes to be written + */ + inline void add_bytes(pbf_tag_type tag, const char* value, size_t size) { + protozero_assert(m_pos == 0 && "you can't add fields to a parent pbf_writer if there is an existing pbf_writer for a submessage"); + protozero_assert(m_data); + assert(size <= std::numeric_limits::max()); + add_length_varint(tag, pbf_length_type(size)); + m_data->append(value, size); + } + + /** + * Add "bytes" field to data. + * + * @param tag Tag (field number) of the field + * @param value Value to be written + */ + inline void add_bytes(pbf_tag_type tag, const std::string& value) { + add_bytes(tag, value.data(), value.size()); + } + + /** + * Add "string" field to data. + * + * @param tag Tag (field number) of the field + * @param value Pointer to value to be written + * @param size Number of bytes to be written + */ + inline void add_string(pbf_tag_type tag, const char* value, size_t size) { + add_bytes(tag, value, size); + } + + /** + * Add "string" field to data. + * + * @param tag Tag (field number) of the field + * @param value Value to be written + */ + inline void add_string(pbf_tag_type tag, const std::string& value) { + add_bytes(tag, value.data(), value.size()); + } + + /** + * Add "string" field to data. Bytes from the value are written until + * a null byte is encountered. The null byte is not added. + * + * @param tag Tag (field number) of the field + * @param value Pointer to value to be written + */ + inline void add_string(pbf_tag_type tag, const char* value) { + add_bytes(tag, value, std::strlen(value)); + } + + /** + * Add "message" field to data. + * + * @param tag Tag (field number) of the field + * @param value Pointer to message to be written + * @param size Length of the message + */ + inline void add_message(pbf_tag_type tag, const char* value, size_t size) { + add_bytes(tag, value, size); + } + + /** + * Add "message" field to data. + * + * @param tag Tag (field number) of the field + * @param value Value to be written. The value must be a complete message. + */ + inline void add_message(pbf_tag_type tag, const std::string& value) { + add_bytes(tag, value.data(), value.size()); + } + + ///@} + + ///@{ + /** + * @name Repeated packed field writer functions + */ + + /** + * Add "repeated packed bool" field to data. + * + * @tparam InputIterator An type satisfying the InputIterator concept. + * Dereferencing the iterator must yield a type assignable to bool. + * @param tag Tag (field number) of the field + * @param first Iterator pointing to the beginning of the data + * @param last Iterator pointing one past the end of data + */ + template + inline void add_packed_bool(pbf_tag_type tag, InputIterator first, InputIterator last) { + add_packed_varint(tag, first, last); + } + + /** + * Add "repeated packed enum" field to data. + * + * @tparam InputIterator An type satisfying the InputIterator concept. + * Dereferencing the iterator must yield a type assignable to int32_t. + * @param tag Tag (field number) of the field + * @param first Iterator pointing to the beginning of the data + * @param last Iterator pointing one past the end of data + */ + template + inline void add_packed_enum(pbf_tag_type tag, InputIterator first, InputIterator last) { + add_packed_varint(tag, first, last); + } + + /** + * Add "repeated packed int32" field to data. + * + * @tparam InputIterator An type satisfying the InputIterator concept. + * Dereferencing the iterator must yield a type assignable to int32_t. + * @param tag Tag (field number) of the field + * @param first Iterator pointing to the beginning of the data + * @param last Iterator pointing one past the end of data + */ + template + inline void add_packed_int32(pbf_tag_type tag, InputIterator first, InputIterator last) { + add_packed_varint(tag, first, last); + } + + /** + * Add "repeated packed sint32" field to data. + * + * @tparam InputIterator An type satisfying the InputIterator concept. + * Dereferencing the iterator must yield a type assignable to int32_t. + * @param tag Tag (field number) of the field + * @param first Iterator pointing to the beginning of the data + * @param last Iterator pointing one past the end of data + */ + template + inline void add_packed_sint32(pbf_tag_type tag, InputIterator first, InputIterator last) { + add_packed_svarint(tag, first, last); + } + + /** + * Add "repeated packed uint32" field to data. + * + * @tparam InputIterator An type satisfying the InputIterator concept. + * Dereferencing the iterator must yield a type assignable to uint32_t. + * @param tag Tag (field number) of the field + * @param first Iterator pointing to the beginning of the data + * @param last Iterator pointing one past the end of data + */ + template + inline void add_packed_uint32(pbf_tag_type tag, InputIterator first, InputIterator last) { + add_packed_varint(tag, first, last); + } + + /** + * Add "repeated packed int64" field to data. + * + * @tparam InputIterator An type satisfying the InputIterator concept. + * Dereferencing the iterator must yield a type assignable to int64_t. + * @param tag Tag (field number) of the field + * @param first Iterator pointing to the beginning of the data + * @param last Iterator pointing one past the end of data + */ + template + inline void add_packed_int64(pbf_tag_type tag, InputIterator first, InputIterator last) { + add_packed_varint(tag, first, last); + } + + /** + * Add "repeated packed sint64" field to data. + * + * @tparam InputIterator An type satisfying the InputIterator concept. + * Dereferencing the iterator must yield a type assignable to int64_t. + * @param tag Tag (field number) of the field + * @param first Iterator pointing to the beginning of the data + * @param last Iterator pointing one past the end of data + */ + template + inline void add_packed_sint64(pbf_tag_type tag, InputIterator first, InputIterator last) { + add_packed_svarint(tag, first, last); + } + + /** + * Add "repeated packed uint64" field to data. + * + * @tparam InputIterator An type satisfying the InputIterator concept. + * Dereferencing the iterator must yield a type assignable to uint64_t. + * @param tag Tag (field number) of the field + * @param first Iterator pointing to the beginning of the data + * @param last Iterator pointing one past the end of data + */ + template + inline void add_packed_uint64(pbf_tag_type tag, InputIterator first, InputIterator last) { + add_packed_varint(tag, first, last); + } + + /** + * Add "repeated packed fixed32" field to data. + * + * @tparam InputIterator An type satisfying the InputIterator concept. + * Dereferencing the iterator must yield a type assignable to uint32_t. + * @param tag Tag (field number) of the field + * @param first Iterator pointing to the beginning of the data + * @param last Iterator pointing one past the end of data + */ + template + inline void add_packed_fixed32(pbf_tag_type tag, InputIterator first, InputIterator last) { + add_packed_fixed(tag, first, last, + typename std::iterator_traits::iterator_category()); + } + + /** + * Add "repeated packed sfixed32" field to data. + * + * @tparam InputIterator An type satisfying the InputIterator concept. + * Dereferencing the iterator must yield a type assignable to int32_t. + * @param tag Tag (field number) of the field + * @param first Iterator pointing to the beginning of the data + * @param last Iterator pointing one past the end of data + */ + template + inline void add_packed_sfixed32(pbf_tag_type tag, InputIterator first, InputIterator last) { + add_packed_fixed(tag, first, last, + typename std::iterator_traits::iterator_category()); + } + + /** + * Add "repeated packed fixed64" field to data. + * + * @tparam InputIterator An type satisfying the InputIterator concept. + * Dereferencing the iterator must yield a type assignable to uint64_t. + * @param tag Tag (field number) of the field + * @param first Iterator pointing to the beginning of the data + * @param last Iterator pointing one past the end of data + */ + template + inline void add_packed_fixed64(pbf_tag_type tag, InputIterator first, InputIterator last) { + add_packed_fixed(tag, first, last, + typename std::iterator_traits::iterator_category()); + } + + /** + * Add "repeated packed sfixed64" field to data. + * + * @tparam InputIterator An type satisfying the InputIterator concept. + * Dereferencing the iterator must yield a type assignable to int64_t. + * @param tag Tag (field number) of the field + * @param first Iterator pointing to the beginning of the data + * @param last Iterator pointing one past the end of data + */ + template + inline void add_packed_sfixed64(pbf_tag_type tag, InputIterator first, InputIterator last) { + add_packed_fixed(tag, first, last, + typename std::iterator_traits::iterator_category()); + } + + /** + * Add "repeated packed float" field to data. + * + * @tparam InputIterator An type satisfying the InputIterator concept. + * Dereferencing the iterator must yield a type assignable to float. + * @param tag Tag (field number) of the field + * @param first Iterator pointing to the beginning of the data + * @param last Iterator pointing one past the end of data + */ + template + inline void add_packed_float(pbf_tag_type tag, InputIterator first, InputIterator last) { + add_packed_fixed(tag, first, last, + typename std::iterator_traits::iterator_category()); + } + + /** + * Add "repeated packed double" field to data. + * + * @tparam InputIterator An type satisfying the InputIterator concept. + * Dereferencing the iterator must yield a type assignable to double. + * @param tag Tag (field number) of the field + * @param first Iterator pointing to the beginning of the data + * @param last Iterator pointing one past the end of data + */ + template + inline void add_packed_double(pbf_tag_type tag, InputIterator first, InputIterator last) { + add_packed_fixed(tag, first, last, + typename std::iterator_traits::iterator_category()); + } + + ///@} + +}; // class pbf_writer + +} // end namespace protozero + +#endif // PROTOZERO_PBF_WRITER_HPP diff --git a/include/protozero/varint.hpp b/include/protozero/varint.hpp new file mode 100644 index 00000000000..bc9c3296d62 --- /dev/null +++ b/include/protozero/varint.hpp @@ -0,0 +1,136 @@ +#ifndef PROTOZERO_VARINT_HPP +#define PROTOZERO_VARINT_HPP + +/***************************************************************************** + +protozero - Minimalistic protocol buffer decoder and encoder in C++. + +This file is from https://github.com/mapbox/protozero where you can find more +documentation. + +*****************************************************************************/ + +/** + * @file varint.hpp + * + * @brief Contains low-level varint and zigzag encoding and decoding functions. + */ + +#if __BYTE_ORDER != __LITTLE_ENDIAN +# error "This code only works on little endian machines." +#endif + +#include + +#include + +namespace protozero { + +/** + * The maximum length of a 64bit varint. + */ +const int8_t max_varint_length = sizeof(uint64_t) * 8 / 7 + 1; + +// from https://github.com/facebook/folly/blob/master/folly/Varint.h +/** + * Decode a 64bit varint. + * + * String exception guarantee: if there is an exception the data pointer will + * not be changed. + * + * @param[in,out] data Pointer to pointer to the input data. After the function + * returns this will point to the next data to be read. + * @param[in] end Pointer one past the end of the input data. + * @returns The decoded integer + * @throws varint_too_long_exception if the varint is longer then the maximum + * length that would fit in a 64bit int. Usually this means your data + * is corrupted or you are trying to read something as a varint that + * isn't. + * @throws end_of_buffer_exception if the *end* of the buffer was reached + * before the end of the varint. + */ +inline uint64_t decode_varint(const char** data, const char* end) { + const int8_t* begin = reinterpret_cast(*data); + const int8_t* iend = reinterpret_cast(end); + const int8_t* p = begin; + uint64_t val = 0; + + if (iend - begin >= max_varint_length) { // fast path + do { + int64_t b; + b = *p++; val = uint64_t((b & 0x7f) ); if (b >= 0) break; + b = *p++; val |= uint64_t((b & 0x7f) << 7); if (b >= 0) break; + b = *p++; val |= uint64_t((b & 0x7f) << 14); if (b >= 0) break; + b = *p++; val |= uint64_t((b & 0x7f) << 21); if (b >= 0) break; + b = *p++; val |= uint64_t((b & 0x7f) << 28); if (b >= 0) break; + b = *p++; val |= uint64_t((b & 0x7f) << 35); if (b >= 0) break; + b = *p++; val |= uint64_t((b & 0x7f) << 42); if (b >= 0) break; + b = *p++; val |= uint64_t((b & 0x7f) << 49); if (b >= 0) break; + b = *p++; val |= uint64_t((b & 0x7f) << 56); if (b >= 0) break; + b = *p++; val |= uint64_t((b & 0x7f) << 63); if (b >= 0) break; + throw varint_too_long_exception(); + } while (false); + } else { + int shift = 0; + while (p != iend && *p < 0) { + val |= uint64_t(*p++ & 0x7f) << shift; + shift += 7; + } + if (p == iend) { + throw end_of_buffer_exception(); + } + val |= uint64_t(*p++) << shift; + } + + *data = reinterpret_cast(p); + return val; +} + +/** + * Varint-encode a 64bit integer. + */ +template +inline int write_varint(OutputIterator data, uint64_t value) { + int n=1; + + while (value >= 0x80) { + *data++ = char((value & 0x7f) | 0x80); + value >>= 7; + ++n; + } + *data++ = char(value); + + return n; +} + +/** + * ZigZag encodes a 32 bit integer. + */ +inline uint32_t encode_zigzag32(int32_t value) noexcept { + return (static_cast(value) << 1) ^ (static_cast(value >> 31)); +} + +/** + * ZigZag encodes a 64 bit integer. + */ +inline uint64_t encode_zigzag64(int64_t value) noexcept { + return (static_cast(value) << 1) ^ (static_cast(value >> 63)); +} + +/** + * Decodes a 32 bit ZigZag-encoded integer. + */ +inline int32_t decode_zigzag32(uint32_t value) noexcept { + return int32_t(value >> 1) ^ -int32_t(value & 1); +} + +/** + * Decodes a 64 bit ZigZag-encoded integer. + */ +inline int64_t decode_zigzag64(uint64_t value) noexcept { + return int64_t(value >> 1) ^ -int64_t(value & 1); +} + +} // end namespace protozero + +#endif // PROTOZERO_VARINT_HPP diff --git a/include/utf8.h b/include/utf8.h new file mode 100644 index 00000000000..82b13f59f98 --- /dev/null +++ b/include/utf8.h @@ -0,0 +1,34 @@ +// Copyright 2006 Nemanja Trifunovic + +/* +Permission is hereby granted, free of charge, to any person or organization +obtaining a copy of the software and accompanying documentation covered by +this license (the "Software") to use, reproduce, display, distribute, +execute, and transmit the Software, and to prepare derivative works of the +Software, and to permit third-parties to whom the Software is furnished to +do so, all subject to the following: + +The copyright notices in the Software and this entire statement, including +the above license grant, this restriction and the following disclaimer, +must be included in all copies of the Software, in whole or in part, and +all derivative works of the Software, unless such copies or derivative +works are solely in the form of machine-executable object code generated by +a source language processor. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT +SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE +FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, +ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. +*/ + + +#ifndef UTF8_FOR_CPP_2675DCD0_9480_4c0c_B92A_CC14C027B731 +#define UTF8_FOR_CPP_2675DCD0_9480_4c0c_B92A_CC14C027B731 + +#include "utf8/checked.h" +#include "utf8/unchecked.h" + +#endif // header guard diff --git a/include/utf8/checked.h b/include/utf8/checked.h new file mode 100644 index 00000000000..13311551381 --- /dev/null +++ b/include/utf8/checked.h @@ -0,0 +1,327 @@ +// Copyright 2006 Nemanja Trifunovic + +/* +Permission is hereby granted, free of charge, to any person or organization +obtaining a copy of the software and accompanying documentation covered by +this license (the "Software") to use, reproduce, display, distribute, +execute, and transmit the Software, and to prepare derivative works of the +Software, and to permit third-parties to whom the Software is furnished to +do so, all subject to the following: + +The copyright notices in the Software and this entire statement, including +the above license grant, this restriction and the following disclaimer, +must be included in all copies of the Software, in whole or in part, and +all derivative works of the Software, unless such copies or derivative +works are solely in the form of machine-executable object code generated by +a source language processor. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT +SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE +FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, +ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. +*/ + + +#ifndef UTF8_FOR_CPP_CHECKED_H_2675DCD0_9480_4c0c_B92A_CC14C027B731 +#define UTF8_FOR_CPP_CHECKED_H_2675DCD0_9480_4c0c_B92A_CC14C027B731 + +#include "core.h" +#include + +namespace utf8 +{ + // Base for the exceptions that may be thrown from the library + class exception : public ::std::exception { + }; + + // Exceptions that may be thrown from the library functions. + class invalid_code_point : public exception { + uint32_t cp; + public: + invalid_code_point(uint32_t cp) : cp(cp) {} + virtual const char* what() const throw() { return "Invalid code point"; } + uint32_t code_point() const {return cp;} + }; + + class invalid_utf8 : public exception { + uint8_t u8; + public: + invalid_utf8 (uint8_t u) : u8(u) {} + virtual const char* what() const throw() { return "Invalid UTF-8"; } + uint8_t utf8_octet() const {return u8;} + }; + + class invalid_utf16 : public exception { + uint16_t u16; + public: + invalid_utf16 (uint16_t u) : u16(u) {} + virtual const char* what() const throw() { return "Invalid UTF-16"; } + uint16_t utf16_word() const {return u16;} + }; + + class not_enough_room : public exception { + public: + virtual const char* what() const throw() { return "Not enough space"; } + }; + + /// The library API - functions intended to be called by the users + + template + octet_iterator append(uint32_t cp, octet_iterator result) + { + if (!utf8::internal::is_code_point_valid(cp)) + throw invalid_code_point(cp); + + if (cp < 0x80) // one octet + *(result++) = static_cast(cp); + else if (cp < 0x800) { // two octets + *(result++) = static_cast((cp >> 6) | 0xc0); + *(result++) = static_cast((cp & 0x3f) | 0x80); + } + else if (cp < 0x10000) { // three octets + *(result++) = static_cast((cp >> 12) | 0xe0); + *(result++) = static_cast(((cp >> 6) & 0x3f) | 0x80); + *(result++) = static_cast((cp & 0x3f) | 0x80); + } + else { // four octets + *(result++) = static_cast((cp >> 18) | 0xf0); + *(result++) = static_cast(((cp >> 12) & 0x3f) | 0x80); + *(result++) = static_cast(((cp >> 6) & 0x3f) | 0x80); + *(result++) = static_cast((cp & 0x3f) | 0x80); + } + return result; + } + + template + output_iterator replace_invalid(octet_iterator start, octet_iterator end, output_iterator out, uint32_t replacement) + { + while (start != end) { + octet_iterator sequence_start = start; + internal::utf_error err_code = utf8::internal::validate_next(start, end); + switch (err_code) { + case internal::UTF8_OK : + for (octet_iterator it = sequence_start; it != start; ++it) + *out++ = *it; + break; + case internal::NOT_ENOUGH_ROOM: + throw not_enough_room(); + case internal::INVALID_LEAD: + out = utf8::append (replacement, out); + ++start; + break; + case internal::INCOMPLETE_SEQUENCE: + case internal::OVERLONG_SEQUENCE: + case internal::INVALID_CODE_POINT: + out = utf8::append (replacement, out); + ++start; + // just one replacement mark for the sequence + while (start != end && utf8::internal::is_trail(*start)) + ++start; + break; + } + } + return out; + } + + template + inline output_iterator replace_invalid(octet_iterator start, octet_iterator end, output_iterator out) + { + static const uint32_t replacement_marker = utf8::internal::mask16(0xfffd); + return utf8::replace_invalid(start, end, out, replacement_marker); + } + + template + uint32_t next(octet_iterator& it, octet_iterator end) + { + uint32_t cp = 0; + internal::utf_error err_code = utf8::internal::validate_next(it, end, cp); + switch (err_code) { + case internal::UTF8_OK : + break; + case internal::NOT_ENOUGH_ROOM : + throw not_enough_room(); + case internal::INVALID_LEAD : + case internal::INCOMPLETE_SEQUENCE : + case internal::OVERLONG_SEQUENCE : + throw invalid_utf8(*it); + case internal::INVALID_CODE_POINT : + throw invalid_code_point(cp); + } + return cp; + } + + template + uint32_t peek_next(octet_iterator it, octet_iterator end) + { + return utf8::next(it, end); + } + + template + uint32_t prior(octet_iterator& it, octet_iterator start) + { + // can't do much if it == start + if (it == start) + throw not_enough_room(); + + octet_iterator end = it; + // Go back until we hit either a lead octet or start + while (utf8::internal::is_trail(*(--it))) + if (it == start) + throw invalid_utf8(*it); // error - no lead byte in the sequence + return utf8::peek_next(it, end); + } + + /// Deprecated in versions that include "prior" + template + uint32_t previous(octet_iterator& it, octet_iterator pass_start) + { + octet_iterator end = it; + while (utf8::internal::is_trail(*(--it))) + if (it == pass_start) + throw invalid_utf8(*it); // error - no lead byte in the sequence + octet_iterator temp = it; + return utf8::next(temp, end); + } + + template + void advance (octet_iterator& it, distance_type n, octet_iterator end) + { + for (distance_type i = 0; i < n; ++i) + utf8::next(it, end); + } + + template + typename std::iterator_traits::difference_type + distance (octet_iterator first, octet_iterator last) + { + typename std::iterator_traits::difference_type dist; + for (dist = 0; first < last; ++dist) + utf8::next(first, last); + return dist; + } + + template + octet_iterator utf16to8 (u16bit_iterator start, u16bit_iterator end, octet_iterator result) + { + while (start != end) { + uint32_t cp = utf8::internal::mask16(*start++); + // Take care of surrogate pairs first + if (utf8::internal::is_lead_surrogate(cp)) { + if (start != end) { + uint32_t trail_surrogate = utf8::internal::mask16(*start++); + if (utf8::internal::is_trail_surrogate(trail_surrogate)) + cp = (cp << 10) + trail_surrogate + internal::SURROGATE_OFFSET; + else + throw invalid_utf16(static_cast(trail_surrogate)); + } + else + throw invalid_utf16(static_cast(cp)); + + } + // Lone trail surrogate + else if (utf8::internal::is_trail_surrogate(cp)) + throw invalid_utf16(static_cast(cp)); + + result = utf8::append(cp, result); + } + return result; + } + + template + u16bit_iterator utf8to16 (octet_iterator start, octet_iterator end, u16bit_iterator result) + { + while (start != end) { + uint32_t cp = utf8::next(start, end); + if (cp > 0xffff) { //make a surrogate pair + *result++ = static_cast((cp >> 10) + internal::LEAD_OFFSET); + *result++ = static_cast((cp & 0x3ff) + internal::TRAIL_SURROGATE_MIN); + } + else + *result++ = static_cast(cp); + } + return result; + } + + template + octet_iterator utf32to8 (u32bit_iterator start, u32bit_iterator end, octet_iterator result) + { + while (start != end) + result = utf8::append(*(start++), result); + + return result; + } + + template + u32bit_iterator utf8to32 (octet_iterator start, octet_iterator end, u32bit_iterator result) + { + while (start != end) + (*result++) = utf8::next(start, end); + + return result; + } + + // The iterator class + template + class iterator : public std::iterator { + octet_iterator it; + octet_iterator range_start; + octet_iterator range_end; + public: + iterator () {} + explicit iterator (const octet_iterator& octet_it, + const octet_iterator& range_start, + const octet_iterator& range_end) : + it(octet_it), range_start(range_start), range_end(range_end) + { + if (it < range_start || it > range_end) + throw std::out_of_range("Invalid utf-8 iterator position"); + } + // the default "big three" are OK + octet_iterator base () const { return it; } + uint32_t operator * () const + { + octet_iterator temp = it; + return utf8::next(temp, range_end); + } + bool operator == (const iterator& rhs) const + { + if (range_start != rhs.range_start || range_end != rhs.range_end) + throw std::logic_error("Comparing utf-8 iterators defined with different ranges"); + return (it == rhs.it); + } + bool operator != (const iterator& rhs) const + { + return !(operator == (rhs)); + } + iterator& operator ++ () + { + utf8::next(it, range_end); + return *this; + } + iterator operator ++ (int) + { + iterator temp = *this; + utf8::next(it, range_end); + return temp; + } + iterator& operator -- () + { + utf8::prior(it, range_start); + return *this; + } + iterator operator -- (int) + { + iterator temp = *this; + utf8::prior(it, range_start); + return temp; + } + }; // class iterator + +} // namespace utf8 + +#endif //header guard + + diff --git a/include/utf8/core.h b/include/utf8/core.h new file mode 100644 index 00000000000..693d388c078 --- /dev/null +++ b/include/utf8/core.h @@ -0,0 +1,329 @@ +// Copyright 2006 Nemanja Trifunovic + +/* +Permission is hereby granted, free of charge, to any person or organization +obtaining a copy of the software and accompanying documentation covered by +this license (the "Software") to use, reproduce, display, distribute, +execute, and transmit the Software, and to prepare derivative works of the +Software, and to permit third-parties to whom the Software is furnished to +do so, all subject to the following: + +The copyright notices in the Software and this entire statement, including +the above license grant, this restriction and the following disclaimer, +must be included in all copies of the Software, in whole or in part, and +all derivative works of the Software, unless such copies or derivative +works are solely in the form of machine-executable object code generated by +a source language processor. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT +SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE +FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, +ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. +*/ + + +#ifndef UTF8_FOR_CPP_CORE_H_2675DCD0_9480_4c0c_B92A_CC14C027B731 +#define UTF8_FOR_CPP_CORE_H_2675DCD0_9480_4c0c_B92A_CC14C027B731 + +#include + +namespace utf8 +{ + // The typedefs for 8-bit, 16-bit and 32-bit unsigned integers + // You may need to change them to match your system. + // These typedefs have the same names as ones from cstdint, or boost/cstdint + typedef unsigned char uint8_t; + typedef unsigned short uint16_t; + typedef unsigned int uint32_t; + +// Helper code - not intended to be directly called by the library users. May be changed at any time +namespace internal +{ + // Unicode constants + // Leading (high) surrogates: 0xd800 - 0xdbff + // Trailing (low) surrogates: 0xdc00 - 0xdfff + const uint16_t LEAD_SURROGATE_MIN = 0xd800u; + const uint16_t LEAD_SURROGATE_MAX = 0xdbffu; + const uint16_t TRAIL_SURROGATE_MIN = 0xdc00u; + const uint16_t TRAIL_SURROGATE_MAX = 0xdfffu; + const uint16_t LEAD_OFFSET = LEAD_SURROGATE_MIN - (0x10000 >> 10); + const uint32_t SURROGATE_OFFSET = 0x10000u - (LEAD_SURROGATE_MIN << 10) - TRAIL_SURROGATE_MIN; + + // Maximum valid value for a Unicode code point + const uint32_t CODE_POINT_MAX = 0x0010ffffu; + + template + inline uint8_t mask8(octet_type oc) + { + return static_cast(0xff & oc); + } + template + inline uint16_t mask16(u16_type oc) + { + return static_cast(0xffff & oc); + } + template + inline bool is_trail(octet_type oc) + { + return ((utf8::internal::mask8(oc) >> 6) == 0x2); + } + + template + inline bool is_lead_surrogate(u16 cp) + { + return (cp >= LEAD_SURROGATE_MIN && cp <= LEAD_SURROGATE_MAX); + } + + template + inline bool is_trail_surrogate(u16 cp) + { + return (cp >= TRAIL_SURROGATE_MIN && cp <= TRAIL_SURROGATE_MAX); + } + + template + inline bool is_surrogate(u16 cp) + { + return (cp >= LEAD_SURROGATE_MIN && cp <= TRAIL_SURROGATE_MAX); + } + + template + inline bool is_code_point_valid(u32 cp) + { + return (cp <= CODE_POINT_MAX && !utf8::internal::is_surrogate(cp)); + } + + template + inline typename std::iterator_traits::difference_type + sequence_length(octet_iterator lead_it) + { + uint8_t lead = utf8::internal::mask8(*lead_it); + if (lead < 0x80) + return 1; + else if ((lead >> 5) == 0x6) + return 2; + else if ((lead >> 4) == 0xe) + return 3; + else if ((lead >> 3) == 0x1e) + return 4; + else + return 0; + } + + template + inline bool is_overlong_sequence(uint32_t cp, octet_difference_type length) + { + if (cp < 0x80) { + if (length != 1) + return true; + } + else if (cp < 0x800) { + if (length != 2) + return true; + } + else if (cp < 0x10000) { + if (length != 3) + return true; + } + + return false; + } + + enum utf_error {UTF8_OK, NOT_ENOUGH_ROOM, INVALID_LEAD, INCOMPLETE_SEQUENCE, OVERLONG_SEQUENCE, INVALID_CODE_POINT}; + + /// Helper for get_sequence_x + template + utf_error increase_safely(octet_iterator& it, octet_iterator end) + { + if (++it == end) + return NOT_ENOUGH_ROOM; + + if (!utf8::internal::is_trail(*it)) + return INCOMPLETE_SEQUENCE; + + return UTF8_OK; + } + + #define UTF8_CPP_INCREASE_AND_RETURN_ON_ERROR(IT, END) {utf_error ret = increase_safely(IT, END); if (ret != UTF8_OK) return ret;} + + /// get_sequence_x functions decode utf-8 sequences of the length x + template + utf_error get_sequence_1(octet_iterator& it, octet_iterator end, uint32_t& code_point) + { + if (it == end) + return NOT_ENOUGH_ROOM; + + code_point = utf8::internal::mask8(*it); + + return UTF8_OK; + } + + template + utf_error get_sequence_2(octet_iterator& it, octet_iterator end, uint32_t& code_point) + { + if (it == end) + return NOT_ENOUGH_ROOM; + + code_point = utf8::internal::mask8(*it); + + UTF8_CPP_INCREASE_AND_RETURN_ON_ERROR(it, end) + + code_point = ((code_point << 6) & 0x7ff) + ((*it) & 0x3f); + + return UTF8_OK; + } + + template + utf_error get_sequence_3(octet_iterator& it, octet_iterator end, uint32_t& code_point) + { + if (it == end) + return NOT_ENOUGH_ROOM; + + code_point = utf8::internal::mask8(*it); + + UTF8_CPP_INCREASE_AND_RETURN_ON_ERROR(it, end) + + code_point = ((code_point << 12) & 0xffff) + ((utf8::internal::mask8(*it) << 6) & 0xfff); + + UTF8_CPP_INCREASE_AND_RETURN_ON_ERROR(it, end) + + code_point += (*it) & 0x3f; + + return UTF8_OK; + } + + template + utf_error get_sequence_4(octet_iterator& it, octet_iterator end, uint32_t& code_point) + { + if (it == end) + return NOT_ENOUGH_ROOM; + + code_point = utf8::internal::mask8(*it); + + UTF8_CPP_INCREASE_AND_RETURN_ON_ERROR(it, end) + + code_point = ((code_point << 18) & 0x1fffff) + ((utf8::internal::mask8(*it) << 12) & 0x3ffff); + + UTF8_CPP_INCREASE_AND_RETURN_ON_ERROR(it, end) + + code_point += (utf8::internal::mask8(*it) << 6) & 0xfff; + + UTF8_CPP_INCREASE_AND_RETURN_ON_ERROR(it, end) + + code_point += (*it) & 0x3f; + + return UTF8_OK; + } + + #undef UTF8_CPP_INCREASE_AND_RETURN_ON_ERROR + + template + utf_error validate_next(octet_iterator& it, octet_iterator end, uint32_t& code_point) + { + // Save the original value of it so we can go back in case of failure + // Of course, it does not make much sense with i.e. stream iterators + octet_iterator original_it = it; + + uint32_t cp = 0; + // Determine the sequence length based on the lead octet + typedef typename std::iterator_traits::difference_type octet_difference_type; + const octet_difference_type length = utf8::internal::sequence_length(it); + + // Get trail octets and calculate the code point + utf_error err = UTF8_OK; + switch (length) { + case 0: + return INVALID_LEAD; + case 1: + err = utf8::internal::get_sequence_1(it, end, cp); + break; + case 2: + err = utf8::internal::get_sequence_2(it, end, cp); + break; + case 3: + err = utf8::internal::get_sequence_3(it, end, cp); + break; + case 4: + err = utf8::internal::get_sequence_4(it, end, cp); + break; + } + + if (err == UTF8_OK) { + // Decoding succeeded. Now, security checks... + if (utf8::internal::is_code_point_valid(cp)) { + if (!utf8::internal::is_overlong_sequence(cp, length)){ + // Passed! Return here. + code_point = cp; + ++it; + return UTF8_OK; + } + else + err = OVERLONG_SEQUENCE; + } + else + err = INVALID_CODE_POINT; + } + + // Failure branch - restore the original value of the iterator + it = original_it; + return err; + } + + template + inline utf_error validate_next(octet_iterator& it, octet_iterator end) { + uint32_t ignored; + return utf8::internal::validate_next(it, end, ignored); + } + +} // namespace internal + + /// The library API - functions intended to be called by the users + + // Byte order mark + const uint8_t bom[] = {0xef, 0xbb, 0xbf}; + + template + octet_iterator find_invalid(octet_iterator start, octet_iterator end) + { + octet_iterator result = start; + while (result != end) { + utf8::internal::utf_error err_code = utf8::internal::validate_next(result, end); + if (err_code != internal::UTF8_OK) + return result; + } + return result; + } + + template + inline bool is_valid(octet_iterator start, octet_iterator end) + { + return (utf8::find_invalid(start, end) == end); + } + + template + inline bool starts_with_bom (octet_iterator it, octet_iterator end) + { + return ( + ((it != end) && (utf8::internal::mask8(*it++)) == bom[0]) && + ((it != end) && (utf8::internal::mask8(*it++)) == bom[1]) && + ((it != end) && (utf8::internal::mask8(*it)) == bom[2]) + ); + } + + //Deprecated in release 2.3 + template + inline bool is_bom (octet_iterator it) + { + return ( + (utf8::internal::mask8(*it++)) == bom[0] && + (utf8::internal::mask8(*it++)) == bom[1] && + (utf8::internal::mask8(*it)) == bom[2] + ); + } +} // namespace utf8 + +#endif // header guard + + diff --git a/include/utf8/unchecked.h b/include/utf8/unchecked.h new file mode 100644 index 00000000000..cb2427166b1 --- /dev/null +++ b/include/utf8/unchecked.h @@ -0,0 +1,228 @@ +// Copyright 2006 Nemanja Trifunovic + +/* +Permission is hereby granted, free of charge, to any person or organization +obtaining a copy of the software and accompanying documentation covered by +this license (the "Software") to use, reproduce, display, distribute, +execute, and transmit the Software, and to prepare derivative works of the +Software, and to permit third-parties to whom the Software is furnished to +do so, all subject to the following: + +The copyright notices in the Software and this entire statement, including +the above license grant, this restriction and the following disclaimer, +must be included in all copies of the Software, in whole or in part, and +all derivative works of the Software, unless such copies or derivative +works are solely in the form of machine-executable object code generated by +a source language processor. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT +SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE +FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, +ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. +*/ + + +#ifndef UTF8_FOR_CPP_UNCHECKED_H_2675DCD0_9480_4c0c_B92A_CC14C027B731 +#define UTF8_FOR_CPP_UNCHECKED_H_2675DCD0_9480_4c0c_B92A_CC14C027B731 + +#include "core.h" + +namespace utf8 +{ + namespace unchecked + { + template + octet_iterator append(uint32_t cp, octet_iterator result) + { + if (cp < 0x80) // one octet + *(result++) = static_cast(cp); + else if (cp < 0x800) { // two octets + *(result++) = static_cast((cp >> 6) | 0xc0); + *(result++) = static_cast((cp & 0x3f) | 0x80); + } + else if (cp < 0x10000) { // three octets + *(result++) = static_cast((cp >> 12) | 0xe0); + *(result++) = static_cast(((cp >> 6) & 0x3f) | 0x80); + *(result++) = static_cast((cp & 0x3f) | 0x80); + } + else { // four octets + *(result++) = static_cast((cp >> 18) | 0xf0); + *(result++) = static_cast(((cp >> 12) & 0x3f)| 0x80); + *(result++) = static_cast(((cp >> 6) & 0x3f) | 0x80); + *(result++) = static_cast((cp & 0x3f) | 0x80); + } + return result; + } + + template + uint32_t next(octet_iterator& it) + { + uint32_t cp = utf8::internal::mask8(*it); + typename std::iterator_traits::difference_type length = utf8::internal::sequence_length(it); + switch (length) { + case 1: + break; + case 2: + it++; + cp = ((cp << 6) & 0x7ff) + ((*it) & 0x3f); + break; + case 3: + ++it; + cp = ((cp << 12) & 0xffff) + ((utf8::internal::mask8(*it) << 6) & 0xfff); + ++it; + cp += (*it) & 0x3f; + break; + case 4: + ++it; + cp = ((cp << 18) & 0x1fffff) + ((utf8::internal::mask8(*it) << 12) & 0x3ffff); + ++it; + cp += (utf8::internal::mask8(*it) << 6) & 0xfff; + ++it; + cp += (*it) & 0x3f; + break; + } + ++it; + return cp; + } + + template + uint32_t peek_next(octet_iterator it) + { + return utf8::unchecked::next(it); + } + + template + uint32_t prior(octet_iterator& it) + { + while (utf8::internal::is_trail(*(--it))) ; + octet_iterator temp = it; + return utf8::unchecked::next(temp); + } + + // Deprecated in versions that include prior, but only for the sake of consistency (see utf8::previous) + template + inline uint32_t previous(octet_iterator& it) + { + return utf8::unchecked::prior(it); + } + + template + void advance (octet_iterator& it, distance_type n) + { + for (distance_type i = 0; i < n; ++i) + utf8::unchecked::next(it); + } + + template + typename std::iterator_traits::difference_type + distance (octet_iterator first, octet_iterator last) + { + typename std::iterator_traits::difference_type dist; + for (dist = 0; first < last; ++dist) + utf8::unchecked::next(first); + return dist; + } + + template + octet_iterator utf16to8 (u16bit_iterator start, u16bit_iterator end, octet_iterator result) + { + while (start != end) { + uint32_t cp = utf8::internal::mask16(*start++); + // Take care of surrogate pairs first + if (utf8::internal::is_lead_surrogate(cp)) { + uint32_t trail_surrogate = utf8::internal::mask16(*start++); + cp = (cp << 10) + trail_surrogate + internal::SURROGATE_OFFSET; + } + result = utf8::unchecked::append(cp, result); + } + return result; + } + + template + u16bit_iterator utf8to16 (octet_iterator start, octet_iterator end, u16bit_iterator result) + { + while (start < end) { + uint32_t cp = utf8::unchecked::next(start); + if (cp > 0xffff) { //make a surrogate pair + *result++ = static_cast((cp >> 10) + internal::LEAD_OFFSET); + *result++ = static_cast((cp & 0x3ff) + internal::TRAIL_SURROGATE_MIN); + } + else + *result++ = static_cast(cp); + } + return result; + } + + template + octet_iterator utf32to8 (u32bit_iterator start, u32bit_iterator end, octet_iterator result) + { + while (start != end) + result = utf8::unchecked::append(*(start++), result); + + return result; + } + + template + u32bit_iterator utf8to32 (octet_iterator start, octet_iterator end, u32bit_iterator result) + { + while (start < end) + (*result++) = utf8::unchecked::next(start); + + return result; + } + + // The iterator class + template + class iterator : public std::iterator { + octet_iterator it; + public: + iterator () {} + explicit iterator (const octet_iterator& octet_it): it(octet_it) {} + // the default "big three" are OK + octet_iterator base () const { return it; } + uint32_t operator * () const + { + octet_iterator temp = it; + return utf8::unchecked::next(temp); + } + bool operator == (const iterator& rhs) const + { + return (it == rhs.it); + } + bool operator != (const iterator& rhs) const + { + return !(operator == (rhs)); + } + iterator& operator ++ () + { + ::std::advance(it, utf8::internal::sequence_length(it)); + return *this; + } + iterator operator ++ (int) + { + iterator temp = *this; + ::std::advance(it, utf8::internal::sequence_length(it)); + return temp; + } + iterator& operator -- () + { + utf8::unchecked::prior(it); + return *this; + } + iterator operator -- (int) + { + iterator temp = *this; + utf8::unchecked::prior(it); + return temp; + } + }; // class iterator + + } // namespace utf8::unchecked +} // namespace utf8 + + +#endif // header guard + diff --git a/scripts/travis_install.sh b/scripts/travis_install.sh new file mode 100755 index 00000000000..119e9fd15e3 --- /dev/null +++ b/scripts/travis_install.sh @@ -0,0 +1,20 @@ +#!/bin/sh +# +# travis_install.sh +# + +if [ "$TRAVIS_OS_NAME" = "osx" ]; then + + brew install google-sparsehash || true + + brew install --without-python boost || true + + # workaround for gdal homebrew problem + brew remove gdal + brew install gdal + +fi + +cd .. +git clone --quiet --depth 1 https://github.com/osmcode/osm-testdata.git + diff --git a/scripts/travis_script.sh b/scripts/travis_script.sh new file mode 100755 index 00000000000..75b3b365748 --- /dev/null +++ b/scripts/travis_script.sh @@ -0,0 +1,29 @@ +#!/bin/sh +# +# travis_script.sh +# + +mkdir build +cd build + +# GCC ignores the pragmas in the code that disable the "return-type" warning +# selectively, so use this workaround. +if [ "${CXX}" = "g++" ]; then + WORKAROUND="-DCMAKE_CXX_FLAGS=-Wno-return-type" +else + WORKAROUND="" +fi + +if [ "${CXX}" = "g++" ]; then + CXX=g++-4.8 + CC=gcc-4.8 +fi + +cmake -LA \ + -DCMAKE_BUILD_TYPE=${BUILD_TYPE} \ + ${WORKAROUND} \ + .. + +make VERBOSE=1 +ctest --output-on-failure + diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 7ba455b0295..00474577a2e 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -15,6 +15,11 @@ add_library(testlib STATIC test_main.cpp) set(ALL_TESTS "") +# Otherwise GCC throws a lot of warnings for REQUIRE(...) from Catch v.1.2.1 +if(CMAKE_COMPILER_IS_GNUCXX) + add_definitions(-Wno-parentheses) +endif() + #----------------------------------------------------------------------------- # @@ -90,6 +95,7 @@ add_unit_test(area test_node_ref_segment) add_unit_test(basic test_box) add_unit_test(basic test_changeset) +add_unit_test(basic test_crc) add_unit_test(basic test_entity_bits) add_unit_test(basic test_location) add_unit_test(basic test_node) @@ -97,6 +103,7 @@ add_unit_test(basic test_node_ref) add_unit_test(basic test_object_comparisons) add_unit_test(basic test_relation) add_unit_test(basic test_timestamp) +add_unit_test(basic test_types_from_string) add_unit_test(basic test_way) add_unit_test(buffer test_buffer_node) @@ -111,23 +118,24 @@ add_unit_test(geom test_factory_with_projection ENABLE_IF ${GEOS_AND_PROJ_FOUND} LIBS ${GEOS_LIBRARY} ${PROJ_LIBRARY}) +add_unit_test(geom test_exception) add_unit_test(geom test_geojson) add_unit_test(geom test_geos ENABLE_IF ${GEOS_FOUND} LIBS ${GEOS_LIBRARY}) add_unit_test(geom test_geos_wkb ENABLE_IF ${GEOS_FOUND} LIBS ${GEOS_LIBRARY}) add_unit_test(geom test_mercator) add_unit_test(geom test_ogr ENABLE_IF ${GDAL_FOUND} LIBS ${GDAL_LIBRARY}) add_unit_test(geom test_projection ENABLE_IF ${PROJ_FOUND} LIBS ${PROJ_LIBRARY}) +add_unit_test(geom test_tile) add_unit_test(geom test_wkb) add_unit_test(geom test_wkt) add_unit_test(index test_id_to_location ENABLE_IF ${SPARSEHASH_FOUND}) -add_unit_test(index test_typed_mmap) -add_unit_test(index test_typed_mmap_grow LABELS "fails_on_windows") add_unit_test(io test_bzip2 ENABLE_IF ${BZIP2_FOUND} LIBS ${BZIP2_LIBRARIES}) add_unit_test(io test_file_formats) add_unit_test(io test_reader LIBS "${OSMIUM_XML_LIBRARIES}") add_unit_test(io test_output_iterator ENABLE_IF ${Threads_FOUND} LIBS ${CMAKE_THREAD_LIBS_INIT}) +add_unit_test(io test_string_table) add_unit_test(tags test_filter) add_unit_test(tags test_operators) @@ -136,7 +144,12 @@ add_unit_test(tags test_tag_list) add_unit_test(thread test_pool ENABLE_IF ${Threads_FOUND} LIBS ${CMAKE_THREAD_LIBS_INIT}) add_unit_test(util test_cast_with_assert) +add_unit_test(util test_data_file) +add_unit_test(util test_delta) add_unit_test(util test_double) +add_unit_test(util test_file) +add_unit_test(util test_memory_mapping) +add_unit_test(util test_minmax) add_unit_test(util test_options) add_unit_test(util test_string) diff --git a/test/data-tests/testdata-testcases.cpp b/test/data-tests/testdata-testcases.cpp index 6ed6ae91a2d..0ea7fc859f3 100644 --- a/test/data-tests/testdata-testcases.cpp +++ b/test/data-tests/testdata-testcases.cpp @@ -6,8 +6,6 @@ #include "testdata-testcases.hpp" -#include - std::string dirname; int main(int argc, char* argv[]) { diff --git a/test/data-tests/testdata-xml.cpp b/test/data-tests/testdata-xml.cpp index 5af4c4f27f6..8102759d1ec 100644 --- a/test/data-tests/testdata-xml.cpp +++ b/test/data-tests/testdata-xml.cpp @@ -12,6 +12,10 @@ #include #include +std::string S_(const char* s) { + return std::string(s); +} + std::string filename(const char* test_id, const char* suffix = "osm") { const char* testdir = getenv("TESTDIR"); if (!testdir) { @@ -286,12 +290,13 @@ TEST_CASE("Reading OSM XML 121") { } SECTION("Using Reader") { - REQUIRE_THROWS_AS({ + // can throw osmium::gzip_error or osmium::xml_error + REQUIRE_THROWS({ osmium::io::Reader reader(filename("121-truncated_gzip_file", "osm.gz")); osmium::io::Header header = reader.header(); osmium::memory::Buffer buffer = reader.read(); reader.close(); - }, osmium::gzip_error); + }); } } @@ -337,25 +342,25 @@ TEST_CASE("Reading OSM XML 140") { auto len = atoi(t["unicode_utf8_length"]); REQUIRE(len == strlen(uc)); - REQUIRE(!strcmp(uc, t["unicode_xml"])); + REQUIRE(S_(uc) == t["unicode_xml"]); // workaround for missing support for u8 string literals on Windows #if !defined(_MSC_VER) switch (count) { case 1: - REQUIRE(!strcmp(uc, u8"a")); + REQUIRE(S_(uc) == u8"a"); break; case 2: - REQUIRE(!strcmp(uc, u8"\u00e4")); + REQUIRE(S_(uc) == u8"\u00e4"); break; case 3: - REQUIRE(!strcmp(uc, u8"\u30dc")); + REQUIRE(S_(uc) == u8"\u30dc"); break; case 4: - REQUIRE(!strcmp(uc, u8"\U0001d11e")); + REQUIRE(S_(uc) == u8"\U0001d11e"); break; case 5: - REQUIRE(!strcmp(uc, u8"\U0001f6eb")); + REQUIRE(S_(uc) == u8"\U0001f6eb"); break; default: REQUIRE(false); // should not be here @@ -382,11 +387,100 @@ TEST_CASE("Reading OSM XML 141") { const osmium::Node& node = buffer.get(0); const osmium::TagList& tags = node.tags(); - REQUIRE(!strcmp(tags["less-than"], "<")); - REQUIRE(!strcmp(tags["greater-than"], ">")); - REQUIRE(!strcmp(tags["apostrophe"], "'")); - REQUIRE(!strcmp(tags["ampersand"], "&")); - REQUIRE(!strcmp(tags["quote"], "\"")); + REQUIRE(S_(tags["less-than"]) == "<"); + REQUIRE(S_(tags["greater-than"]) == ">"); + REQUIRE(S_(tags["apostrophe"]) == "'"); + REQUIRE(S_(tags["ampersand"]) == "&"); + REQUIRE(S_(tags["quote"]) == "\""); + } + +} + + +// ============================================= + +TEST_CASE("Reading OSM XML 142") { + + SECTION("Using Reader to read nodes") { + osmium::io::Reader reader(filename("142-whitespace")); + osmium::memory::Buffer buffer = reader.read(); + reader.close(); + + int count = 0; + for (auto it = buffer.begin(); it != buffer.end(); ++it) { + ++count; + REQUIRE(it->id() == count); + REQUIRE(it->tags().size() == 1); + const osmium::Tag& tag = *(it->tags().begin()); + + switch (count) { + case 1: + REQUIRE(S_(it->user()) == "user name"); + REQUIRE(S_(tag.key()) == "key with space"); + REQUIRE(S_(tag.value()) == "value with space"); + break; + case 2: + REQUIRE(S_(it->user()) == "line\nfeed"); + REQUIRE(S_(tag.key()) == "key with\nlinefeed"); + REQUIRE(S_(tag.value()) == "value with\nlinefeed"); + break; + case 3: + REQUIRE(S_(it->user()) == "carriage\rreturn"); + REQUIRE(S_(tag.key()) == "key with\rcarriage\rreturn"); + REQUIRE(S_(tag.value()) == "value with\rcarriage\rreturn"); + break; + case 4: + REQUIRE(S_(it->user()) == "tab\tulator"); + REQUIRE(S_(tag.key()) == "key with\ttab"); + REQUIRE(S_(tag.value()) == "value with\ttab"); + break; + case 5: + REQUIRE(S_(it->user()) == "unencoded linefeed"); + REQUIRE(S_(tag.key()) == "key with unencoded linefeed"); + REQUIRE(S_(tag.value()) == "value with unencoded linefeed"); + break; + default: + REQUIRE(false); // should not be here + } + } + REQUIRE(count == 5); + } + + SECTION("Using Reader to read relation") { + osmium::io::Reader reader(filename("142-whitespace")); + osmium::memory::Buffer buffer = reader.read(); + reader.close(); + + auto it = buffer.begin(); + REQUIRE(it != buffer.end()); + REQUIRE(it->id() == 21); + const auto& members = it->members(); + REQUIRE(members.size() == 5); + + int count = 0; + for (const auto& member : members) { + ++count; + switch (count) { + case 1: + REQUIRE(S_(member.role()) == "role with whitespace"); + break; + case 2: + REQUIRE(S_(member.role()) == "role with\nlinefeed"); + break; + case 3: + REQUIRE(S_(member.role()) == "role with\rcarriage\rreturn"); + break; + case 4: + REQUIRE(S_(member.role()) == "role with\ttab"); + break; + case 5: + REQUIRE(S_(member.role()) == "role with unencoded linefeed"); + break; + default: + REQUIRE(false); // should not be here + } + } + REQUIRE(count == 5); } } diff --git a/test/include/catch.hpp b/test/include/catch.hpp index bb87af296d9..73abfe8c6c2 100644 --- a/test/include/catch.hpp +++ b/test/include/catch.hpp @@ -1,10 +1,6 @@ - -// This is needed for Windows -#define CATCH_CONFIG_CPP11_NULLPTR - /* - * CATCH v1.0 build 53 (master branch) - * Generated: 2014-08-20 08:08:19.533804 + * Catch v1.2.1 + * Generated: 2015-06-30 18:23:27.961086 * ---------------------------------------------------------- * This file has been merged from multiple headers. Please don't edit it directly * Copyright (c) 2012 Two Blue Cubes Ltd. All rights reserved. @@ -17,33 +13,43 @@ #define TWOBLUECUBES_CATCH_HPP_INCLUDED +#ifdef __clang__ +# pragma clang system_header +#elif defined __GNUC__ +# pragma GCC system_header +#endif + // #included from: internal/catch_suppress_warnings.h #define TWOBLUECUBES_CATCH_SUPPRESS_WARNINGS_H_INCLUDED #ifdef __clang__ -#pragma clang diagnostic ignored "-Wglobal-constructors" -#pragma clang diagnostic ignored "-Wvariadic-macros" -#pragma clang diagnostic ignored "-Wc99-extensions" -#pragma clang diagnostic ignored "-Wunused-variable" -#pragma clang diagnostic push -#pragma clang diagnostic ignored "-Wpadded" -#pragma clang diagnostic ignored "-Wc++98-compat" -#pragma clang diagnostic ignored "-Wc++98-compat-pedantic" +# ifdef __ICC // icpc defines the __clang__ macro +# pragma warning(push) +# pragma warning(disable: 161 1682) +# else // __ICC +# pragma clang diagnostic ignored "-Wglobal-constructors" +# pragma clang diagnostic ignored "-Wvariadic-macros" +# pragma clang diagnostic ignored "-Wc99-extensions" +# pragma clang diagnostic ignored "-Wunused-variable" +# pragma clang diagnostic push +# pragma clang diagnostic ignored "-Wpadded" +# pragma clang diagnostic ignored "-Wc++98-compat" +# pragma clang diagnostic ignored "-Wc++98-compat-pedantic" +# pragma clang diagnostic ignored "-Wswitch-enum" +# endif #elif defined __GNUC__ -#pragma GCC diagnostic ignored "-Wvariadic-macros" -#pragma GCC diagnostic ignored "-Wunused-variable" -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wpadded" -#pragma GCC diagnostic ignored "-Wctor-dtor-privacy" -#pragma GCC diagnostic ignored "-Wsign-promo" +# pragma GCC diagnostic ignored "-Wvariadic-macros" +# pragma GCC diagnostic ignored "-Wunused-variable" +# pragma GCC diagnostic push +# pragma GCC diagnostic ignored "-Wpadded" #endif -#ifdef CATCH_CONFIG_MAIN -# define CATCH_CONFIG_RUNNER +#if defined(CATCH_CONFIG_MAIN) || defined(CATCH_CONFIG_RUNNER) +# define CATCH_IMPL #endif -#ifdef CATCH_CONFIG_RUNNER +#ifdef CATCH_IMPL # ifndef CLARA_CONFIG_MAIN # define CLARA_CONFIG_MAIN_NOT_DEFINED # define CLARA_CONFIG_MAIN @@ -70,16 +76,34 @@ // #included from: catch_compiler_capabilities.h #define TWOBLUECUBES_CATCH_COMPILER_CAPABILITIES_HPP_INCLUDED -// Much of the following code is based on Boost (1.53) +// Detect a number of compiler features - mostly C++11/14 conformance - by compiler +// The following features are defined: +// +// CATCH_CONFIG_CPP11_NULLPTR : is nullptr supported? +// CATCH_CONFIG_CPP11_NOEXCEPT : is noexcept supported? +// CATCH_CONFIG_CPP11_GENERATED_METHODS : The delete and default keywords for compiler generated methods +// CATCH_CONFIG_CPP11_IS_ENUM : std::is_enum is supported? +// CATCH_CONFIG_CPP11_TUPLE : std::tuple is supported + +// CATCH_CONFIG_CPP11_OR_GREATER : Is C++11 supported? + +// CATCH_CONFIG_VARIADIC_MACROS : are variadic macros supported? + +// In general each macro has a _NO_ form +// (e.g. CATCH_CONFIG_CPP11_NO_NULLPTR) which disables the feature. +// Many features, at point of detection, define an _INTERNAL_ macro, so they +// can be combined, en-mass, with the _NO_ forms later. + +// All the C++11 features can be disabled with CATCH_CONFIG_NO_CPP11 #ifdef __clang__ # if __has_feature(cxx_nullptr) -# define CATCH_CONFIG_CPP11_NULLPTR +# define CATCH_INTERNAL_CONFIG_CPP11_NULLPTR # endif # if __has_feature(cxx_noexcept) -# define CATCH_CONFIG_CPP11_NOEXCEPT +# define CATCH_INTERNAL_CONFIG_CPP11_NOEXCEPT # endif #endif // __clang__ @@ -88,51 +112,26 @@ // Borland #ifdef __BORLANDC__ -#if (__BORLANDC__ > 0x582 ) -//#define CATCH_CONFIG_SFINAE // Not confirmed -#endif - #endif // __BORLANDC__ //////////////////////////////////////////////////////////////////////////////// // EDG #ifdef __EDG_VERSION__ -#if (__EDG_VERSION__ > 238 ) -//#define CATCH_CONFIG_SFINAE // Not confirmed -#endif - #endif // __EDG_VERSION__ //////////////////////////////////////////////////////////////////////////////// // Digital Mars #ifdef __DMC__ -#if (__DMC__ > 0x840 ) -//#define CATCH_CONFIG_SFINAE // Not confirmed -#endif - #endif // __DMC__ //////////////////////////////////////////////////////////////////////////////// // GCC #ifdef __GNUC__ -#if __GNUC__ < 3 - -#if (__GNUC_MINOR__ >= 96 ) -//#define CATCH_CONFIG_SFINAE -#endif - -#elif __GNUC__ >= 3 - -// #define CATCH_CONFIG_SFINAE // Taking this out completely for now - -#endif // __GNUC__ < 3 - #if __GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 6 && defined(__GXX_EXPERIMENTAL_CXX0X__) ) - -#define CATCH_CONFIG_CPP11_NULLPTR +# define CATCH_INTERNAL_CONFIG_CPP11_NULLPTR #endif #endif // __GNUC__ @@ -141,8 +140,13 @@ // Visual C++ #ifdef _MSC_VER -#if (_MSC_VER >= 1310 ) // (VC++ 7.0+) -//#define CATCH_CONFIG_SFINAE // Not confirmed +#if (_MSC_VER >= 1600) +# define CATCH_INTERNAL_CONFIG_CPP11_NULLPTR +#endif + +#if (_MSC_VER >= 1900 ) // (VC++ 13 (VS2015)) +#define CATCH_INTERNAL_CONFIG_CPP11_NOEXCEPT +#define CATCH_INTERNAL_CONFIG_CPP11_GENERATED_METHODS #endif #endif // _MSC_VER @@ -153,21 +157,62 @@ ( defined __GNUC__ && __GNUC__ >= 3 ) || \ ( !defined __cplusplus && __STDC_VERSION__ >= 199901L || __cplusplus >= 201103L ) -#ifndef CATCH_CONFIG_NO_VARIADIC_MACROS -#define CATCH_CONFIG_VARIADIC_MACROS -#endif +#define CATCH_INTERNAL_CONFIG_VARIADIC_MACROS #endif //////////////////////////////////////////////////////////////////////////////// // C++ language feature support -// detect language version: -#if (__cplusplus == 201103L) -# define CATCH_CPP11 -# define CATCH_CPP11_OR_GREATER -#elif (__cplusplus >= 201103L) +// catch all support for C++11 +#if (__cplusplus >= 201103L) + # define CATCH_CPP11_OR_GREATER + +# if !defined(CATCH_INTERNAL_CONFIG_CPP11_NULLPTR) +# define CATCH_INTERNAL_CONFIG_CPP11_NULLPTR +# endif + +# ifndef CATCH_INTERNAL_CONFIG_CPP11_NOEXCEPT +# define CATCH_INTERNAL_CONFIG_CPP11_NOEXCEPT +# endif + +# ifndef CATCH_INTERNAL_CONFIG_CPP11_GENERATED_METHODS +# define CATCH_INTERNAL_CONFIG_CPP11_GENERATED_METHODS +# endif + +# ifndef CATCH_INTERNAL_CONFIG_CPP11_IS_ENUM +# define CATCH_INTERNAL_CONFIG_CPP11_IS_ENUM +# endif + +# ifndef CATCH_INTERNAL_CONFIG_CPP11_TUPLE +# define CATCH_INTERNAL_CONFIG_CPP11_TUPLE +# endif + +# ifndef CATCH_INTERNAL_CONFIG_VARIADIC_MACROS +# define CATCH_INTERNAL_CONFIG_VARIADIC_MACROS +# endif + +#endif // __cplusplus >= 201103L + +// Now set the actual defines based on the above + anything the user has configured +#if defined(CATCH_INTERNAL_CONFIG_CPP11_NULLPTR) && !defined(CATCH_CONFIG_CPP11_NO_NULLPTR) && !defined(CATCH_CONFIG_CPP11_NULLPTR) && !defined(CATCH_CONFIG_NO_CPP11) +# define CATCH_CONFIG_CPP11_NULLPTR +#endif +#if defined(CATCH_INTERNAL_CONFIG_CPP11_NOEXCEPT) && !defined(CATCH_CONFIG_CPP11_NO_NOEXCEPT) && !defined(CATCH_CONFIG_CPP11_NOEXCEPT) && !defined(CATCH_CONFIG_NO_CPP11) +# define CATCH_CONFIG_CPP11_NOEXCEPT +#endif +#if defined(CATCH_INTERNAL_CONFIG_CPP11_GENERATED_METHODS) && !defined(CATCH_CONFIG_CPP11_NO_GENERATED_METHODS) && !defined(CATCH_CONFIG_CPP11_GENERATED_METHODS) && !defined(CATCH_CONFIG_NO_CPP11) +# define CATCH_CONFIG_CPP11_GENERATED_METHODS +#endif +#if defined(CATCH_INTERNAL_CONFIG_CPP11_IS_ENUM) && !defined(CATCH_CONFIG_CPP11_NO_IS_ENUM) && !defined(CATCH_CONFIG_CPP11_IS_ENUM) && !defined(CATCH_CONFIG_NO_CPP11) +# define CATCH_CONFIG_CPP11_IS_ENUM +#endif +#if defined(CATCH_INTERNAL_CONFIG_CPP11_TUPLE) && !defined(CATCH_CONFIG_CPP11_NO_TUPLE) && !defined(CATCH_CONFIG_CPP11_TUPLE) && !defined(CATCH_CONFIG_NO_CPP11) +# define CATCH_CONFIG_CPP11_TUPLE +#endif +#if defined(CATCH_INTERNAL_CONFIG_VARIADIC_MACROS) && !defined(CATCH_CONFIG_NO_VARIADIC_MACROS) && !defined(CATCH_CONFIG_VARIADIC_MACROS) +#define CATCH_CONFIG_VARIADIC_MACROS #endif // noexcept support: @@ -182,8 +227,16 @@ namespace Catch { class NonCopyable { - NonCopyable( NonCopyable const& ); - void operator = ( NonCopyable const& ); +#ifdef CATCH_CONFIG_CPP11_GENERATED_METHODS + NonCopyable( NonCopyable const& ) = delete; + NonCopyable( NonCopyable && ) = delete; + NonCopyable& operator = ( NonCopyable const& ) = delete; + NonCopyable& operator = ( NonCopyable && ) = delete; +#else + NonCopyable( NonCopyable const& info ); + NonCopyable& operator = ( NonCopyable const& ); +#endif + protected: NonCopyable() {} virtual ~NonCopyable(); @@ -221,6 +274,7 @@ namespace Catch { void toLowerInPlace( std::string& s ); std::string toLower( std::string const& s ); std::string trim( std::string const& str ); + bool replaceInPlace( std::string& str, std::string const& replaceThis, std::string const& withThis ); struct pluralise { pluralise( std::size_t count, std::string const& label ); @@ -236,13 +290,14 @@ namespace Catch { SourceLineInfo(); SourceLineInfo( char const* _file, std::size_t _line ); SourceLineInfo( SourceLineInfo const& other ); -# ifdef CATCH_CPP11_OR_GREATER +# ifdef CATCH_CONFIG_CPP11_GENERATED_METHODS SourceLineInfo( SourceLineInfo && ) = default; SourceLineInfo& operator = ( SourceLineInfo const& ) = default; SourceLineInfo& operator = ( SourceLineInfo && ) = default; # endif bool empty() const; bool operator == ( SourceLineInfo const& other ) const; + bool operator < ( SourceLineInfo const& other ) const; std::string file; std::size_t line; @@ -473,7 +528,7 @@ namespace Catch { struct ITestCaseRegistry { virtual ~ITestCaseRegistry(); virtual std::vector const& getAllTests() const = 0; - virtual void getFilteredTests( TestSpec const& testSpec, IConfig const& config, std::vector& matchingTestCases ) const = 0; + virtual void getFilteredTests( TestSpec const& testSpec, IConfig const& config, std::vector& matchingTestCases, bool negated = false ) const = 0; }; } @@ -609,7 +664,9 @@ namespace Catch { Exception = 0x100 | FailureBit, ThrewException = Exception | 1, - DidntThrowException = Exception | 2 + DidntThrowException = Exception | 2, + + FatalErrorCondition = 0x200 | FailureBit }; }; @@ -622,11 +679,11 @@ namespace Catch { // ResultDisposition::Flags enum struct ResultDisposition { enum Flags { - Normal = 0x00, + Normal = 0x01, - ContinueOnFailure = 0x01, // Failures fail test, but execution continues - FalseTest = 0x02, // Prefix expression with ! - SuppressFail = 0x04 // Failures are reported but do not fail the test + ContinueOnFailure = 0x02, // Failures fail test, but execution continues + FalseTest = 0x04, // Prefix expression with ! + SuppressFail = 0x08 // Failures are reported but do not fail the test }; }; inline ResultDisposition::Flags operator | ( ResultDisposition::Flags lhs, ResultDisposition::Flags rhs ) { @@ -674,7 +731,7 @@ namespace Catch { AssertionResult(); AssertionResult( AssertionInfo const& info, AssertionResultData const& data ); ~AssertionResult(); -# ifdef CATCH_CPP11_OR_GREATER +# ifdef CATCH_CONFIG_CPP11_GENERATED_METHODS AssertionResult( AssertionResult const& ) = default; AssertionResult( AssertionResult && ) = default; AssertionResult& operator = ( AssertionResult const& ) = default; @@ -730,8 +787,8 @@ namespace Catch { ResultDisposition::Flags resultDisposition ); template - ExpressionLhs operator->* ( T const& operand ); - ExpressionLhs operator->* ( bool value ); + ExpressionLhs operator <= ( T const& operand ); + ExpressionLhs operator <= ( bool value ); template ResultBuilder& operator << ( T const& value ) { @@ -954,40 +1011,6 @@ namespace Internal { // #included from: catch_tostring.h #define TWOBLUECUBES_CATCH_TOSTRING_H_INCLUDED -// #included from: catch_sfinae.hpp -#define TWOBLUECUBES_CATCH_SFINAE_HPP_INCLUDED - -// Try to detect if the current compiler supports SFINAE - -namespace Catch { - - struct TrueType { - static const bool value = true; - typedef void Enable; - char sizer[1]; - }; - struct FalseType { - static const bool value = false; - typedef void Disable; - char sizer[2]; - }; - -#ifdef CATCH_CONFIG_SFINAE - - template struct NotABooleanExpression; - - template struct If : NotABooleanExpression {}; - template<> struct If : TrueType {}; - template<> struct If : FalseType {}; - - template struct SizedIf; - template<> struct SizedIf : TrueType {}; - template<> struct SizedIf : FalseType {}; - -#endif // CATCH_CONFIG_SFINAE - -} // end namespace Catch - #include #include #include @@ -1040,35 +1063,59 @@ inline id performOptionalSelector( id obj, SEL sel ) { #endif +#ifdef CATCH_CONFIG_CPP11_TUPLE +#include +#endif + +#ifdef CATCH_CONFIG_CPP11_IS_ENUM +#include +#endif + namespace Catch { -namespace Detail { -// SFINAE is currently disabled by default for all compilers. -// If the non SFINAE version of IsStreamInsertable is ambiguous for you -// and your compiler supports SFINAE, try #defining CATCH_CONFIG_SFINAE -#ifdef CATCH_CONFIG_SFINAE +// Why we're here. +template +std::string toString( T const& value ); - template - class IsStreamInsertableHelper { - template struct TrueIfSizeable : TrueType {}; +// Built in overloads + +std::string toString( std::string const& value ); +std::string toString( std::wstring const& value ); +std::string toString( const char* const value ); +std::string toString( char* const value ); +std::string toString( const wchar_t* const value ); +std::string toString( wchar_t* const value ); +std::string toString( int value ); +std::string toString( unsigned long value ); +std::string toString( unsigned int value ); +std::string toString( const double value ); +std::string toString( const float value ); +std::string toString( bool value ); +std::string toString( char value ); +std::string toString( signed char value ); +std::string toString( unsigned char value ); - template - static TrueIfSizeable dummy(T2*); - static FalseType dummy(...); +#ifdef CATCH_CONFIG_CPP11_NULLPTR +std::string toString( std::nullptr_t ); +#endif - public: - typedef SizedIf type; - }; +#ifdef __OBJC__ + std::string toString( NSString const * const& nsstring ); + std::string toString( NSString * CATCH_ARC_STRONG const& nsstring ); + std::string toString( NSObject* const& nsObject ); +#endif - template - struct IsStreamInsertable : IsStreamInsertableHelper::type {}; +namespace Detail { -#else + extern std::string unprintableString; struct BorgType { template BorgType( T const& ); }; + struct TrueType { char sizer[1]; }; + struct FalseType { char sizer[2]; }; + TrueType& testStreamable( std::ostream& ); FalseType testStreamable( FalseType ); @@ -1081,12 +1128,38 @@ namespace Detail { enum { value = sizeof( testStreamable(s << t) ) == sizeof( TrueType ) }; }; -#endif +#if defined(CATCH_CONFIG_CPP11_IS_ENUM) + template::value + > + struct EnumStringMaker + { + static std::string convert( T const& ) { return unprintableString; } + }; + template + struct EnumStringMaker + { + static std::string convert( T const& v ) + { + return ::Catch::toString( + static_cast::type>(v) + ); + } + }; +#endif template struct StringMakerBase { +#if defined(CATCH_CONFIG_CPP11_IS_ENUM) + template + static std::string convert( T const& v ) + { + return EnumStringMaker::convert( v ); + } +#else template - static std::string convert( T const& ) { return "{?}"; } + static std::string convert( T const& ) { return unprintableString; } +#endif }; template<> @@ -1108,9 +1181,6 @@ namespace Detail { } // end namespace Detail -template -std::string toString( T const& value ); - template struct StringMaker : Detail::StringMakerBase::value> {}; @@ -1141,12 +1211,59 @@ namespace Detail { std::string rangeToString( InputIterator first, InputIterator last ); } +//template +//struct StringMaker > { +// static std::string convert( std::vector const& v ) { +// return Detail::rangeToString( v.begin(), v.end() ); +// } +//}; + template -struct StringMaker > { - static std::string convert( std::vector const& v ) { - return Detail::rangeToString( v.begin(), v.end() ); +std::string toString( std::vector const& v ) { + return Detail::rangeToString( v.begin(), v.end() ); +} + +#ifdef CATCH_CONFIG_CPP11_TUPLE + +// toString for tuples +namespace TupleDetail { + template< + typename Tuple, + std::size_t N = 0, + bool = (N < std::tuple_size::value) + > + struct ElementPrinter { + static void print( const Tuple& tuple, std::ostream& os ) + { + os << ( N ? ", " : " " ) + << Catch::toString(std::get(tuple)); + ElementPrinter::print(tuple,os); + } + }; + + template< + typename Tuple, + std::size_t N + > + struct ElementPrinter { + static void print( const Tuple&, std::ostream& ) {} + }; + +} + +template +struct StringMaker> { + + static std::string convert( const std::tuple& tuple ) + { + std::ostringstream os; + os << '{'; + TupleDetail::ElementPrinter>::print( tuple, os ); + os << " }"; + return os.str(); } }; +#endif // CATCH_CONFIG_CPP11_TUPLE namespace Detail { template @@ -1167,44 +1284,15 @@ std::string toString( T const& value ) { return StringMaker::convert( value ); } -// Built in overloads - -std::string toString( std::string const& value ); -std::string toString( std::wstring const& value ); -std::string toString( const char* const value ); -std::string toString( char* const value ); -std::string toString( const wchar_t* const value ); -std::string toString( wchar_t* const value ); -std::string toString( int value ); -std::string toString( unsigned long value ); -std::string toString( unsigned int value ); -std::string toString( const double value ); -std::string toString( const float value ); -std::string toString( bool value ); -std::string toString( char value ); -std::string toString( signed char value ); -std::string toString( unsigned char value ); - -#ifdef CATCH_CONFIG_CPP11_NULLPTR -std::string toString( std::nullptr_t ); -#endif - -#ifdef __OBJC__ - std::string toString( NSString const * const& nsstring ); - std::string toString( NSString * CATCH_ARC_STRONG const& nsstring ); - std::string toString( NSObject* const& nsObject ); -#endif - namespace Detail { template std::string rangeToString( InputIterator first, InputIterator last ) { std::ostringstream oss; oss << "{ "; if( first != last ) { - oss << toString( *first ); - for( ++first ; first != last ; ++first ) { - oss << ", " << toString( *first ); - } + oss << Catch::toString( *first ); + for( ++first ; first != last ; ++first ) + oss << ", " << Catch::toString( *first ); } oss << " }"; return oss.str(); @@ -1220,13 +1308,13 @@ namespace Catch { template class ExpressionLhs { ExpressionLhs& operator = ( ExpressionLhs const& ); -# ifdef CATCH_CPP11_OR_GREATER +# ifdef CATCH_CONFIG_CPP11_GENERATED_METHODS ExpressionLhs& operator = ( ExpressionLhs && ) = delete; # endif public: ExpressionLhs( ResultBuilder& rb, T lhs ) : m_rb( rb ), m_lhs( lhs ) {} -# ifdef CATCH_CPP11_OR_GREATER +# ifdef CATCH_CONFIG_CPP11_GENERATED_METHODS ExpressionLhs( ExpressionLhs const& ) = default; ExpressionLhs( ExpressionLhs && ) = default; # endif @@ -1307,11 +1395,11 @@ class ExpressionLhs { namespace Catch { template - inline ExpressionLhs ResultBuilder::operator->* ( T const& operand ) { + inline ExpressionLhs ResultBuilder::operator <= ( T const& operand ) { return ExpressionLhs( *this, operand ); } - inline ExpressionLhs ResultBuilder::operator->* ( bool value ) { + inline ExpressionLhs ResultBuilder::operator <= ( bool value ) { return ExpressionLhs( *this, value ); } @@ -1401,6 +1489,8 @@ namespace Catch { virtual std::string getCurrentTestName() const = 0; virtual const AssertionResult* getLastResult() const = 0; + + virtual void handleFatalErrorCondition( std::string const& message ) = 0; }; IResultCapture& getResultCapture(); @@ -1481,7 +1571,7 @@ namespace Catch { do { \ Catch::ResultBuilder __catchResult( macroName, CATCH_INTERNAL_LINEINFO, #expr, resultDisposition ); \ try { \ - ( __catchResult->*expr ).endExpression(); \ + ( __catchResult <= expr ).endExpression(); \ } \ catch( ... ) { \ __catchResult.useActiveException( Catch::ResultDisposition::Normal ); \ @@ -1581,7 +1671,7 @@ namespace Catch { std::string matcherAsString = ::Catch::Matchers::matcher.toString(); \ __catchResult \ .setLhs( Catch::toString( arg ) ) \ - .setRhs( matcherAsString == "{?}" ? #matcher : matcherAsString ) \ + .setRhs( matcherAsString == Catch::Detail::unprintableString ? #matcher : matcherAsString ) \ .setOp( "matches" ) \ .setResultType( ::Catch::Matchers::matcher.match( arg ) ); \ __catchResult.captureExpression(); \ @@ -1642,6 +1732,9 @@ namespace Catch { bool allPassed() const { return failed == 0 && failedButOk == 0; } + bool allOk() const { + return failed == 0; + } std::size_t passed; std::size_t failed; @@ -1694,7 +1787,7 @@ namespace Catch { public: Timer() : m_ticks( 0 ) {} void start(); - unsigned int getElapsedNanoseconds() const; + unsigned int getElapsedMicroseconds() const; unsigned int getElapsedMilliseconds() const; double getElapsedSeconds() const; @@ -1708,7 +1801,7 @@ namespace Catch { namespace Catch { - class Section { + class Section : NonCopyable { public: Section( SectionInfo const& info ); ~Section(); @@ -1717,15 +1810,6 @@ namespace Catch { operator bool() const; private: -#ifdef CATCH_CPP11_OR_GREATER - Section( Section const& ) = delete; - Section( Section && ) = delete; - Section& operator = ( Section const& ) = delete; - Section& operator = ( Section && ) = delete; -#else - Section( Section const& info ); - Section& operator = ( Section const& ); -#endif SectionInfo m_info; std::string m_name; @@ -2700,7 +2784,7 @@ return @ desc; \ #endif -#ifdef CATCH_CONFIG_RUNNER +#ifdef CATCH_IMPL // #included from: internal/catch_impl.hpp #define TWOBLUECUBES_CATCH_IMPL_HPP_INCLUDED @@ -2712,7 +2796,7 @@ return @ desc; \ #pragma clang diagnostic ignored "-Wweak-vtables" #endif -// #included from: catch_runner.hpp +// #included from: ../catch_runner.hpp #define TWOBLUECUBES_CATCH_RUNNER_HPP_INCLUDED // #included from: internal/catch_commandline.hpp @@ -2968,6 +3052,11 @@ namespace Catch { Always, Never }; }; + struct RunTests { enum InWhatOrder { + InDeclarationOrder, + InLexicographicalOrder, + InRandomOrder + }; }; class TestSpec; @@ -2985,6 +3074,9 @@ namespace Catch { virtual bool showInvisibles() const = 0; virtual ShowDurations::OrNot showDurations() const = 0; virtual TestSpec const& testSpec() const = 0; + virtual RunTests::InWhatOrder runOrder() const = 0; + virtual unsigned int rngSeed() const = 0; + virtual bool forceColour() const = 0; }; } @@ -3010,12 +3102,16 @@ namespace Catch { private: bool isOwned; }; + + std::ostream& cout(); + std::ostream& cerr(); } #include #include #include #include +#include #ifndef CATCH_CONFIG_CONSOLE_WIDTH #define CATCH_CONFIG_CONSOLE_WIDTH 80 @@ -3035,10 +3131,13 @@ namespace Catch { noThrow( false ), showHelp( false ), showInvisibles( false ), + forceColour( false ), abortAfter( -1 ), + rngSeed( 0 ), verbosity( Verbosity::Normal ), warnings( WarnAbout::Nothing ), - showDurations( ShowDurations::DefaultForReporter ) + showDurations( ShowDurations::DefaultForReporter ), + runOrder( RunTests::InDeclarationOrder ) {} bool listTests; @@ -3051,12 +3150,15 @@ namespace Catch { bool noThrow; bool showHelp; bool showInvisibles; + bool forceColour; int abortAfter; + unsigned int rngSeed; Verbosity::Level verbosity; WarnAbout::What warnings; ShowDurations::OrNot showDurations; + RunTests::InWhatOrder runOrder; std::string reporterName; std::string outputFilename; @@ -3074,12 +3176,12 @@ namespace Catch { public: Config() - : m_os( std::cout.rdbuf() ) + : m_os( Catch::cout().rdbuf() ) {} Config( ConfigData const& data ) : m_data( data ), - m_os( std::cout.rdbuf() ) + m_os( Catch::cout().rdbuf() ) { if( !data.testsOrTags.empty() ) { TestSpecParser parser( ITagAliasRegistry::get() ); @@ -3090,7 +3192,7 @@ namespace Catch { } virtual ~Config() { - m_os.rdbuf( std::cout.rdbuf() ); + m_os.rdbuf( Catch::cout().rdbuf() ); m_stream.release(); } @@ -3112,7 +3214,7 @@ namespace Catch { bool shouldDebugBreak() const { return m_data.shouldDebugBreak; } void setStreamBuf( std::streambuf* buf ) { - m_os.rdbuf( buf ? buf : std::cout.rdbuf() ); + m_os.rdbuf( buf ? buf : Catch::cout().rdbuf() ); } void useStream( std::string const& streamName ) { @@ -3138,6 +3240,9 @@ namespace Catch { virtual bool includeSuccessfulResults() const { return m_data.showSuccessfulTests; } virtual bool warnAboutMissingAssertions() const { return m_data.warnings & WarnAbout::NoAssertions; } virtual ShowDurations::OrNot showDurations() const { return m_data.showDurations; } + virtual RunTests::InWhatOrder runOrder() const { return m_data.runOrder; } + virtual unsigned int rngSeed() const { return m_data.rngSeed; } + virtual bool forceColour() const { return m_data.forceColour; } private: ConfigData m_data; @@ -3399,7 +3504,7 @@ namespace Clara { template struct IArgFunction { virtual ~IArgFunction() {} -# ifdef CATCH_CPP11_OR_GREATER +# ifdef CATCH_CONFIG_CPP11_GENERATED_METHODS IArgFunction() = default; IArgFunction( IArgFunction const& ) = default; # endif @@ -3766,7 +3871,7 @@ namespace Clara { m_throwOnUnrecognisedTokens( other.m_throwOnUnrecognisedTokens ) { if( other.m_floatingArg.get() ) - m_floatingArg = ArgAutoPtr( new Arg( *other.m_floatingArg ) ); + m_floatingArg.reset( new Arg( *other.m_floatingArg ) ); } CommandLine& setThrowOnUnrecognisedTokens( bool shouldThrow = true ) { @@ -3794,7 +3899,7 @@ namespace Clara { ArgBuilder operator[]( UnpositionalTag ) { if( m_floatingArg.get() ) throw std::logic_error( "Only one unpositional argument can be added" ); - m_floatingArg = ArgAutoPtr( new Arg() ); + m_floatingArg.reset( new Arg() ); ArgBuilder builder( m_floatingArg.get() ); return builder; } @@ -3936,7 +4041,7 @@ namespace Clara { if( it == itEnd ) { if( token.type == Parser::Token::Positional || !m_throwOnUnrecognisedTokens ) unusedTokens.push_back( token ); - else if( m_throwOnUnrecognisedTokens ) + else if( errors.empty() && m_throwOnUnrecognisedTokens ) errors.push_back( "unrecognised option: " + token.data ); } } @@ -4034,7 +4139,28 @@ namespace Catch { config.warnings = static_cast( config.warnings | WarnAbout::NoAssertions ); else throw std::runtime_error( "Unrecognised warning: '" + _warning + "'" ); - + } + inline void setOrder( ConfigData& config, std::string const& order ) { + if( startsWith( "declared", order ) ) + config.runOrder = RunTests::InDeclarationOrder; + else if( startsWith( "lexical", order ) ) + config.runOrder = RunTests::InLexicographicalOrder; + else if( startsWith( "random", order ) ) + config.runOrder = RunTests::InRandomOrder; + else + throw std::runtime_error( "Unrecognised ordering: '" + order + "'" ); + } + inline void setRngSeed( ConfigData& config, std::string const& seed ) { + if( seed == "time" ) { + config.rngSeed = static_cast( std::time(0) ); + } + else { + std::stringstream ss; + ss << seed; + ss >> config.rngSeed; + if( ss.fail() ) + throw std::runtime_error( "Argment to --rng-seed should be the word 'time' or a number" ); + } } inline void setVerbosity( ConfigData& config, int level ) { // !TBD: accept strings? @@ -4146,6 +4272,18 @@ namespace Catch { .describe( "list all reporters" ) .bind( &ConfigData::listReporters ); + cli["--order"] + .describe( "test case order (defaults to decl)" ) + .bind( &setOrder, "decl|lex|rand" ); + + cli["--rng-seed"] + .describe( "set a specific seed for random numbers" ) + .bind( &setRngSeed, "'time'|number" ); + + cli["--force-colour"] + .describe( "force colourised output" ) + .bind( &ConfigData::forceColour ); + return cli; } @@ -4319,10 +4457,6 @@ namespace Catch { namespace Catch { - namespace Detail { - struct IColourImpl; - } - struct Colour { enum Code { None = 0, @@ -4368,7 +4502,6 @@ namespace Catch { static void use( Code _colourCode ); private: - static Detail::IColourImpl* impl(); bool m_moved; }; @@ -4462,7 +4595,7 @@ namespace Catch } virtual ~AssertionStats(); -# ifdef CATCH_CPP11_OR_GREATER +# ifdef CATCH_CONFIG_CPP11_GENERATED_METHODS AssertionStats( AssertionStats const& ) = default; AssertionStats( AssertionStats && ) = default; AssertionStats& operator = ( AssertionStats const& ) = default; @@ -4485,7 +4618,7 @@ namespace Catch missingAssertions( _missingAssertions ) {} virtual ~SectionStats(); -# ifdef CATCH_CPP11_OR_GREATER +# ifdef CATCH_CONFIG_CPP11_GENERATED_METHODS SectionStats( SectionStats const& ) = default; SectionStats( SectionStats && ) = default; SectionStats& operator = ( SectionStats const& ) = default; @@ -4512,7 +4645,7 @@ namespace Catch {} virtual ~TestCaseStats(); -# ifdef CATCH_CPP11_OR_GREATER +# ifdef CATCH_CONFIG_CPP11_GENERATED_METHODS TestCaseStats( TestCaseStats const& ) = default; TestCaseStats( TestCaseStats && ) = default; TestCaseStats& operator = ( TestCaseStats const& ) = default; @@ -4540,7 +4673,7 @@ namespace Catch {} virtual ~TestGroupStats(); -# ifdef CATCH_CPP11_OR_GREATER +# ifdef CATCH_CONFIG_CPP11_GENERATED_METHODS TestGroupStats( TestGroupStats const& ) = default; TestGroupStats( TestGroupStats && ) = default; TestGroupStats& operator = ( TestGroupStats const& ) = default; @@ -4562,7 +4695,7 @@ namespace Catch {} virtual ~TestRunStats(); -# ifndef CATCH_CPP11_OR_GREATER +# ifndef CATCH_CONFIG_CPP11_GENERATED_METHODS TestRunStats( TestRunStats const& _other ) : runInfo( _other.runInfo ), totals( _other.totals ), @@ -4598,11 +4731,14 @@ namespace Catch virtual void assertionStarting( AssertionInfo const& assertionInfo ) = 0; + // The return value indicates if the messages buffer should be cleared: virtual bool assertionEnded( AssertionStats const& assertionStats ) = 0; virtual void sectionEnded( SectionStats const& sectionStats ) = 0; virtual void testCaseEnded( TestCaseStats const& testCaseStats ) = 0; virtual void testGroupEnded( TestGroupStats const& testGroupStats ) = 0; virtual void testRunEnded( TestRunStats const& testRunStats ) = 0; + + virtual void skipTest( TestCaseInfo const& testInfo ) = 0; }; struct IReporterFactory { @@ -4630,9 +4766,9 @@ namespace Catch { TestSpec testSpec = config.testSpec(); if( config.testSpec().hasFilters() ) - std::cout << "Matching test cases:\n"; + Catch::cout() << "Matching test cases:\n"; else { - std::cout << "All available test cases:\n"; + Catch::cout() << "All available test cases:\n"; testSpec = TestSpecParser( ITagAliasRegistry::get() ).parse( "*" ).testSpec(); } @@ -4653,15 +4789,15 @@ namespace Catch { : Colour::None; Colour colourGuard( colour ); - std::cout << Text( testCaseInfo.name, nameAttr ) << std::endl; + Catch::cout() << Text( testCaseInfo.name, nameAttr ) << std::endl; if( !testCaseInfo.tags.empty() ) - std::cout << Text( testCaseInfo.tagsAsString, tagsAttr ) << std::endl; + Catch::cout() << Text( testCaseInfo.tagsAsString, tagsAttr ) << std::endl; } if( !config.testSpec().hasFilters() ) - std::cout << pluralise( matchedTests, "test case" ) << "\n" << std::endl; + Catch::cout() << pluralise( matchedTests, "test case" ) << "\n" << std::endl; else - std::cout << pluralise( matchedTests, "matching test case" ) << "\n" << std::endl; + Catch::cout() << pluralise( matchedTests, "matching test case" ) << "\n" << std::endl; return matchedTests; } @@ -4677,7 +4813,7 @@ namespace Catch { ++it ) { matchedTests++; TestCaseInfo const& testCaseInfo = it->getTestCaseInfo(); - std::cout << testCaseInfo.name << std::endl; + Catch::cout() << testCaseInfo.name << std::endl; } return matchedTests; } @@ -4703,9 +4839,9 @@ namespace Catch { inline std::size_t listTags( Config const& config ) { TestSpec testSpec = config.testSpec(); if( config.testSpec().hasFilters() ) - std::cout << "Tags for matching test cases:\n"; + Catch::cout() << "Tags for matching test cases:\n"; else { - std::cout << "All available tags:\n"; + Catch::cout() << "All available tags:\n"; testSpec = TestSpecParser( ITagAliasRegistry::get() ).parse( "*" ).testSpec(); } @@ -4739,14 +4875,14 @@ namespace Catch { .setInitialIndent( 0 ) .setIndent( oss.str().size() ) .setWidth( CATCH_CONFIG_CONSOLE_WIDTH-10 ) ); - std::cout << oss.str() << wrapper << "\n"; + Catch::cout() << oss.str() << wrapper << "\n"; } - std::cout << pluralise( tagCounts.size(), "tag" ) << "\n" << std::endl; + Catch::cout() << pluralise( tagCounts.size(), "tag" ) << "\n" << std::endl; return tagCounts.size(); } inline std::size_t listReporters( Config const& /*config*/ ) { - std::cout << "Available reports:\n"; + Catch::cout() << "Available reporters:\n"; IReporterRegistry::FactoryMap const& factories = getRegistryHub().getReporterRegistry().getFactories(); IReporterRegistry::FactoryMap::const_iterator itBegin = factories.begin(), itEnd = factories.end(), it; std::size_t maxNameLen = 0; @@ -4758,13 +4894,13 @@ namespace Catch { .setInitialIndent( 0 ) .setIndent( 7+maxNameLen ) .setWidth( CATCH_CONFIG_CONSOLE_WIDTH - maxNameLen-8 ) ); - std::cout << " " + Catch::cout() << " " << it->first << ":" << std::string( maxNameLen - it->first.size() + 2, ' ' ) << wrapper << "\n"; } - std::cout << std::endl; + Catch::cout() << std::endl; return factories.size(); } @@ -4814,32 +4950,15 @@ namespace SectionTracking { RunState runState() const { return m_runState; } - TrackedSection* findChild( std::string const& childName ) { - TrackedSections::iterator it = m_children.find( childName ); - return it != m_children.end() - ? &it->second - : NULL; - } - TrackedSection* acquireChild( std::string const& childName ) { - if( TrackedSection* child = findChild( childName ) ) - return child; - m_children.insert( std::make_pair( childName, TrackedSection( childName, this ) ) ); - return findChild( childName ); - } + TrackedSection* findChild( std::string const& childName ); + TrackedSection* acquireChild( std::string const& childName ); + void enter() { if( m_runState == NotStarted ) m_runState = Executing; } - void leave() { - for( TrackedSections::const_iterator it = m_children.begin(), itEnd = m_children.end(); - it != itEnd; - ++it ) - if( it->second.runState() != Completed ) { - m_runState = ExecutingChildren; - return; - } - m_runState = Completed; - } + void leave(); + TrackedSection* getParent() { return m_parent; } @@ -4852,9 +4971,31 @@ namespace SectionTracking { RunState m_runState; TrackedSections m_children; TrackedSection* m_parent; - }; + inline TrackedSection* TrackedSection::findChild( std::string const& childName ) { + TrackedSections::iterator it = m_children.find( childName ); + return it != m_children.end() + ? &it->second + : NULL; + } + inline TrackedSection* TrackedSection::acquireChild( std::string const& childName ) { + if( TrackedSection* child = findChild( childName ) ) + return child; + m_children.insert( std::make_pair( childName, TrackedSection( childName, this ) ) ); + return findChild( childName ); + } + inline void TrackedSection::leave() { + for( TrackedSections::const_iterator it = m_children.begin(), itEnd = m_children.end(); + it != itEnd; + ++it ) + if( it->second.runState() != Completed ) { + m_runState = ExecutingChildren; + return; + } + m_runState = Completed; + } + class TestCaseTracker { public: TestCaseTracker( std::string const& testCaseName ) @@ -4921,6 +5062,81 @@ using SectionTracking::TestCaseTracker; } // namespace Catch +// #included from: catch_fatal_condition.hpp +#define TWOBLUECUBES_CATCH_FATAL_CONDITION_H_INCLUDED + +namespace Catch { + + // Report the error condition then exit the process + inline void fatal( std::string const& message, int exitCode ) { + IContext& context = Catch::getCurrentContext(); + IResultCapture* resultCapture = context.getResultCapture(); + resultCapture->handleFatalErrorCondition( message ); + + if( Catch::alwaysTrue() ) // avoids "no return" warnings + exit( exitCode ); + } + +} // namespace Catch + +#if defined ( CATCH_PLATFORM_WINDOWS ) ///////////////////////////////////////// + +namespace Catch { + + struct FatalConditionHandler { + void reset() {} + }; + +} // namespace Catch + +#else // Not Windows - assumed to be POSIX compatible ////////////////////////// + +#include + +namespace Catch { + + struct SignalDefs { int id; const char* name; }; + extern SignalDefs signalDefs[]; + SignalDefs signalDefs[] = { + { SIGINT, "SIGINT - Terminal interrupt signal" }, + { SIGILL, "SIGILL - Illegal instruction signal" }, + { SIGFPE, "SIGFPE - Floating point error signal" }, + { SIGSEGV, "SIGSEGV - Segmentation violation signal" }, + { SIGTERM, "SIGTERM - Termination request signal" }, + { SIGABRT, "SIGABRT - Abort (abnormal termination) signal" } + }; + + struct FatalConditionHandler { + + static void handleSignal( int sig ) { + for( std::size_t i = 0; i < sizeof(signalDefs)/sizeof(SignalDefs); ++i ) + if( sig == signalDefs[i].id ) + fatal( signalDefs[i].name, -sig ); + fatal( "", -sig ); + } + + FatalConditionHandler() : m_isSet( true ) { + for( std::size_t i = 0; i < sizeof(signalDefs)/sizeof(SignalDefs); ++i ) + signal( signalDefs[i].id, handleSignal ); + } + ~FatalConditionHandler() { + reset(); + } + void reset() { + if( m_isSet ) { + for( std::size_t i = 0; i < sizeof(signalDefs)/sizeof(SignalDefs); ++i ) + signal( signalDefs[i].id, SIG_DFL ); + m_isSet = false; + } + } + + bool m_isSet; + }; + +} // namespace Catch + +#endif // not Windows + #include #include @@ -5076,7 +5292,7 @@ namespace Catch { } virtual void sectionEnded( SectionInfo const& info, Counts const& prevAssertions, double _durationInSeconds ) { - /*if( std::uncaught_exception() ) { // XXX Hack that makes Catch not run in loop in certain situations + /* if( std::uncaught_exception() ) { // XXX Hack that makes Catch not run in loop in certain situations m_unfinishedSections.push_back( UnfinishedSections( info, prevAssertions, _durationInSeconds ) ); return; }*/ @@ -5108,6 +5324,37 @@ namespace Catch { return &m_lastResult; } + virtual void handleFatalErrorCondition( std::string const& message ) { + ResultBuilder resultBuilder = makeUnexpectedResultBuilder(); + resultBuilder.setResultType( ResultWas::FatalErrorCondition ); + resultBuilder << message; + resultBuilder.captureExpression(); + + handleUnfinishedSections(); + + // Recreate section for test case (as we will lose the one that was in scope) + TestCaseInfo const& testCaseInfo = m_activeTestCase->getTestCaseInfo(); + SectionInfo testCaseSection( testCaseInfo.lineInfo, testCaseInfo.name, testCaseInfo.description ); + + Counts assertions; + assertions.failed = 1; + SectionStats testCaseSectionStats( testCaseSection, assertions, 0, false ); + m_reporter->sectionEnded( testCaseSectionStats ); + + TestCaseInfo testInfo = m_activeTestCase->getTestCaseInfo(); + + Totals deltaTotals; + deltaTotals.testCases.failed = 1; + m_reporter->testCaseEnded( TestCaseStats( testInfo, + deltaTotals, + "", + "", + false ) ); + m_totals.testCases.failed++; + testGroupEnded( "", m_totals, 1, 1 ); + m_reporter->testRunEnded( TestRunStats( m_runInfo, m_totals, false ) ); + } + public: // !TBD We need to do this another way! bool aborting() const { @@ -5129,12 +5376,12 @@ namespace Catch { Timer timer; timer.start(); if( m_reporter->getPreferences().shouldRedirectStdOut ) { - StreamRedirect coutRedir( std::cout, redirectedCout ); - StreamRedirect cerrRedir( std::cerr, redirectedCerr ); - m_activeTestCase->invoke(); + StreamRedirect coutRedir( Catch::cout(), redirectedCout ); + StreamRedirect cerrRedir( Catch::cerr(), redirectedCerr ); + invokeActiveTestCase(); } else { - m_activeTestCase->invoke(); + invokeActiveTestCase(); } duration = timer.getElapsedSeconds(); } @@ -5142,20 +5389,9 @@ namespace Catch { // This just means the test was aborted due to failure } catch(...) { - ResultBuilder exResult( m_lastAssertionInfo.macroName.c_str(), - m_lastAssertionInfo.lineInfo, - m_lastAssertionInfo.capturedExpression.c_str(), - m_lastAssertionInfo.resultDisposition ); - exResult.useActiveException(); + makeUnexpectedResultBuilder().useActiveException(); } - // If sections ended prematurely due to an exception we stored their - // infos here so we can tear them down outside the unwind process. - for( std::vector::const_reverse_iterator it = m_unfinishedSections.rbegin(), - itEnd = m_unfinishedSections.rend(); - it != itEnd; - ++it ) - sectionEnded( it->info, it->prevAssertions, it->durationInSeconds ); - m_unfinishedSections.clear(); + handleUnfinishedSections(); m_messages.clear(); Counts assertions = m_totals.assertions - prevAssertions; @@ -5171,7 +5407,32 @@ namespace Catch { m_reporter->sectionEnded( testCaseSectionStats ); } + void invokeActiveTestCase() { + FatalConditionHandler fatalConditionHandler; // Handle signals + m_activeTestCase->invoke(); + fatalConditionHandler.reset(); + } + private: + + ResultBuilder makeUnexpectedResultBuilder() const { + return ResultBuilder( m_lastAssertionInfo.macroName.c_str(), + m_lastAssertionInfo.lineInfo, + m_lastAssertionInfo.capturedExpression.c_str(), + m_lastAssertionInfo.resultDisposition ); + } + + void handleUnfinishedSections() { + // If sections ended prematurely due to an exception we stored their + // infos here so we can tear them down outside the unwind process. + for( std::vector::const_reverse_iterator it = m_unfinishedSections.rbegin(), + itEnd = m_unfinishedSections.rend(); + it != itEnd; + ++it ) + sectionEnded( it->info, it->prevAssertions, it->durationInSeconds ); + m_unfinishedSections.clear(); + } + struct UnfinishedSections { UnfinishedSections( SectionInfo const& _info, Counts const& _prevAssertions, double _durationInSeconds ) : info( _info ), prevAssertions( _prevAssertions ), durationInSeconds( _durationInSeconds ) @@ -5217,18 +5478,19 @@ namespace Catch { struct Version { Version( unsigned int _majorVersion, unsigned int _minorVersion, - unsigned int _buildNumber, - char const* const _branchName ) - : majorVersion( _majorVersion ), - minorVersion( _minorVersion ), - buildNumber( _buildNumber ), - branchName( _branchName ) - {} + unsigned int _patchNumber, + std::string const& _branchName, + unsigned int _buildNumber ); unsigned int const majorVersion; unsigned int const minorVersion; + unsigned int const patchNumber; + + // buildNumber is only used if branchName is not null + std::string const branchName; unsigned int const buildNumber; - char const* const branchName; + + friend std::ostream& operator << ( std::ostream& os, Version const& version ); private: void operator=( Version const& ); @@ -5259,7 +5521,7 @@ namespace Catch { Totals totals; - context.testGroupStarting( "", 1, 1 ); // deprecated? + context.testGroupStarting( "all tests", 1, 1 ); // deprecated? TestSpec testSpec = m_config->testSpec(); if( !testSpec.hasFilters() ) @@ -5282,7 +5544,15 @@ namespace Catch { m_testsAlreadyRun.insert( *it ); } } - context.testGroupEnded( "", totals, 1, 1 ); + std::vector skippedTestCases; + getRegistryHub().getTestCaseRegistry().getFilteredTests( testSpec, *m_config, skippedTestCases, true ); + + for( std::vector::const_iterator it = skippedTestCases.begin(), itEnd = skippedTestCases.end(); + it != itEnd; + ++it ) + m_reporter->skipTest( *it ); + + context.testGroupEnded( "all tests", totals, 1, 1 ); return totals; } @@ -5319,7 +5589,7 @@ namespace Catch { std::set m_testsAlreadyRun; }; - class Session { + class Session : NonCopyable { static bool alreadyInstantiated; public: @@ -5330,7 +5600,7 @@ namespace Catch { : m_cli( makeCommandLineParser() ) { if( alreadyInstantiated ) { std::string msg = "Only one instance of Catch::Session can ever be used"; - std::cerr << msg << std::endl; + Catch::cerr() << msg << std::endl; throw std::logic_error( msg ); } alreadyInstantiated = true; @@ -5340,15 +5610,10 @@ namespace Catch { } void showHelp( std::string const& processName ) { - std::cout << "\nCatch v" << libraryVersion.majorVersion << "." - << libraryVersion.minorVersion << " build " - << libraryVersion.buildNumber; - if( libraryVersion.branchName != std::string( "master" ) ) - std::cout << " (" << libraryVersion.branchName << " branch)"; - std::cout << "\n"; + Catch::cout() << "\nCatch v" << libraryVersion << "\n"; - m_cli.usage( std::cout, processName ); - std::cout << "For more detail usage please see the project docs\n" << std::endl; + m_cli.usage( Catch::cout(), processName ); + Catch::cout() << "For more detail usage please see the project docs\n" << std::endl; } int applyCommandLine( int argc, char* const argv[], OnUnusedOptions::DoWhat unusedOptionBehaviour = OnUnusedOptions::Fail ) { @@ -5362,11 +5627,12 @@ namespace Catch { catch( std::exception& ex ) { { Colour colourGuard( Colour::Red ); - std::cerr << "\nError(s) in input:\n" - << Text( ex.what(), TextAttributes().setIndent(2) ) - << "\n\n"; + Catch::cerr() + << "\nError(s) in input:\n" + << Text( ex.what(), TextAttributes().setIndent(2) ) + << "\n\n"; } - m_cli.usage( std::cout, m_configData.processName ); + m_cli.usage( Catch::cout(), m_configData.processName ); return (std::numeric_limits::max)(); } return 0; @@ -5392,6 +5658,9 @@ namespace Catch { try { config(); // Force config to be constructed + + std::srand( m_configData.rngSeed ); + Runner runner( m_config ); // Handle list request @@ -5401,7 +5670,7 @@ namespace Catch { return static_cast( runner.runTests().assertions.failed ); } catch( std::exception& ex ) { - std::cerr << ex.what() << std::endl; + Catch::cerr() << ex.what() << std::endl; return (std::numeric_limits::max)(); } } @@ -5442,10 +5711,18 @@ namespace Catch { #include #include #include +#include namespace Catch { class TestRegistry : public ITestCaseRegistry { + struct LexSort { + bool operator() (TestCase i,TestCase j) const { return (i& matchingTestCases ) const { + virtual void getFilteredTests( TestSpec const& testSpec, IConfig const& config, std::vector& matchingTestCases, bool negated = false ) const { + for( std::vector::const_iterator it = m_functionsInOrder.begin(), itEnd = m_functionsInOrder.end(); it != itEnd; ++it ) { - if( testSpec.matches( *it ) && ( config.allowThrows() || !it->throws() ) ) + bool includeTest = testSpec.matches( *it ) && ( config.allowThrows() || !it->throws() ); + if( includeTest != negated ) matchingTestCases.push_back( *it ); } + sortTests( config, matchingTestCases ); } private: + static void sortTests( IConfig const& config, std::vector& matchingTestCases ) { + + switch( config.runOrder() ) { + case RunTests::InLexicographicalOrder: + std::sort( matchingTestCases.begin(), matchingTestCases.end(), LexSort() ); + break; + case RunTests::InRandomOrder: + { + RandomNumberGenerator rng; + std::random_shuffle( matchingTestCases.begin(), matchingTestCases.end(), rng ); + } + break; + case RunTests::InDeclarationOrder: + // already in declaration order + break; + } + } std::set m_functions; std::vector m_functionsInOrder; std::vector m_nonHiddenFunctions; @@ -5619,7 +5916,7 @@ namespace Catch { throw; } @catch (NSException *exception) { - return toString( [exception description] ); + return Catch::toString( [exception description] ); } #else throw; @@ -5766,6 +6063,7 @@ namespace Catch { #include #include +#include namespace Catch { @@ -5829,6 +6127,15 @@ namespace Catch { isOwned = false; } } + +#ifndef CATCH_CONFIG_NOSTDOUT // If you #define this you must implement this functions + std::ostream& cout() { + return std::cout; + } + std::ostream& cerr() { + return std::cerr; + } +#endif } namespace Catch { @@ -5878,7 +6185,7 @@ namespace Catch { std::string testName = getResultCapture()->getCurrentTestName(); std::map::const_iterator it = - m_generatorsByTestName.find( testName ); + m_generatorsByTestName.find( testName ); return it != m_generatorsByTestName.end() ? it->second : NULL; @@ -5914,8 +6221,8 @@ namespace Catch { } Stream createStream( std::string const& streamName ) { - if( streamName == "stdout" ) return Stream( std::cout.rdbuf(), false ); - if( streamName == "stderr" ) return Stream( std::cerr.rdbuf(), false ); + if( streamName == "stdout" ) return Stream( Catch::cout().rdbuf(), false ); + if( streamName == "stderr" ) return Stream( Catch::cerr().rdbuf(), false ); if( streamName == "debug" ) return Stream( new StreamBufImpl, true ); throw std::domain_error( "Unknown stream: " + streamName ); @@ -5930,14 +6237,35 @@ namespace Catch { // #included from: catch_console_colour_impl.hpp #define TWOBLUECUBES_CATCH_CONSOLE_COLOUR_IMPL_HPP_INCLUDED -namespace Catch { namespace Detail { - struct IColourImpl { - virtual ~IColourImpl() {} - virtual void use( Colour::Code _colourCode ) = 0; - }; -}} +namespace Catch { + namespace { -#if defined ( CATCH_PLATFORM_WINDOWS ) ///////////////////////////////////////// + struct IColourImpl { + virtual ~IColourImpl() {} + virtual void use( Colour::Code _colourCode ) = 0; + }; + + struct NoColourImpl : IColourImpl { + void use( Colour::Code ) {} + + static IColourImpl* instance() { + static NoColourImpl s_instance; + return &s_instance; + } + }; + + } // anon namespace +} // namespace Catch + +#if !defined( CATCH_CONFIG_COLOUR_NONE ) && !defined( CATCH_CONFIG_COLOUR_WINDOWS ) && !defined( CATCH_CONFIG_COLOUR_ANSI ) +# ifdef CATCH_PLATFORM_WINDOWS +# define CATCH_CONFIG_COLOUR_WINDOWS +# else +# define CATCH_CONFIG_COLOUR_ANSI +# endif +#endif + +#if defined ( CATCH_CONFIG_COLOUR_WINDOWS ) ///////////////////////////////////////// #ifndef NOMINMAX #define NOMINMAX @@ -5952,7 +6280,7 @@ namespace Catch { namespace Detail { namespace Catch { namespace { - class Win32ColourImpl : public Detail::IColourImpl { + class Win32ColourImpl : public IColourImpl { public: Win32ColourImpl() : stdoutHandle( GetStdHandle(STD_OUTPUT_HANDLE) ) { @@ -5989,11 +6317,7 @@ namespace { WORD originalAttributes; }; - inline bool shouldUseColourForPlatform() { - return true; - } - - static Detail::IColourImpl* platformColourInstance() { + IColourImpl* platformColourInstance() { static Win32ColourImpl s_instance; return &s_instance; } @@ -6001,7 +6325,7 @@ namespace { } // end anon namespace } // end namespace Catch -#else // Not Windows - assumed to be POSIX compatible ////////////////////////// +#elif defined( CATCH_CONFIG_COLOUR_ANSI ) ////////////////////////////////////// #include @@ -6012,7 +6336,7 @@ namespace { // Thanks to Adam Strzelecki for original contribution // (http://github.com/nanoant) // https://github.com/philsquared/Catch/pull/131 - class PosixColourImpl : public Detail::IColourImpl { + class PosixColourImpl : public IColourImpl { public: virtual void use( Colour::Code _colourCode ) { switch( _colourCode ) { @@ -6033,53 +6357,48 @@ namespace { case Colour::Bright: throw std::logic_error( "not a colour" ); } } + static IColourImpl* instance() { + static PosixColourImpl s_instance; + return &s_instance; + } + private: void setColour( const char* _escapeCode ) { - std::cout << '\033' << _escapeCode; + Catch::cout() << '\033' << _escapeCode; } }; - inline bool shouldUseColourForPlatform() { - return isatty(STDOUT_FILENO); - } - - static Detail::IColourImpl* platformColourInstance() { - static PosixColourImpl s_instance; - return &s_instance; + IColourImpl* platformColourInstance() { + Ptr config = getCurrentContext().getConfig(); + return (config && config->forceColour()) || isatty(STDOUT_FILENO) + ? PosixColourImpl::instance() + : NoColourImpl::instance(); } } // end anon namespace } // end namespace Catch -#endif // not Windows +#else // not Windows or ANSI /////////////////////////////////////////////// namespace Catch { - namespace { - struct NoColourImpl : Detail::IColourImpl { - void use( Colour::Code ) {} + static IColourImpl* platformColourInstance() { return NoColourImpl::instance(); } - static IColourImpl* instance() { - static NoColourImpl s_instance; - return &s_instance; - } - }; - static bool shouldUseColour() { - return shouldUseColourForPlatform() && !isDebuggerActive(); - } - } +} // end namespace Catch + +#endif // Windows/ ANSI/ None + +namespace Catch { Colour::Colour( Code _colourCode ) : m_moved( false ) { use( _colourCode ); } Colour::Colour( Colour const& _other ) : m_moved( false ) { const_cast( _other ).m_moved = true; } Colour::~Colour(){ if( !m_moved ) use( None ); } - void Colour::use( Code _colourCode ) { - impl()->use( _colourCode ); - } - Detail::IColourImpl* Colour::impl() { - return shouldUseColour() - ? platformColourInstance() - : NoColourImpl::instance(); + void Colour::use( Code _colourCode ) { + static IColourImpl* impl = isDebuggerActive() + ? NoColourImpl::instance() + : platformColourInstance(); + impl->use( _colourCode ); } } // end namespace Catch @@ -6244,7 +6563,7 @@ namespace Catch { namespace Catch { inline TestCaseInfo::SpecialProperties parseSpecialTag( std::string const& tag ) { - if( tag == "." || + if( startsWith( tag, "." ) || tag == "hide" || tag == "!hide" ) return TestCaseInfo::IsHidden; @@ -6264,13 +6583,13 @@ namespace Catch { if( isReservedTag( tag ) ) { { Colour colourGuard( Colour::Red ); - std::cerr + Catch::cerr() << "Tag name [" << tag << "] not allowed.\n" << "Tag names starting with non alpha-numeric characters are reserved\n"; } { Colour colourGuard( Colour::FileName ); - std::cerr << _lineInfo << std::endl; + Catch::cerr() << _lineInfo << std::endl; } exit(1); } @@ -6298,14 +6617,15 @@ namespace Catch { } else { if( c == ']' ) { - enforceNotReservedTag( tag, _lineInfo ); - - inTag = false; - if( tag == "hide" || tag == "." ) + TestCaseInfo::SpecialProperties prop = parseSpecialTag( tag ); + if( prop == TestCaseInfo::IsHidden ) isHidden = true; - else - tags.insert( tag ); + else if( prop == TestCaseInfo::None ) + enforceNotReservedTag( tag, _lineInfo ); + + tags.insert( tag ); tag.clear(); + inTag = false; } else tag += c; @@ -6422,8 +6742,33 @@ namespace Catch { namespace Catch { - // These numbers are maintained by a script - Version libraryVersion( 1, 0, 53, "master" ); + Version::Version + ( unsigned int _majorVersion, + unsigned int _minorVersion, + unsigned int _patchNumber, + std::string const& _branchName, + unsigned int _buildNumber ) + : majorVersion( _majorVersion ), + minorVersion( _minorVersion ), + patchNumber( _patchNumber ), + branchName( _branchName ), + buildNumber( _buildNumber ) + {} + + std::ostream& operator << ( std::ostream& os, Version const& version ) { + os << version.majorVersion << "." + << version.minorVersion << "." + << version.patchNumber; + + if( !version.branchName.empty() ) { + os << "-" << version.branchName + << "." << version.buildNumber; + } + return os; + } + + Version libraryVersion( 1, 2, 1, "", 0 ); + } // #included from: catch_message.hpp @@ -6507,6 +6852,7 @@ namespace Catch virtual void testCaseEnded( TestCaseStats const& testCaseStats ); virtual void testGroupEnded( TestGroupStats const& testGroupStats ); virtual void testRunEnded( TestRunStats const& testRunStats ); + virtual void skipTest( TestCaseInfo const& ); private: Ptr m_legacyReporter; @@ -6580,6 +6926,8 @@ namespace Catch void LegacyReporterAdapter::testRunEnded( TestRunStats const& testRunStats ) { m_legacyReporter->EndTesting( testRunStats.totals ); } + void LegacyReporterAdapter::skipTest( TestCaseInfo const& ) { + } } // #included from: catch_timer.hpp @@ -6602,11 +6950,11 @@ namespace Catch { uint64_t getCurrentTicks() { static uint64_t hz=0, hzo=0; if (!hz) { - QueryPerformanceFrequency((LARGE_INTEGER*)&hz); - QueryPerformanceCounter((LARGE_INTEGER*)&hzo); + QueryPerformanceFrequency( reinterpret_cast( &hz ) ); + QueryPerformanceCounter( reinterpret_cast( &hzo ) ); } uint64_t t; - QueryPerformanceCounter((LARGE_INTEGER*)&t); + QueryPerformanceCounter( reinterpret_cast( &t ) ); return ((t-hzo)*1000000)/hz; } #else @@ -6621,14 +6969,14 @@ namespace Catch { void Timer::start() { m_ticks = getCurrentTicks(); } - unsigned int Timer::getElapsedNanoseconds() const { + unsigned int Timer::getElapsedMicroseconds() const { return static_cast(getCurrentTicks() - m_ticks); } unsigned int Timer::getElapsedMilliseconds() const { - return static_cast((getCurrentTicks() - m_ticks)/1000); + return static_cast(getElapsedMicroseconds()/1000); } double Timer::getElapsedSeconds() const { - return (getCurrentTicks() - m_ticks)/1000000.0; + return getElapsedMicroseconds()/1000000.0; } } // namespace Catch @@ -6666,6 +7014,20 @@ namespace Catch { return start != std::string::npos ? str.substr( start, 1+end-start ) : ""; } + bool replaceInPlace( std::string& str, std::string const& replaceThis, std::string const& withThis ) { + bool replaced = false; + std::size_t i = str.find( replaceThis ); + while( i != std::string::npos ) { + replaced = true; + str = str.substr( 0, i ) + withThis + str.substr( i+replaceThis.size() ); + if( i < str.size()-withThis.size() ) + i = str.find( replaceThis, i+withThis.size() ); + else + i = std::string::npos; + } + return replaced; + } + pluralise::pluralise( std::size_t count, std::string const& label ) : m_count( count ), m_label( label ) @@ -6693,6 +7055,9 @@ namespace Catch { bool SourceLineInfo::operator == ( SourceLineInfo const& other ) const { return line == other.line && file == other.file; } + bool SourceLineInfo::operator < ( SourceLineInfo const& other ) const { + return line < other.line || ( line == other.line && file < other.file ); + } std::ostream& operator << ( std::ostream& os, SourceLineInfo const& info ) { #ifndef __GNUG__ @@ -6787,7 +7152,7 @@ namespace Catch { size = sizeof(info); if( sysctl(mib, sizeof(mib) / sizeof(*mib), &info, &size, NULL, 0) != 0 ) { - std::cerr << "\n** Call to sysctl failed - unable to determine if debugger is active **\n" << std::endl; + Catch::cerr() << "\n** Call to sysctl failed - unable to determine if debugger is active **\n" << std::endl; return false; } @@ -6828,7 +7193,7 @@ namespace Catch { namespace Catch { void writeToDebugConsole( std::string const& text ) { // !TBD: Need a version for Mac/ XCode and other IDEs - std::cout << text; + Catch::cout() << text; } } #endif // Platform @@ -6840,6 +7205,8 @@ namespace Catch { namespace Detail { + std::string unprintableString = "{?}"; + namespace { struct Endianness { enum Arch { Big, Little }; @@ -6898,7 +7265,7 @@ std::string toString( std::wstring const& value ) { s.reserve( value.size() ); for(size_t i = 0; i < value.size(); ++i ) s += value[i] <= 0xff ? static_cast( value[i] ) : '?'; - return toString( s ); + return Catch::toString( s ); } std::string toString( const char* const value ) { @@ -6922,20 +7289,21 @@ std::string toString( wchar_t* const value ) std::string toString( int value ) { std::ostringstream oss; oss << value; + if( value >= 255 ) + oss << " (0x" << std::hex << value << ")"; return oss.str(); } std::string toString( unsigned long value ) { std::ostringstream oss; - if( value > 8192 ) - oss << "0x" << std::hex << value; - else - oss << value; + oss << value; + if( value >= 255 ) + oss << " (0x" << std::hex << value << ")"; return oss.str(); } std::string toString( unsigned int value ) { - return toString( static_cast( value ) ); + return Catch::toString( static_cast( value ) ); } template @@ -7061,7 +7429,7 @@ namespace Catch { if( !result.isOk() ) { if( getCurrentContext().getConfig()->shouldDebugBreak() ) m_shouldDebugBreak = true; - if( getCurrentContext().getRunner()->aborting() || m_assertionInfo.resultDisposition == ResultDisposition::Normal ) + if( getCurrentContext().getRunner()->aborting() || (m_assertionInfo.resultDisposition & ResultDisposition::Normal) ) m_shouldThrow = true; } } @@ -7201,7 +7569,7 @@ namespace Catch { } catch( std::exception& ex ) { Colour colourGuard( Colour::Red ); - std::cerr << ex.what() << std::endl; + Catch::cerr() << ex.what() << std::endl; exit(1); } } @@ -7214,6 +7582,8 @@ namespace Catch { // #included from: catch_reporter_bases.hpp #define TWOBLUECUBES_CATCH_REPORTER_BASES_HPP_INCLUDED +#include + namespace Catch { struct StreamingReporterBase : SharedImpl { @@ -7246,7 +7616,6 @@ namespace Catch { } virtual void testCaseEnded( TestCaseStats const& /* _testCaseStats */ ) { currentTestCaseInfo.reset(); - assert( m_sectionStack.empty() ); } virtual void testGroupEnded( TestGroupStats const& /* _testGroupStats */ ) { currentGroupInfo.reset(); @@ -7257,6 +7626,11 @@ namespace Catch { currentTestRunInfo.reset(); } + virtual void skipTest( TestCaseInfo const& ) { + // Don't do anything with this by default. + // It can optionally be overridden in the derived class. + } + Ptr m_config; std::ostream& stream; @@ -7386,6 +7760,8 @@ namespace Catch { } virtual void testRunEndedCumulative() = 0; + virtual void skipTest( TestCaseInfo const& ) {} + Ptr m_config; std::ostream& stream; std::vector m_assertions; @@ -7401,6 +7777,16 @@ namespace Catch { }; + template + char const* getLineOfChars() { + static char line[CATCH_CONFIG_CONSOLE_WIDTH] = {0}; + if( !*line ) { + memset( line, C, CATCH_CONFIG_CONSOLE_WIDTH-1 ); + line[CATCH_CONFIG_CONSOLE_WIDTH-1] = 0; + } + return line; + } + } // end namespace Catch // #included from: ../internal/catch_reporter_registrars.hpp @@ -7470,7 +7856,6 @@ namespace Catch { #define TWOBLUECUBES_CATCH_XMLWRITER_HPP_INCLUDED #include -#include #include #include @@ -7513,7 +7898,7 @@ namespace Catch { XmlWriter() : m_tagIsOpen( false ), m_needsNewline( false ), - m_os( &std::cout ) + m_os( &Catch::cout() ) {} XmlWriter( std::ostream& os ) @@ -7527,27 +7912,6 @@ namespace Catch { endElement(); } -//# ifndef CATCH_CPP11_OR_GREATER -// XmlWriter& operator = ( XmlWriter const& other ) { -// XmlWriter temp( other ); -// swap( temp ); -// return *this; -// } -//# else -// XmlWriter( XmlWriter const& ) = default; -// XmlWriter( XmlWriter && ) = default; -// XmlWriter& operator = ( XmlWriter const& ) = default; -// XmlWriter& operator = ( XmlWriter && ) = default; -//# endif -// -// void swap( XmlWriter& other ) { -// std::swap( m_tagIsOpen, other.m_tagIsOpen ); -// std::swap( m_needsNewline, other.m_needsNewline ); -// std::swap( m_tags, other.m_tags ); -// std::swap( m_indent, other.m_indent ); -// std::swap( m_os, other.m_os ); -// } - XmlWriter& startElement( std::string const& name ) { ensureTagClosed(); newlineIfNecessary(); @@ -7683,81 +8047,90 @@ namespace Catch { } namespace Catch { - class XmlReporter : public SharedImpl { + class XmlReporter : public StreamingReporterBase { public: - XmlReporter( ReporterConfig const& config ) : m_config( config ), m_sectionDepth( 0 ) {} + XmlReporter( ReporterConfig const& _config ) + : StreamingReporterBase( _config ), + m_sectionDepth( 0 ) + {} + + virtual ~XmlReporter(); static std::string getDescription() { return "Reports test results as an XML document"; } - virtual ~XmlReporter(); - private: // IReporter - - virtual bool shouldRedirectStdout() const { - return true; + public: // StreamingReporterBase + virtual ReporterPreferences getPreferences() const { + ReporterPreferences prefs; + prefs.shouldRedirectStdOut = true; + return prefs; } - virtual void StartTesting() { - m_xml.setStream( m_config.stream() ); - m_xml.startElement( "Catch" ); - if( !m_config.fullConfig()->name().empty() ) - m_xml.writeAttribute( "name", m_config.fullConfig()->name() ); + virtual void noMatchingTestCases( std::string const& s ) { + StreamingReporterBase::noMatchingTestCases( s ); } - virtual void EndTesting( const Totals& totals ) { - m_xml.scopedElement( "OverallResults" ) - .writeAttribute( "successes", totals.assertions.passed ) - .writeAttribute( "failures", totals.assertions.failed ) - .writeAttribute( "expectedFailures", totals.assertions.failedButOk ); - m_xml.endElement(); + virtual void testRunStarting( TestRunInfo const& testInfo ) { + StreamingReporterBase::testRunStarting( testInfo ); + m_xml.setStream( stream ); + m_xml.startElement( "Catch" ); + if( !m_config->name().empty() ) + m_xml.writeAttribute( "name", m_config->name() ); } - virtual void StartGroup( const std::string& groupName ) { + virtual void testGroupStarting( GroupInfo const& groupInfo ) { + StreamingReporterBase::testGroupStarting( groupInfo ); m_xml.startElement( "Group" ) - .writeAttribute( "name", groupName ); + .writeAttribute( "name", groupInfo.name ); } - virtual void EndGroup( const std::string&, const Totals& totals ) { - m_xml.scopedElement( "OverallResults" ) - .writeAttribute( "successes", totals.assertions.passed ) - .writeAttribute( "failures", totals.assertions.failed ) - .writeAttribute( "expectedFailures", totals.assertions.failedButOk ); - m_xml.endElement(); + virtual void testCaseStarting( TestCaseInfo const& testInfo ) { + StreamingReporterBase::testCaseStarting(testInfo); + m_xml.startElement( "TestCase" ).writeAttribute( "name", trim( testInfo.name ) ); + + if ( m_config->showDurations() == ShowDurations::Always ) + m_testCaseTimer.start(); } - virtual void StartSection( const std::string& sectionName, const std::string& description ) { + virtual void sectionStarting( SectionInfo const& sectionInfo ) { + StreamingReporterBase::sectionStarting( sectionInfo ); if( m_sectionDepth++ > 0 ) { m_xml.startElement( "Section" ) - .writeAttribute( "name", trim( sectionName ) ) - .writeAttribute( "description", description ); + .writeAttribute( "name", trim( sectionInfo.name ) ) + .writeAttribute( "description", sectionInfo.description ); } } - virtual void NoAssertionsInSection( const std::string& ) {} - virtual void NoAssertionsInTestCase( const std::string& ) {} - virtual void EndSection( const std::string& /*sectionName*/, const Counts& assertions ) { - if( --m_sectionDepth > 0 ) { - m_xml.scopedElement( "OverallResults" ) - .writeAttribute( "successes", assertions.passed ) - .writeAttribute( "failures", assertions.failed ) - .writeAttribute( "expectedFailures", assertions.failedButOk ); - m_xml.endElement(); - } - } + virtual void assertionStarting( AssertionInfo const& ) { } - virtual void StartTestCase( const Catch::TestCaseInfo& testInfo ) { - m_xml.startElement( "TestCase" ).writeAttribute( "name", trim( testInfo.name ) ); - m_currentTestSuccess = true; - } + virtual bool assertionEnded( AssertionStats const& assertionStats ) { + const AssertionResult& assertionResult = assertionStats.assertionResult; - virtual void Result( const Catch::AssertionResult& assertionResult ) { - if( !m_config.fullConfig()->includeSuccessfulResults() && assertionResult.getResultType() == ResultWas::Ok ) - return; + // Print any info messages in tags. + if( assertionStats.assertionResult.getResultType() != ResultWas::Ok ) { + for( std::vector::const_iterator it = assertionStats.infoMessages.begin(), itEnd = assertionStats.infoMessages.end(); + it != itEnd; + ++it ) { + if( it->type == ResultWas::Info ) { + m_xml.scopedElement( "Info" ) + .writeText( it->message ); + } else if ( it->type == ResultWas::Warning ) { + m_xml.scopedElement( "Warning" ) + .writeText( it->message ); + } + } + } + // Drop out if result was successful but we're not printing them. + if( !m_config->includeSuccessfulResults() && isOk(assertionResult.getResultType()) ) + return true; + + // Print the expression if there is one. if( assertionResult.hasExpression() ) { m_xml.startElement( "Expression" ) .writeAttribute( "success", assertionResult.succeeded() ) + .writeAttribute( "type", assertionResult.getTestMacroName() ) .writeAttribute( "filename", assertionResult.getSourceInfo().file ) .writeAttribute( "line", assertionResult.getSourceInfo().line ); @@ -7765,58 +8138,96 @@ namespace Catch { .writeText( assertionResult.getExpression() ); m_xml.scopedElement( "Expanded" ) .writeText( assertionResult.getExpandedExpression() ); - m_currentTestSuccess &= assertionResult.succeeded(); } + // And... Print a result applicable to each result type. switch( assertionResult.getResultType() ) { case ResultWas::ThrewException: m_xml.scopedElement( "Exception" ) .writeAttribute( "filename", assertionResult.getSourceInfo().file ) .writeAttribute( "line", assertionResult.getSourceInfo().line ) .writeText( assertionResult.getMessage() ); - m_currentTestSuccess = false; + break; + case ResultWas::FatalErrorCondition: + m_xml.scopedElement( "Fatal Error Condition" ) + .writeAttribute( "filename", assertionResult.getSourceInfo().file ) + .writeAttribute( "line", assertionResult.getSourceInfo().line ) + .writeText( assertionResult.getMessage() ); break; case ResultWas::Info: m_xml.scopedElement( "Info" ) .writeText( assertionResult.getMessage() ); break; case ResultWas::Warning: - m_xml.scopedElement( "Warning" ) - .writeText( assertionResult.getMessage() ); + // Warning will already have been written break; case ResultWas::ExplicitFailure: m_xml.scopedElement( "Failure" ) .writeText( assertionResult.getMessage() ); - m_currentTestSuccess = false; break; - case ResultWas::Unknown: - case ResultWas::Ok: - case ResultWas::FailureBit: - case ResultWas::ExpressionFailed: - case ResultWas::Exception: - case ResultWas::DidntThrowException: + default: break; } + if( assertionResult.hasExpression() ) m_xml.endElement(); + + return true; } - virtual void Aborted() { - // !TBD + virtual void sectionEnded( SectionStats const& sectionStats ) { + StreamingReporterBase::sectionEnded( sectionStats ); + if( --m_sectionDepth > 0 ) { + XmlWriter::ScopedElement e = m_xml.scopedElement( "OverallResults" ); + e.writeAttribute( "successes", sectionStats.assertions.passed ); + e.writeAttribute( "failures", sectionStats.assertions.failed ); + e.writeAttribute( "expectedFailures", sectionStats.assertions.failedButOk ); + + if ( m_config->showDurations() == ShowDurations::Always ) + e.writeAttribute( "durationInSeconds", sectionStats.durationInSeconds ); + + m_xml.endElement(); + } } - virtual void EndTestCase( const Catch::TestCaseInfo&, const Totals&, const std::string&, const std::string& ) { - m_xml.scopedElement( "OverallResult" ).writeAttribute( "success", m_currentTestSuccess ); + virtual void testCaseEnded( TestCaseStats const& testCaseStats ) { + StreamingReporterBase::testCaseEnded( testCaseStats ); + XmlWriter::ScopedElement e = m_xml.scopedElement( "OverallResult" ); + e.writeAttribute( "success", testCaseStats.totals.assertions.allOk() ); + + if ( m_config->showDurations() == ShowDurations::Always ) + e.writeAttribute( "durationInSeconds", m_testCaseTimer.getElapsedSeconds() ); + + m_xml.endElement(); + } + + virtual void testGroupEnded( TestGroupStats const& testGroupStats ) { + StreamingReporterBase::testGroupEnded( testGroupStats ); + // TODO: Check testGroupStats.aborting and act accordingly. + m_xml.scopedElement( "OverallResults" ) + .writeAttribute( "successes", testGroupStats.totals.assertions.passed ) + .writeAttribute( "failures", testGroupStats.totals.assertions.failed ) + .writeAttribute( "expectedFailures", testGroupStats.totals.assertions.failedButOk ); + m_xml.endElement(); + } + + virtual void testRunEnded( TestRunStats const& testRunStats ) { + StreamingReporterBase::testRunEnded( testRunStats ); + m_xml.scopedElement( "OverallResults" ) + .writeAttribute( "successes", testRunStats.totals.assertions.passed ) + .writeAttribute( "failures", testRunStats.totals.assertions.failed ) + .writeAttribute( "expectedFailures", testRunStats.totals.assertions.failedButOk ); m_xml.endElement(); } private: - ReporterConfig m_config; - bool m_currentTestSuccess; + Timer m_testCaseTimer; XmlWriter m_xml; int m_sectionDepth; }; + INTERNAL_CATCH_REGISTER_REPORTER( "xml", XmlReporter ) + } // end namespace Catch // #included from: ../reporters/catch_reporter_junit.hpp @@ -7943,7 +8354,7 @@ namespace Catch { xml.writeAttribute( "classname", className ); xml.writeAttribute( "name", name ); } - xml.writeAttribute( "time", toString( sectionNode.stats.durationInSeconds ) ); + xml.writeAttribute( "time", Catch::toString( sectionNode.stats.durationInSeconds ) ); writeAssertions( sectionNode ); @@ -7976,6 +8387,7 @@ namespace Catch { std::string elementName; switch( result.getResultType() ) { case ResultWas::ThrewException: + case ResultWas::FatalErrorCondition: elementName = "error"; break; case ResultWas::ExplicitFailure: @@ -8034,8 +8446,6 @@ namespace Catch { // #included from: ../reporters/catch_reporter_console.hpp #define TWOBLUECUBES_CATCH_REPORTER_CONSOLE_HPP_INCLUDED -#include - namespace Catch { struct ConsoleReporter : StreamingReporterBase { @@ -8170,6 +8580,11 @@ namespace Catch { passOrFail = "FAILED"; messageLabel = "due to unexpected exception with message"; break; + case ResultWas::FatalErrorCondition: + colour = Colour::Error; + passOrFail = "FAILED"; + messageLabel = "due to a fatal error condition"; + break; case ResultWas::DidntThrowException: colour = Colour::Error; passOrFail = "FAILED"; @@ -8279,14 +8694,12 @@ namespace Catch { stream << "\n" << getLineOfChars<'~'>() << "\n"; Colour colour( Colour::SecondaryText ); stream << currentTestRunInfo->name - << " is a Catch v" << libraryVersion.majorVersion << "." - << libraryVersion.minorVersion << " b" - << libraryVersion.buildNumber; - if( libraryVersion.branchName != std::string( "master" ) ) - stream << " (" << libraryVersion.branchName << ")"; - stream << " host application.\n" + << " is a Catch v" << libraryVersion << " host application.\n" << "Run with -? for options\n\n"; + if( m_config->rngSeed() != 0 ) + stream << "Randomness seeded to: " << m_config->rngSeed() << "\n\n"; + currentTestRunInfo.used = true; } void lazyPrintGroupInfo() { @@ -8458,15 +8871,6 @@ namespace Catch { void printSummaryDivider() { stream << getLineOfChars<'-'>() << "\n"; } - template - static char const* getLineOfChars() { - static char line[CATCH_CONFIG_CONSOLE_WIDTH] = {0}; - if( !*line ) { - memset( line, C, CATCH_CONFIG_CONSOLE_WIDTH-1 ); - line[CATCH_CONFIG_CONSOLE_WIDTH-1] = 0; - } - return line; - } private: bool m_headerPrinted; @@ -8575,6 +8979,13 @@ namespace Catch { printExpressionWas(); printRemainingMessages(); break; + case ResultWas::FatalErrorCondition: + printResultType( Colour::Error, failedString() ); + printIssue( "fatal error condition with message:" ); + printMessage(); + printExpressionWas(); + printRemainingMessages(); + break; case ResultWas::DidntThrowException: printResultType( Colour::Error, failedString() ); printIssue( "expected exception, got none" ); @@ -8804,8 +9215,6 @@ namespace Catch { Matchers::Impl::StdString::EndsWith::~EndsWith() {} void Config::dummy() {} - - INTERNAL_CATCH_REGISTER_LEGACY_REPORTER( "xml", XmlReporter ) } #ifdef __clang__ @@ -8994,9 +9403,13 @@ using Catch::Detail::Approx; #define TWOBLUECUBES_CATCH_REENABLE_WARNINGS_H_INCLUDED #ifdef __clang__ -#pragma clang diagnostic pop +# ifdef __ICC // icpc defines the __clang__ macro +# pragma warning(pop) +# else +# pragma clang diagnostic pop +# endif #elif defined __GNUC__ -#pragma GCC diagnostic pop +# pragma GCC diagnostic pop #endif #endif // TWOBLUECUBES_SINGLE_INCLUDE_CATCH_HPP_INCLUDED diff --git a/test/include/catch_orig.hpp b/test/include/catch_orig.hpp index 6b8dfb5ebd0..de61226cf68 100644 --- a/test/include/catch_orig.hpp +++ b/test/include/catch_orig.hpp @@ -1,6 +1,6 @@ /* - * CATCH v1.0 build 53 (master branch) - * Generated: 2014-08-20 08:08:19.533804 + * Catch v1.2.1 + * Generated: 2015-06-30 18:23:27.961086 * ---------------------------------------------------------- * This file has been merged from multiple headers. Please don't edit it directly * Copyright (c) 2012 Two Blue Cubes Ltd. All rights reserved. @@ -13,31 +13,43 @@ #define TWOBLUECUBES_CATCH_HPP_INCLUDED +#ifdef __clang__ +# pragma clang system_header +#elif defined __GNUC__ +# pragma GCC system_header +#endif + // #included from: internal/catch_suppress_warnings.h #define TWOBLUECUBES_CATCH_SUPPRESS_WARNINGS_H_INCLUDED #ifdef __clang__ -#pragma clang diagnostic ignored "-Wglobal-constructors" -#pragma clang diagnostic ignored "-Wvariadic-macros" -#pragma clang diagnostic ignored "-Wc99-extensions" -#pragma clang diagnostic ignored "-Wunused-variable" -#pragma clang diagnostic push -#pragma clang diagnostic ignored "-Wpadded" -#pragma clang diagnostic ignored "-Wc++98-compat" -#pragma clang diagnostic ignored "-Wc++98-compat-pedantic" +# ifdef __ICC // icpc defines the __clang__ macro +# pragma warning(push) +# pragma warning(disable: 161 1682) +# else // __ICC +# pragma clang diagnostic ignored "-Wglobal-constructors" +# pragma clang diagnostic ignored "-Wvariadic-macros" +# pragma clang diagnostic ignored "-Wc99-extensions" +# pragma clang diagnostic ignored "-Wunused-variable" +# pragma clang diagnostic push +# pragma clang diagnostic ignored "-Wpadded" +# pragma clang diagnostic ignored "-Wc++98-compat" +# pragma clang diagnostic ignored "-Wc++98-compat-pedantic" +# pragma clang diagnostic ignored "-Wswitch-enum" +# endif #elif defined __GNUC__ -#pragma GCC diagnostic ignored "-Wvariadic-macros" -#pragma GCC diagnostic ignored "-Wunused-variable" -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wpadded" +# pragma GCC diagnostic ignored "-Wvariadic-macros" +# pragma GCC diagnostic ignored "-Wunused-variable" +# pragma GCC diagnostic push +# pragma GCC diagnostic ignored "-Wpadded" #endif -#ifdef CATCH_CONFIG_MAIN -# define CATCH_CONFIG_RUNNER +#if defined(CATCH_CONFIG_MAIN) || defined(CATCH_CONFIG_RUNNER) +# define CATCH_IMPL #endif -#ifdef CATCH_CONFIG_RUNNER +#ifdef CATCH_IMPL # ifndef CLARA_CONFIG_MAIN # define CLARA_CONFIG_MAIN_NOT_DEFINED # define CLARA_CONFIG_MAIN @@ -64,16 +76,34 @@ // #included from: catch_compiler_capabilities.h #define TWOBLUECUBES_CATCH_COMPILER_CAPABILITIES_HPP_INCLUDED -// Much of the following code is based on Boost (1.53) +// Detect a number of compiler features - mostly C++11/14 conformance - by compiler +// The following features are defined: +// +// CATCH_CONFIG_CPP11_NULLPTR : is nullptr supported? +// CATCH_CONFIG_CPP11_NOEXCEPT : is noexcept supported? +// CATCH_CONFIG_CPP11_GENERATED_METHODS : The delete and default keywords for compiler generated methods +// CATCH_CONFIG_CPP11_IS_ENUM : std::is_enum is supported? +// CATCH_CONFIG_CPP11_TUPLE : std::tuple is supported + +// CATCH_CONFIG_CPP11_OR_GREATER : Is C++11 supported? + +// CATCH_CONFIG_VARIADIC_MACROS : are variadic macros supported? + +// In general each macro has a _NO_ form +// (e.g. CATCH_CONFIG_CPP11_NO_NULLPTR) which disables the feature. +// Many features, at point of detection, define an _INTERNAL_ macro, so they +// can be combined, en-mass, with the _NO_ forms later. + +// All the C++11 features can be disabled with CATCH_CONFIG_NO_CPP11 #ifdef __clang__ # if __has_feature(cxx_nullptr) -# define CATCH_CONFIG_CPP11_NULLPTR +# define CATCH_INTERNAL_CONFIG_CPP11_NULLPTR # endif # if __has_feature(cxx_noexcept) -# define CATCH_CONFIG_CPP11_NOEXCEPT +# define CATCH_INTERNAL_CONFIG_CPP11_NOEXCEPT # endif #endif // __clang__ @@ -82,51 +112,26 @@ // Borland #ifdef __BORLANDC__ -#if (__BORLANDC__ > 0x582 ) -//#define CATCH_CONFIG_SFINAE // Not confirmed -#endif - #endif // __BORLANDC__ //////////////////////////////////////////////////////////////////////////////// // EDG #ifdef __EDG_VERSION__ -#if (__EDG_VERSION__ > 238 ) -//#define CATCH_CONFIG_SFINAE // Not confirmed -#endif - #endif // __EDG_VERSION__ //////////////////////////////////////////////////////////////////////////////// // Digital Mars #ifdef __DMC__ -#if (__DMC__ > 0x840 ) -//#define CATCH_CONFIG_SFINAE // Not confirmed -#endif - #endif // __DMC__ //////////////////////////////////////////////////////////////////////////////// // GCC #ifdef __GNUC__ -#if __GNUC__ < 3 - -#if (__GNUC_MINOR__ >= 96 ) -//#define CATCH_CONFIG_SFINAE -#endif - -#elif __GNUC__ >= 3 - -// #define CATCH_CONFIG_SFINAE // Taking this out completely for now - -#endif // __GNUC__ < 3 - #if __GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 6 && defined(__GXX_EXPERIMENTAL_CXX0X__) ) - -#define CATCH_CONFIG_CPP11_NULLPTR +# define CATCH_INTERNAL_CONFIG_CPP11_NULLPTR #endif #endif // __GNUC__ @@ -135,8 +140,13 @@ // Visual C++ #ifdef _MSC_VER -#if (_MSC_VER >= 1310 ) // (VC++ 7.0+) -//#define CATCH_CONFIG_SFINAE // Not confirmed +#if (_MSC_VER >= 1600) +# define CATCH_INTERNAL_CONFIG_CPP11_NULLPTR +#endif + +#if (_MSC_VER >= 1900 ) // (VC++ 13 (VS2015)) +#define CATCH_INTERNAL_CONFIG_CPP11_NOEXCEPT +#define CATCH_INTERNAL_CONFIG_CPP11_GENERATED_METHODS #endif #endif // _MSC_VER @@ -147,21 +157,62 @@ ( defined __GNUC__ && __GNUC__ >= 3 ) || \ ( !defined __cplusplus && __STDC_VERSION__ >= 199901L || __cplusplus >= 201103L ) -#ifndef CATCH_CONFIG_NO_VARIADIC_MACROS -#define CATCH_CONFIG_VARIADIC_MACROS -#endif +#define CATCH_INTERNAL_CONFIG_VARIADIC_MACROS #endif //////////////////////////////////////////////////////////////////////////////// // C++ language feature support -// detect language version: -#if (__cplusplus == 201103L) -# define CATCH_CPP11 -# define CATCH_CPP11_OR_GREATER -#elif (__cplusplus >= 201103L) +// catch all support for C++11 +#if (__cplusplus >= 201103L) + # define CATCH_CPP11_OR_GREATER + +# if !defined(CATCH_INTERNAL_CONFIG_CPP11_NULLPTR) +# define CATCH_INTERNAL_CONFIG_CPP11_NULLPTR +# endif + +# ifndef CATCH_INTERNAL_CONFIG_CPP11_NOEXCEPT +# define CATCH_INTERNAL_CONFIG_CPP11_NOEXCEPT +# endif + +# ifndef CATCH_INTERNAL_CONFIG_CPP11_GENERATED_METHODS +# define CATCH_INTERNAL_CONFIG_CPP11_GENERATED_METHODS +# endif + +# ifndef CATCH_INTERNAL_CONFIG_CPP11_IS_ENUM +# define CATCH_INTERNAL_CONFIG_CPP11_IS_ENUM +# endif + +# ifndef CATCH_INTERNAL_CONFIG_CPP11_TUPLE +# define CATCH_INTERNAL_CONFIG_CPP11_TUPLE +# endif + +# ifndef CATCH_INTERNAL_CONFIG_VARIADIC_MACROS +# define CATCH_INTERNAL_CONFIG_VARIADIC_MACROS +# endif + +#endif // __cplusplus >= 201103L + +// Now set the actual defines based on the above + anything the user has configured +#if defined(CATCH_INTERNAL_CONFIG_CPP11_NULLPTR) && !defined(CATCH_CONFIG_CPP11_NO_NULLPTR) && !defined(CATCH_CONFIG_CPP11_NULLPTR) && !defined(CATCH_CONFIG_NO_CPP11) +# define CATCH_CONFIG_CPP11_NULLPTR +#endif +#if defined(CATCH_INTERNAL_CONFIG_CPP11_NOEXCEPT) && !defined(CATCH_CONFIG_CPP11_NO_NOEXCEPT) && !defined(CATCH_CONFIG_CPP11_NOEXCEPT) && !defined(CATCH_CONFIG_NO_CPP11) +# define CATCH_CONFIG_CPP11_NOEXCEPT +#endif +#if defined(CATCH_INTERNAL_CONFIG_CPP11_GENERATED_METHODS) && !defined(CATCH_CONFIG_CPP11_NO_GENERATED_METHODS) && !defined(CATCH_CONFIG_CPP11_GENERATED_METHODS) && !defined(CATCH_CONFIG_NO_CPP11) +# define CATCH_CONFIG_CPP11_GENERATED_METHODS +#endif +#if defined(CATCH_INTERNAL_CONFIG_CPP11_IS_ENUM) && !defined(CATCH_CONFIG_CPP11_NO_IS_ENUM) && !defined(CATCH_CONFIG_CPP11_IS_ENUM) && !defined(CATCH_CONFIG_NO_CPP11) +# define CATCH_CONFIG_CPP11_IS_ENUM +#endif +#if defined(CATCH_INTERNAL_CONFIG_CPP11_TUPLE) && !defined(CATCH_CONFIG_CPP11_NO_TUPLE) && !defined(CATCH_CONFIG_CPP11_TUPLE) && !defined(CATCH_CONFIG_NO_CPP11) +# define CATCH_CONFIG_CPP11_TUPLE +#endif +#if defined(CATCH_INTERNAL_CONFIG_VARIADIC_MACROS) && !defined(CATCH_CONFIG_NO_VARIADIC_MACROS) && !defined(CATCH_CONFIG_VARIADIC_MACROS) +#define CATCH_CONFIG_VARIADIC_MACROS #endif // noexcept support: @@ -176,8 +227,16 @@ namespace Catch { class NonCopyable { - NonCopyable( NonCopyable const& ); - void operator = ( NonCopyable const& ); +#ifdef CATCH_CONFIG_CPP11_GENERATED_METHODS + NonCopyable( NonCopyable const& ) = delete; + NonCopyable( NonCopyable && ) = delete; + NonCopyable& operator = ( NonCopyable const& ) = delete; + NonCopyable& operator = ( NonCopyable && ) = delete; +#else + NonCopyable( NonCopyable const& info ); + NonCopyable& operator = ( NonCopyable const& ); +#endif + protected: NonCopyable() {} virtual ~NonCopyable(); @@ -215,6 +274,7 @@ namespace Catch { void toLowerInPlace( std::string& s ); std::string toLower( std::string const& s ); std::string trim( std::string const& str ); + bool replaceInPlace( std::string& str, std::string const& replaceThis, std::string const& withThis ); struct pluralise { pluralise( std::size_t count, std::string const& label ); @@ -230,13 +290,14 @@ namespace Catch { SourceLineInfo(); SourceLineInfo( char const* _file, std::size_t _line ); SourceLineInfo( SourceLineInfo const& other ); -# ifdef CATCH_CPP11_OR_GREATER +# ifdef CATCH_CONFIG_CPP11_GENERATED_METHODS SourceLineInfo( SourceLineInfo && ) = default; SourceLineInfo& operator = ( SourceLineInfo const& ) = default; SourceLineInfo& operator = ( SourceLineInfo && ) = default; # endif bool empty() const; bool operator == ( SourceLineInfo const& other ) const; + bool operator < ( SourceLineInfo const& other ) const; std::string file; std::size_t line; @@ -467,7 +528,7 @@ namespace Catch { struct ITestCaseRegistry { virtual ~ITestCaseRegistry(); virtual std::vector const& getAllTests() const = 0; - virtual void getFilteredTests( TestSpec const& testSpec, IConfig const& config, std::vector& matchingTestCases ) const = 0; + virtual void getFilteredTests( TestSpec const& testSpec, IConfig const& config, std::vector& matchingTestCases, bool negated = false ) const = 0; }; } @@ -603,7 +664,9 @@ namespace Catch { Exception = 0x100 | FailureBit, ThrewException = Exception | 1, - DidntThrowException = Exception | 2 + DidntThrowException = Exception | 2, + + FatalErrorCondition = 0x200 | FailureBit }; }; @@ -616,11 +679,11 @@ namespace Catch { // ResultDisposition::Flags enum struct ResultDisposition { enum Flags { - Normal = 0x00, + Normal = 0x01, - ContinueOnFailure = 0x01, // Failures fail test, but execution continues - FalseTest = 0x02, // Prefix expression with ! - SuppressFail = 0x04 // Failures are reported but do not fail the test + ContinueOnFailure = 0x02, // Failures fail test, but execution continues + FalseTest = 0x04, // Prefix expression with ! + SuppressFail = 0x08 // Failures are reported but do not fail the test }; }; inline ResultDisposition::Flags operator | ( ResultDisposition::Flags lhs, ResultDisposition::Flags rhs ) { @@ -668,7 +731,7 @@ namespace Catch { AssertionResult(); AssertionResult( AssertionInfo const& info, AssertionResultData const& data ); ~AssertionResult(); -# ifdef CATCH_CPP11_OR_GREATER +# ifdef CATCH_CONFIG_CPP11_GENERATED_METHODS AssertionResult( AssertionResult const& ) = default; AssertionResult( AssertionResult && ) = default; AssertionResult& operator = ( AssertionResult const& ) = default; @@ -724,8 +787,8 @@ namespace Catch { ResultDisposition::Flags resultDisposition ); template - ExpressionLhs operator->* ( T const& operand ); - ExpressionLhs operator->* ( bool value ); + ExpressionLhs operator <= ( T const& operand ); + ExpressionLhs operator <= ( bool value ); template ResultBuilder& operator << ( T const& value ) { @@ -948,40 +1011,6 @@ namespace Internal { // #included from: catch_tostring.h #define TWOBLUECUBES_CATCH_TOSTRING_H_INCLUDED -// #included from: catch_sfinae.hpp -#define TWOBLUECUBES_CATCH_SFINAE_HPP_INCLUDED - -// Try to detect if the current compiler supports SFINAE - -namespace Catch { - - struct TrueType { - static const bool value = true; - typedef void Enable; - char sizer[1]; - }; - struct FalseType { - static const bool value = false; - typedef void Disable; - char sizer[2]; - }; - -#ifdef CATCH_CONFIG_SFINAE - - template struct NotABooleanExpression; - - template struct If : NotABooleanExpression {}; - template<> struct If : TrueType {}; - template<> struct If : FalseType {}; - - template struct SizedIf; - template<> struct SizedIf : TrueType {}; - template<> struct SizedIf : FalseType {}; - -#endif // CATCH_CONFIG_SFINAE - -} // end namespace Catch - #include #include #include @@ -1034,35 +1063,59 @@ inline id performOptionalSelector( id obj, SEL sel ) { #endif +#ifdef CATCH_CONFIG_CPP11_TUPLE +#include +#endif + +#ifdef CATCH_CONFIG_CPP11_IS_ENUM +#include +#endif + namespace Catch { -namespace Detail { -// SFINAE is currently disabled by default for all compilers. -// If the non SFINAE version of IsStreamInsertable is ambiguous for you -// and your compiler supports SFINAE, try #defining CATCH_CONFIG_SFINAE -#ifdef CATCH_CONFIG_SFINAE +// Why we're here. +template +std::string toString( T const& value ); - template - class IsStreamInsertableHelper { - template struct TrueIfSizeable : TrueType {}; +// Built in overloads - template - static TrueIfSizeable dummy(T2*); - static FalseType dummy(...); +std::string toString( std::string const& value ); +std::string toString( std::wstring const& value ); +std::string toString( const char* const value ); +std::string toString( char* const value ); +std::string toString( const wchar_t* const value ); +std::string toString( wchar_t* const value ); +std::string toString( int value ); +std::string toString( unsigned long value ); +std::string toString( unsigned int value ); +std::string toString( const double value ); +std::string toString( const float value ); +std::string toString( bool value ); +std::string toString( char value ); +std::string toString( signed char value ); +std::string toString( unsigned char value ); - public: - typedef SizedIf type; - }; +#ifdef CATCH_CONFIG_CPP11_NULLPTR +std::string toString( std::nullptr_t ); +#endif - template - struct IsStreamInsertable : IsStreamInsertableHelper::type {}; +#ifdef __OBJC__ + std::string toString( NSString const * const& nsstring ); + std::string toString( NSString * CATCH_ARC_STRONG const& nsstring ); + std::string toString( NSObject* const& nsObject ); +#endif -#else +namespace Detail { + + extern std::string unprintableString; struct BorgType { template BorgType( T const& ); }; + struct TrueType { char sizer[1]; }; + struct FalseType { char sizer[2]; }; + TrueType& testStreamable( std::ostream& ); FalseType testStreamable( FalseType ); @@ -1075,12 +1128,38 @@ namespace Detail { enum { value = sizeof( testStreamable(s << t) ) == sizeof( TrueType ) }; }; -#endif +#if defined(CATCH_CONFIG_CPP11_IS_ENUM) + template::value + > + struct EnumStringMaker + { + static std::string convert( T const& ) { return unprintableString; } + }; + template + struct EnumStringMaker + { + static std::string convert( T const& v ) + { + return ::Catch::toString( + static_cast::type>(v) + ); + } + }; +#endif template struct StringMakerBase { +#if defined(CATCH_CONFIG_CPP11_IS_ENUM) + template + static std::string convert( T const& v ) + { + return EnumStringMaker::convert( v ); + } +#else template - static std::string convert( T const& ) { return "{?}"; } + static std::string convert( T const& ) { return unprintableString; } +#endif }; template<> @@ -1102,9 +1181,6 @@ namespace Detail { } // end namespace Detail -template -std::string toString( T const& value ); - template struct StringMaker : Detail::StringMakerBase::value> {}; @@ -1135,12 +1211,59 @@ namespace Detail { std::string rangeToString( InputIterator first, InputIterator last ); } +//template +//struct StringMaker > { +// static std::string convert( std::vector const& v ) { +// return Detail::rangeToString( v.begin(), v.end() ); +// } +//}; + template -struct StringMaker > { - static std::string convert( std::vector const& v ) { - return Detail::rangeToString( v.begin(), v.end() ); +std::string toString( std::vector const& v ) { + return Detail::rangeToString( v.begin(), v.end() ); +} + +#ifdef CATCH_CONFIG_CPP11_TUPLE + +// toString for tuples +namespace TupleDetail { + template< + typename Tuple, + std::size_t N = 0, + bool = (N < std::tuple_size::value) + > + struct ElementPrinter { + static void print( const Tuple& tuple, std::ostream& os ) + { + os << ( N ? ", " : " " ) + << Catch::toString(std::get(tuple)); + ElementPrinter::print(tuple,os); + } + }; + + template< + typename Tuple, + std::size_t N + > + struct ElementPrinter { + static void print( const Tuple&, std::ostream& ) {} + }; + +} + +template +struct StringMaker> { + + static std::string convert( const std::tuple& tuple ) + { + std::ostringstream os; + os << '{'; + TupleDetail::ElementPrinter>::print( tuple, os ); + os << " }"; + return os.str(); } }; +#endif // CATCH_CONFIG_CPP11_TUPLE namespace Detail { template @@ -1161,44 +1284,15 @@ std::string toString( T const& value ) { return StringMaker::convert( value ); } -// Built in overloads - -std::string toString( std::string const& value ); -std::string toString( std::wstring const& value ); -std::string toString( const char* const value ); -std::string toString( char* const value ); -std::string toString( const wchar_t* const value ); -std::string toString( wchar_t* const value ); -std::string toString( int value ); -std::string toString( unsigned long value ); -std::string toString( unsigned int value ); -std::string toString( const double value ); -std::string toString( const float value ); -std::string toString( bool value ); -std::string toString( char value ); -std::string toString( signed char value ); -std::string toString( unsigned char value ); - -#ifdef CATCH_CONFIG_CPP11_NULLPTR -std::string toString( std::nullptr_t ); -#endif - -#ifdef __OBJC__ - std::string toString( NSString const * const& nsstring ); - std::string toString( NSString * CATCH_ARC_STRONG const& nsstring ); - std::string toString( NSObject* const& nsObject ); -#endif - namespace Detail { template std::string rangeToString( InputIterator first, InputIterator last ) { std::ostringstream oss; oss << "{ "; if( first != last ) { - oss << toString( *first ); - for( ++first ; first != last ; ++first ) { - oss << ", " << toString( *first ); - } + oss << Catch::toString( *first ); + for( ++first ; first != last ; ++first ) + oss << ", " << Catch::toString( *first ); } oss << " }"; return oss.str(); @@ -1214,13 +1308,13 @@ namespace Catch { template class ExpressionLhs { ExpressionLhs& operator = ( ExpressionLhs const& ); -# ifdef CATCH_CPP11_OR_GREATER +# ifdef CATCH_CONFIG_CPP11_GENERATED_METHODS ExpressionLhs& operator = ( ExpressionLhs && ) = delete; # endif public: ExpressionLhs( ResultBuilder& rb, T lhs ) : m_rb( rb ), m_lhs( lhs ) {} -# ifdef CATCH_CPP11_OR_GREATER +# ifdef CATCH_CONFIG_CPP11_GENERATED_METHODS ExpressionLhs( ExpressionLhs const& ) = default; ExpressionLhs( ExpressionLhs && ) = default; # endif @@ -1301,11 +1395,11 @@ class ExpressionLhs { namespace Catch { template - inline ExpressionLhs ResultBuilder::operator->* ( T const& operand ) { + inline ExpressionLhs ResultBuilder::operator <= ( T const& operand ) { return ExpressionLhs( *this, operand ); } - inline ExpressionLhs ResultBuilder::operator->* ( bool value ) { + inline ExpressionLhs ResultBuilder::operator <= ( bool value ) { return ExpressionLhs( *this, value ); } @@ -1395,6 +1489,8 @@ namespace Catch { virtual std::string getCurrentTestName() const = 0; virtual const AssertionResult* getLastResult() const = 0; + + virtual void handleFatalErrorCondition( std::string const& message ) = 0; }; IResultCapture& getResultCapture(); @@ -1475,7 +1571,7 @@ namespace Catch { do { \ Catch::ResultBuilder __catchResult( macroName, CATCH_INTERNAL_LINEINFO, #expr, resultDisposition ); \ try { \ - ( __catchResult->*expr ).endExpression(); \ + ( __catchResult <= expr ).endExpression(); \ } \ catch( ... ) { \ __catchResult.useActiveException( Catch::ResultDisposition::Normal ); \ @@ -1575,7 +1671,7 @@ namespace Catch { std::string matcherAsString = ::Catch::Matchers::matcher.toString(); \ __catchResult \ .setLhs( Catch::toString( arg ) ) \ - .setRhs( matcherAsString == "{?}" ? #matcher : matcherAsString ) \ + .setRhs( matcherAsString == Catch::Detail::unprintableString ? #matcher : matcherAsString ) \ .setOp( "matches" ) \ .setResultType( ::Catch::Matchers::matcher.match( arg ) ); \ __catchResult.captureExpression(); \ @@ -1636,6 +1732,9 @@ namespace Catch { bool allPassed() const { return failed == 0 && failedButOk == 0; } + bool allOk() const { + return failed == 0; + } std::size_t passed; std::size_t failed; @@ -1688,7 +1787,7 @@ namespace Catch { public: Timer() : m_ticks( 0 ) {} void start(); - unsigned int getElapsedNanoseconds() const; + unsigned int getElapsedMicroseconds() const; unsigned int getElapsedMilliseconds() const; double getElapsedSeconds() const; @@ -1702,7 +1801,7 @@ namespace Catch { namespace Catch { - class Section { + class Section : NonCopyable { public: Section( SectionInfo const& info ); ~Section(); @@ -1711,15 +1810,6 @@ namespace Catch { operator bool() const; private: -#ifdef CATCH_CPP11_OR_GREATER - Section( Section const& ) = delete; - Section( Section && ) = delete; - Section& operator = ( Section const& ) = delete; - Section& operator = ( Section && ) = delete; -#else - Section( Section const& info ); - Section& operator = ( Section const& ); -#endif SectionInfo m_info; std::string m_name; @@ -2694,7 +2784,7 @@ return @ desc; \ #endif -#ifdef CATCH_CONFIG_RUNNER +#ifdef CATCH_IMPL // #included from: internal/catch_impl.hpp #define TWOBLUECUBES_CATCH_IMPL_HPP_INCLUDED @@ -2706,7 +2796,7 @@ return @ desc; \ #pragma clang diagnostic ignored "-Wweak-vtables" #endif -// #included from: catch_runner.hpp +// #included from: ../catch_runner.hpp #define TWOBLUECUBES_CATCH_RUNNER_HPP_INCLUDED // #included from: internal/catch_commandline.hpp @@ -2962,6 +3052,11 @@ namespace Catch { Always, Never }; }; + struct RunTests { enum InWhatOrder { + InDeclarationOrder, + InLexicographicalOrder, + InRandomOrder + }; }; class TestSpec; @@ -2979,6 +3074,9 @@ namespace Catch { virtual bool showInvisibles() const = 0; virtual ShowDurations::OrNot showDurations() const = 0; virtual TestSpec const& testSpec() const = 0; + virtual RunTests::InWhatOrder runOrder() const = 0; + virtual unsigned int rngSeed() const = 0; + virtual bool forceColour() const = 0; }; } @@ -3004,12 +3102,16 @@ namespace Catch { private: bool isOwned; }; + + std::ostream& cout(); + std::ostream& cerr(); } #include #include #include #include +#include #ifndef CATCH_CONFIG_CONSOLE_WIDTH #define CATCH_CONFIG_CONSOLE_WIDTH 80 @@ -3029,10 +3131,13 @@ namespace Catch { noThrow( false ), showHelp( false ), showInvisibles( false ), + forceColour( false ), abortAfter( -1 ), + rngSeed( 0 ), verbosity( Verbosity::Normal ), warnings( WarnAbout::Nothing ), - showDurations( ShowDurations::DefaultForReporter ) + showDurations( ShowDurations::DefaultForReporter ), + runOrder( RunTests::InDeclarationOrder ) {} bool listTests; @@ -3045,12 +3150,15 @@ namespace Catch { bool noThrow; bool showHelp; bool showInvisibles; + bool forceColour; int abortAfter; + unsigned int rngSeed; Verbosity::Level verbosity; WarnAbout::What warnings; ShowDurations::OrNot showDurations; + RunTests::InWhatOrder runOrder; std::string reporterName; std::string outputFilename; @@ -3068,12 +3176,12 @@ namespace Catch { public: Config() - : m_os( std::cout.rdbuf() ) + : m_os( Catch::cout().rdbuf() ) {} Config( ConfigData const& data ) : m_data( data ), - m_os( std::cout.rdbuf() ) + m_os( Catch::cout().rdbuf() ) { if( !data.testsOrTags.empty() ) { TestSpecParser parser( ITagAliasRegistry::get() ); @@ -3084,7 +3192,7 @@ namespace Catch { } virtual ~Config() { - m_os.rdbuf( std::cout.rdbuf() ); + m_os.rdbuf( Catch::cout().rdbuf() ); m_stream.release(); } @@ -3106,7 +3214,7 @@ namespace Catch { bool shouldDebugBreak() const { return m_data.shouldDebugBreak; } void setStreamBuf( std::streambuf* buf ) { - m_os.rdbuf( buf ? buf : std::cout.rdbuf() ); + m_os.rdbuf( buf ? buf : Catch::cout().rdbuf() ); } void useStream( std::string const& streamName ) { @@ -3132,6 +3240,9 @@ namespace Catch { virtual bool includeSuccessfulResults() const { return m_data.showSuccessfulTests; } virtual bool warnAboutMissingAssertions() const { return m_data.warnings & WarnAbout::NoAssertions; } virtual ShowDurations::OrNot showDurations() const { return m_data.showDurations; } + virtual RunTests::InWhatOrder runOrder() const { return m_data.runOrder; } + virtual unsigned int rngSeed() const { return m_data.rngSeed; } + virtual bool forceColour() const { return m_data.forceColour; } private: ConfigData m_data; @@ -3393,7 +3504,7 @@ namespace Clara { template struct IArgFunction { virtual ~IArgFunction() {} -# ifdef CATCH_CPP11_OR_GREATER +# ifdef CATCH_CONFIG_CPP11_GENERATED_METHODS IArgFunction() = default; IArgFunction( IArgFunction const& ) = default; # endif @@ -3760,7 +3871,7 @@ namespace Clara { m_throwOnUnrecognisedTokens( other.m_throwOnUnrecognisedTokens ) { if( other.m_floatingArg.get() ) - m_floatingArg = ArgAutoPtr( new Arg( *other.m_floatingArg ) ); + m_floatingArg.reset( new Arg( *other.m_floatingArg ) ); } CommandLine& setThrowOnUnrecognisedTokens( bool shouldThrow = true ) { @@ -3788,7 +3899,7 @@ namespace Clara { ArgBuilder operator[]( UnpositionalTag ) { if( m_floatingArg.get() ) throw std::logic_error( "Only one unpositional argument can be added" ); - m_floatingArg = ArgAutoPtr( new Arg() ); + m_floatingArg.reset( new Arg() ); ArgBuilder builder( m_floatingArg.get() ); return builder; } @@ -3930,7 +4041,7 @@ namespace Clara { if( it == itEnd ) { if( token.type == Parser::Token::Positional || !m_throwOnUnrecognisedTokens ) unusedTokens.push_back( token ); - else if( m_throwOnUnrecognisedTokens ) + else if( errors.empty() && m_throwOnUnrecognisedTokens ) errors.push_back( "unrecognised option: " + token.data ); } } @@ -4028,7 +4139,28 @@ namespace Catch { config.warnings = static_cast( config.warnings | WarnAbout::NoAssertions ); else throw std::runtime_error( "Unrecognised warning: '" + _warning + "'" ); - + } + inline void setOrder( ConfigData& config, std::string const& order ) { + if( startsWith( "declared", order ) ) + config.runOrder = RunTests::InDeclarationOrder; + else if( startsWith( "lexical", order ) ) + config.runOrder = RunTests::InLexicographicalOrder; + else if( startsWith( "random", order ) ) + config.runOrder = RunTests::InRandomOrder; + else + throw std::runtime_error( "Unrecognised ordering: '" + order + "'" ); + } + inline void setRngSeed( ConfigData& config, std::string const& seed ) { + if( seed == "time" ) { + config.rngSeed = static_cast( std::time(0) ); + } + else { + std::stringstream ss; + ss << seed; + ss >> config.rngSeed; + if( ss.fail() ) + throw std::runtime_error( "Argment to --rng-seed should be the word 'time' or a number" ); + } } inline void setVerbosity( ConfigData& config, int level ) { // !TBD: accept strings? @@ -4140,6 +4272,18 @@ namespace Catch { .describe( "list all reporters" ) .bind( &ConfigData::listReporters ); + cli["--order"] + .describe( "test case order (defaults to decl)" ) + .bind( &setOrder, "decl|lex|rand" ); + + cli["--rng-seed"] + .describe( "set a specific seed for random numbers" ) + .bind( &setRngSeed, "'time'|number" ); + + cli["--force-colour"] + .describe( "force colourised output" ) + .bind( &ConfigData::forceColour ); + return cli; } @@ -4313,10 +4457,6 @@ namespace Catch { namespace Catch { - namespace Detail { - struct IColourImpl; - } - struct Colour { enum Code { None = 0, @@ -4362,7 +4502,6 @@ namespace Catch { static void use( Code _colourCode ); private: - static Detail::IColourImpl* impl(); bool m_moved; }; @@ -4456,7 +4595,7 @@ namespace Catch } virtual ~AssertionStats(); -# ifdef CATCH_CPP11_OR_GREATER +# ifdef CATCH_CONFIG_CPP11_GENERATED_METHODS AssertionStats( AssertionStats const& ) = default; AssertionStats( AssertionStats && ) = default; AssertionStats& operator = ( AssertionStats const& ) = default; @@ -4479,7 +4618,7 @@ namespace Catch missingAssertions( _missingAssertions ) {} virtual ~SectionStats(); -# ifdef CATCH_CPP11_OR_GREATER +# ifdef CATCH_CONFIG_CPP11_GENERATED_METHODS SectionStats( SectionStats const& ) = default; SectionStats( SectionStats && ) = default; SectionStats& operator = ( SectionStats const& ) = default; @@ -4506,7 +4645,7 @@ namespace Catch {} virtual ~TestCaseStats(); -# ifdef CATCH_CPP11_OR_GREATER +# ifdef CATCH_CONFIG_CPP11_GENERATED_METHODS TestCaseStats( TestCaseStats const& ) = default; TestCaseStats( TestCaseStats && ) = default; TestCaseStats& operator = ( TestCaseStats const& ) = default; @@ -4534,7 +4673,7 @@ namespace Catch {} virtual ~TestGroupStats(); -# ifdef CATCH_CPP11_OR_GREATER +# ifdef CATCH_CONFIG_CPP11_GENERATED_METHODS TestGroupStats( TestGroupStats const& ) = default; TestGroupStats( TestGroupStats && ) = default; TestGroupStats& operator = ( TestGroupStats const& ) = default; @@ -4556,7 +4695,7 @@ namespace Catch {} virtual ~TestRunStats(); -# ifndef CATCH_CPP11_OR_GREATER +# ifndef CATCH_CONFIG_CPP11_GENERATED_METHODS TestRunStats( TestRunStats const& _other ) : runInfo( _other.runInfo ), totals( _other.totals ), @@ -4592,11 +4731,14 @@ namespace Catch virtual void assertionStarting( AssertionInfo const& assertionInfo ) = 0; + // The return value indicates if the messages buffer should be cleared: virtual bool assertionEnded( AssertionStats const& assertionStats ) = 0; virtual void sectionEnded( SectionStats const& sectionStats ) = 0; virtual void testCaseEnded( TestCaseStats const& testCaseStats ) = 0; virtual void testGroupEnded( TestGroupStats const& testGroupStats ) = 0; virtual void testRunEnded( TestRunStats const& testRunStats ) = 0; + + virtual void skipTest( TestCaseInfo const& testInfo ) = 0; }; struct IReporterFactory { @@ -4624,9 +4766,9 @@ namespace Catch { TestSpec testSpec = config.testSpec(); if( config.testSpec().hasFilters() ) - std::cout << "Matching test cases:\n"; + Catch::cout() << "Matching test cases:\n"; else { - std::cout << "All available test cases:\n"; + Catch::cout() << "All available test cases:\n"; testSpec = TestSpecParser( ITagAliasRegistry::get() ).parse( "*" ).testSpec(); } @@ -4647,15 +4789,15 @@ namespace Catch { : Colour::None; Colour colourGuard( colour ); - std::cout << Text( testCaseInfo.name, nameAttr ) << std::endl; + Catch::cout() << Text( testCaseInfo.name, nameAttr ) << std::endl; if( !testCaseInfo.tags.empty() ) - std::cout << Text( testCaseInfo.tagsAsString, tagsAttr ) << std::endl; + Catch::cout() << Text( testCaseInfo.tagsAsString, tagsAttr ) << std::endl; } if( !config.testSpec().hasFilters() ) - std::cout << pluralise( matchedTests, "test case" ) << "\n" << std::endl; + Catch::cout() << pluralise( matchedTests, "test case" ) << "\n" << std::endl; else - std::cout << pluralise( matchedTests, "matching test case" ) << "\n" << std::endl; + Catch::cout() << pluralise( matchedTests, "matching test case" ) << "\n" << std::endl; return matchedTests; } @@ -4671,7 +4813,7 @@ namespace Catch { ++it ) { matchedTests++; TestCaseInfo const& testCaseInfo = it->getTestCaseInfo(); - std::cout << testCaseInfo.name << std::endl; + Catch::cout() << testCaseInfo.name << std::endl; } return matchedTests; } @@ -4697,9 +4839,9 @@ namespace Catch { inline std::size_t listTags( Config const& config ) { TestSpec testSpec = config.testSpec(); if( config.testSpec().hasFilters() ) - std::cout << "Tags for matching test cases:\n"; + Catch::cout() << "Tags for matching test cases:\n"; else { - std::cout << "All available tags:\n"; + Catch::cout() << "All available tags:\n"; testSpec = TestSpecParser( ITagAliasRegistry::get() ).parse( "*" ).testSpec(); } @@ -4733,14 +4875,14 @@ namespace Catch { .setInitialIndent( 0 ) .setIndent( oss.str().size() ) .setWidth( CATCH_CONFIG_CONSOLE_WIDTH-10 ) ); - std::cout << oss.str() << wrapper << "\n"; + Catch::cout() << oss.str() << wrapper << "\n"; } - std::cout << pluralise( tagCounts.size(), "tag" ) << "\n" << std::endl; + Catch::cout() << pluralise( tagCounts.size(), "tag" ) << "\n" << std::endl; return tagCounts.size(); } inline std::size_t listReporters( Config const& /*config*/ ) { - std::cout << "Available reports:\n"; + Catch::cout() << "Available reporters:\n"; IReporterRegistry::FactoryMap const& factories = getRegistryHub().getReporterRegistry().getFactories(); IReporterRegistry::FactoryMap::const_iterator itBegin = factories.begin(), itEnd = factories.end(), it; std::size_t maxNameLen = 0; @@ -4752,13 +4894,13 @@ namespace Catch { .setInitialIndent( 0 ) .setIndent( 7+maxNameLen ) .setWidth( CATCH_CONFIG_CONSOLE_WIDTH - maxNameLen-8 ) ); - std::cout << " " + Catch::cout() << " " << it->first << ":" << std::string( maxNameLen - it->first.size() + 2, ' ' ) << wrapper << "\n"; } - std::cout << std::endl; + Catch::cout() << std::endl; return factories.size(); } @@ -4808,32 +4950,15 @@ namespace SectionTracking { RunState runState() const { return m_runState; } - TrackedSection* findChild( std::string const& childName ) { - TrackedSections::iterator it = m_children.find( childName ); - return it != m_children.end() - ? &it->second - : NULL; - } - TrackedSection* acquireChild( std::string const& childName ) { - if( TrackedSection* child = findChild( childName ) ) - return child; - m_children.insert( std::make_pair( childName, TrackedSection( childName, this ) ) ); - return findChild( childName ); - } + TrackedSection* findChild( std::string const& childName ); + TrackedSection* acquireChild( std::string const& childName ); + void enter() { if( m_runState == NotStarted ) m_runState = Executing; } - void leave() { - for( TrackedSections::const_iterator it = m_children.begin(), itEnd = m_children.end(); - it != itEnd; - ++it ) - if( it->second.runState() != Completed ) { - m_runState = ExecutingChildren; - return; - } - m_runState = Completed; - } + void leave(); + TrackedSection* getParent() { return m_parent; } @@ -4846,9 +4971,31 @@ namespace SectionTracking { RunState m_runState; TrackedSections m_children; TrackedSection* m_parent; - }; + inline TrackedSection* TrackedSection::findChild( std::string const& childName ) { + TrackedSections::iterator it = m_children.find( childName ); + return it != m_children.end() + ? &it->second + : NULL; + } + inline TrackedSection* TrackedSection::acquireChild( std::string const& childName ) { + if( TrackedSection* child = findChild( childName ) ) + return child; + m_children.insert( std::make_pair( childName, TrackedSection( childName, this ) ) ); + return findChild( childName ); + } + inline void TrackedSection::leave() { + for( TrackedSections::const_iterator it = m_children.begin(), itEnd = m_children.end(); + it != itEnd; + ++it ) + if( it->second.runState() != Completed ) { + m_runState = ExecutingChildren; + return; + } + m_runState = Completed; + } + class TestCaseTracker { public: TestCaseTracker( std::string const& testCaseName ) @@ -4915,6 +5062,81 @@ using SectionTracking::TestCaseTracker; } // namespace Catch +// #included from: catch_fatal_condition.hpp +#define TWOBLUECUBES_CATCH_FATAL_CONDITION_H_INCLUDED + +namespace Catch { + + // Report the error condition then exit the process + inline void fatal( std::string const& message, int exitCode ) { + IContext& context = Catch::getCurrentContext(); + IResultCapture* resultCapture = context.getResultCapture(); + resultCapture->handleFatalErrorCondition( message ); + + if( Catch::alwaysTrue() ) // avoids "no return" warnings + exit( exitCode ); + } + +} // namespace Catch + +#if defined ( CATCH_PLATFORM_WINDOWS ) ///////////////////////////////////////// + +namespace Catch { + + struct FatalConditionHandler { + void reset() {} + }; + +} // namespace Catch + +#else // Not Windows - assumed to be POSIX compatible ////////////////////////// + +#include + +namespace Catch { + + struct SignalDefs { int id; const char* name; }; + extern SignalDefs signalDefs[]; + SignalDefs signalDefs[] = { + { SIGINT, "SIGINT - Terminal interrupt signal" }, + { SIGILL, "SIGILL - Illegal instruction signal" }, + { SIGFPE, "SIGFPE - Floating point error signal" }, + { SIGSEGV, "SIGSEGV - Segmentation violation signal" }, + { SIGTERM, "SIGTERM - Termination request signal" }, + { SIGABRT, "SIGABRT - Abort (abnormal termination) signal" } + }; + + struct FatalConditionHandler { + + static void handleSignal( int sig ) { + for( std::size_t i = 0; i < sizeof(signalDefs)/sizeof(SignalDefs); ++i ) + if( sig == signalDefs[i].id ) + fatal( signalDefs[i].name, -sig ); + fatal( "", -sig ); + } + + FatalConditionHandler() : m_isSet( true ) { + for( std::size_t i = 0; i < sizeof(signalDefs)/sizeof(SignalDefs); ++i ) + signal( signalDefs[i].id, handleSignal ); + } + ~FatalConditionHandler() { + reset(); + } + void reset() { + if( m_isSet ) { + for( std::size_t i = 0; i < sizeof(signalDefs)/sizeof(SignalDefs); ++i ) + signal( signalDefs[i].id, SIG_DFL ); + m_isSet = false; + } + } + + bool m_isSet; + }; + +} // namespace Catch + +#endif // not Windows + #include #include @@ -5102,6 +5324,37 @@ namespace Catch { return &m_lastResult; } + virtual void handleFatalErrorCondition( std::string const& message ) { + ResultBuilder resultBuilder = makeUnexpectedResultBuilder(); + resultBuilder.setResultType( ResultWas::FatalErrorCondition ); + resultBuilder << message; + resultBuilder.captureExpression(); + + handleUnfinishedSections(); + + // Recreate section for test case (as we will lose the one that was in scope) + TestCaseInfo const& testCaseInfo = m_activeTestCase->getTestCaseInfo(); + SectionInfo testCaseSection( testCaseInfo.lineInfo, testCaseInfo.name, testCaseInfo.description ); + + Counts assertions; + assertions.failed = 1; + SectionStats testCaseSectionStats( testCaseSection, assertions, 0, false ); + m_reporter->sectionEnded( testCaseSectionStats ); + + TestCaseInfo testInfo = m_activeTestCase->getTestCaseInfo(); + + Totals deltaTotals; + deltaTotals.testCases.failed = 1; + m_reporter->testCaseEnded( TestCaseStats( testInfo, + deltaTotals, + "", + "", + false ) ); + m_totals.testCases.failed++; + testGroupEnded( "", m_totals, 1, 1 ); + m_reporter->testRunEnded( TestRunStats( m_runInfo, m_totals, false ) ); + } + public: // !TBD We need to do this another way! bool aborting() const { @@ -5123,12 +5376,12 @@ namespace Catch { Timer timer; timer.start(); if( m_reporter->getPreferences().shouldRedirectStdOut ) { - StreamRedirect coutRedir( std::cout, redirectedCout ); - StreamRedirect cerrRedir( std::cerr, redirectedCerr ); - m_activeTestCase->invoke(); + StreamRedirect coutRedir( Catch::cout(), redirectedCout ); + StreamRedirect cerrRedir( Catch::cerr(), redirectedCerr ); + invokeActiveTestCase(); } else { - m_activeTestCase->invoke(); + invokeActiveTestCase(); } duration = timer.getElapsedSeconds(); } @@ -5136,20 +5389,9 @@ namespace Catch { // This just means the test was aborted due to failure } catch(...) { - ResultBuilder exResult( m_lastAssertionInfo.macroName.c_str(), - m_lastAssertionInfo.lineInfo, - m_lastAssertionInfo.capturedExpression.c_str(), - m_lastAssertionInfo.resultDisposition ); - exResult.useActiveException(); + makeUnexpectedResultBuilder().useActiveException(); } - // If sections ended prematurely due to an exception we stored their - // infos here so we can tear them down outside the unwind process. - for( std::vector::const_reverse_iterator it = m_unfinishedSections.rbegin(), - itEnd = m_unfinishedSections.rend(); - it != itEnd; - ++it ) - sectionEnded( it->info, it->prevAssertions, it->durationInSeconds ); - m_unfinishedSections.clear(); + handleUnfinishedSections(); m_messages.clear(); Counts assertions = m_totals.assertions - prevAssertions; @@ -5165,7 +5407,32 @@ namespace Catch { m_reporter->sectionEnded( testCaseSectionStats ); } + void invokeActiveTestCase() { + FatalConditionHandler fatalConditionHandler; // Handle signals + m_activeTestCase->invoke(); + fatalConditionHandler.reset(); + } + private: + + ResultBuilder makeUnexpectedResultBuilder() const { + return ResultBuilder( m_lastAssertionInfo.macroName.c_str(), + m_lastAssertionInfo.lineInfo, + m_lastAssertionInfo.capturedExpression.c_str(), + m_lastAssertionInfo.resultDisposition ); + } + + void handleUnfinishedSections() { + // If sections ended prematurely due to an exception we stored their + // infos here so we can tear them down outside the unwind process. + for( std::vector::const_reverse_iterator it = m_unfinishedSections.rbegin(), + itEnd = m_unfinishedSections.rend(); + it != itEnd; + ++it ) + sectionEnded( it->info, it->prevAssertions, it->durationInSeconds ); + m_unfinishedSections.clear(); + } + struct UnfinishedSections { UnfinishedSections( SectionInfo const& _info, Counts const& _prevAssertions, double _durationInSeconds ) : info( _info ), prevAssertions( _prevAssertions ), durationInSeconds( _durationInSeconds ) @@ -5211,18 +5478,19 @@ namespace Catch { struct Version { Version( unsigned int _majorVersion, unsigned int _minorVersion, - unsigned int _buildNumber, - char const* const _branchName ) - : majorVersion( _majorVersion ), - minorVersion( _minorVersion ), - buildNumber( _buildNumber ), - branchName( _branchName ) - {} + unsigned int _patchNumber, + std::string const& _branchName, + unsigned int _buildNumber ); unsigned int const majorVersion; unsigned int const minorVersion; + unsigned int const patchNumber; + + // buildNumber is only used if branchName is not null + std::string const branchName; unsigned int const buildNumber; - char const* const branchName; + + friend std::ostream& operator << ( std::ostream& os, Version const& version ); private: void operator=( Version const& ); @@ -5253,7 +5521,7 @@ namespace Catch { Totals totals; - context.testGroupStarting( "", 1, 1 ); // deprecated? + context.testGroupStarting( "all tests", 1, 1 ); // deprecated? TestSpec testSpec = m_config->testSpec(); if( !testSpec.hasFilters() ) @@ -5276,7 +5544,15 @@ namespace Catch { m_testsAlreadyRun.insert( *it ); } } - context.testGroupEnded( "", totals, 1, 1 ); + std::vector skippedTestCases; + getRegistryHub().getTestCaseRegistry().getFilteredTests( testSpec, *m_config, skippedTestCases, true ); + + for( std::vector::const_iterator it = skippedTestCases.begin(), itEnd = skippedTestCases.end(); + it != itEnd; + ++it ) + m_reporter->skipTest( *it ); + + context.testGroupEnded( "all tests", totals, 1, 1 ); return totals; } @@ -5313,7 +5589,7 @@ namespace Catch { std::set m_testsAlreadyRun; }; - class Session { + class Session : NonCopyable { static bool alreadyInstantiated; public: @@ -5324,7 +5600,7 @@ namespace Catch { : m_cli( makeCommandLineParser() ) { if( alreadyInstantiated ) { std::string msg = "Only one instance of Catch::Session can ever be used"; - std::cerr << msg << std::endl; + Catch::cerr() << msg << std::endl; throw std::logic_error( msg ); } alreadyInstantiated = true; @@ -5334,15 +5610,10 @@ namespace Catch { } void showHelp( std::string const& processName ) { - std::cout << "\nCatch v" << libraryVersion.majorVersion << "." - << libraryVersion.minorVersion << " build " - << libraryVersion.buildNumber; - if( libraryVersion.branchName != std::string( "master" ) ) - std::cout << " (" << libraryVersion.branchName << " branch)"; - std::cout << "\n"; + Catch::cout() << "\nCatch v" << libraryVersion << "\n"; - m_cli.usage( std::cout, processName ); - std::cout << "For more detail usage please see the project docs\n" << std::endl; + m_cli.usage( Catch::cout(), processName ); + Catch::cout() << "For more detail usage please see the project docs\n" << std::endl; } int applyCommandLine( int argc, char* const argv[], OnUnusedOptions::DoWhat unusedOptionBehaviour = OnUnusedOptions::Fail ) { @@ -5356,11 +5627,12 @@ namespace Catch { catch( std::exception& ex ) { { Colour colourGuard( Colour::Red ); - std::cerr << "\nError(s) in input:\n" - << Text( ex.what(), TextAttributes().setIndent(2) ) - << "\n\n"; + Catch::cerr() + << "\nError(s) in input:\n" + << Text( ex.what(), TextAttributes().setIndent(2) ) + << "\n\n"; } - m_cli.usage( std::cout, m_configData.processName ); + m_cli.usage( Catch::cout(), m_configData.processName ); return (std::numeric_limits::max)(); } return 0; @@ -5386,6 +5658,9 @@ namespace Catch { try { config(); // Force config to be constructed + + std::srand( m_configData.rngSeed ); + Runner runner( m_config ); // Handle list request @@ -5395,7 +5670,7 @@ namespace Catch { return static_cast( runner.runTests().assertions.failed ); } catch( std::exception& ex ) { - std::cerr << ex.what() << std::endl; + Catch::cerr() << ex.what() << std::endl; return (std::numeric_limits::max)(); } } @@ -5436,10 +5711,18 @@ namespace Catch { #include #include #include +#include namespace Catch { class TestRegistry : public ITestCaseRegistry { + struct LexSort { + bool operator() (TestCase i,TestCase j) const { return (i& matchingTestCases ) const { + virtual void getFilteredTests( TestSpec const& testSpec, IConfig const& config, std::vector& matchingTestCases, bool negated = false ) const { + for( std::vector::const_iterator it = m_functionsInOrder.begin(), itEnd = m_functionsInOrder.end(); it != itEnd; ++it ) { - if( testSpec.matches( *it ) && ( config.allowThrows() || !it->throws() ) ) + bool includeTest = testSpec.matches( *it ) && ( config.allowThrows() || !it->throws() ); + if( includeTest != negated ) matchingTestCases.push_back( *it ); } + sortTests( config, matchingTestCases ); } private: + static void sortTests( IConfig const& config, std::vector& matchingTestCases ) { + + switch( config.runOrder() ) { + case RunTests::InLexicographicalOrder: + std::sort( matchingTestCases.begin(), matchingTestCases.end(), LexSort() ); + break; + case RunTests::InRandomOrder: + { + RandomNumberGenerator rng; + std::random_shuffle( matchingTestCases.begin(), matchingTestCases.end(), rng ); + } + break; + case RunTests::InDeclarationOrder: + // already in declaration order + break; + } + } std::set m_functions; std::vector m_functionsInOrder; std::vector m_nonHiddenFunctions; @@ -5613,7 +5916,7 @@ namespace Catch { throw; } @catch (NSException *exception) { - return toString( [exception description] ); + return Catch::toString( [exception description] ); } #else throw; @@ -5760,6 +6063,7 @@ namespace Catch { #include #include +#include namespace Catch { @@ -5823,6 +6127,15 @@ namespace Catch { isOwned = false; } } + +#ifndef CATCH_CONFIG_NOSTDOUT // If you #define this you must implement this functions + std::ostream& cout() { + return std::cout; + } + std::ostream& cerr() { + return std::cerr; + } +#endif } namespace Catch { @@ -5872,7 +6185,7 @@ namespace Catch { std::string testName = getResultCapture()->getCurrentTestName(); std::map::const_iterator it = - m_generatorsByTestName.find( testName ); + m_generatorsByTestName.find( testName ); return it != m_generatorsByTestName.end() ? it->second : NULL; @@ -5908,8 +6221,8 @@ namespace Catch { } Stream createStream( std::string const& streamName ) { - if( streamName == "stdout" ) return Stream( std::cout.rdbuf(), false ); - if( streamName == "stderr" ) return Stream( std::cerr.rdbuf(), false ); + if( streamName == "stdout" ) return Stream( Catch::cout().rdbuf(), false ); + if( streamName == "stderr" ) return Stream( Catch::cerr().rdbuf(), false ); if( streamName == "debug" ) return Stream( new StreamBufImpl, true ); throw std::domain_error( "Unknown stream: " + streamName ); @@ -5924,14 +6237,35 @@ namespace Catch { // #included from: catch_console_colour_impl.hpp #define TWOBLUECUBES_CATCH_CONSOLE_COLOUR_IMPL_HPP_INCLUDED -namespace Catch { namespace Detail { - struct IColourImpl { - virtual ~IColourImpl() {} - virtual void use( Colour::Code _colourCode ) = 0; - }; -}} +namespace Catch { + namespace { -#if defined ( CATCH_PLATFORM_WINDOWS ) ///////////////////////////////////////// + struct IColourImpl { + virtual ~IColourImpl() {} + virtual void use( Colour::Code _colourCode ) = 0; + }; + + struct NoColourImpl : IColourImpl { + void use( Colour::Code ) {} + + static IColourImpl* instance() { + static NoColourImpl s_instance; + return &s_instance; + } + }; + + } // anon namespace +} // namespace Catch + +#if !defined( CATCH_CONFIG_COLOUR_NONE ) && !defined( CATCH_CONFIG_COLOUR_WINDOWS ) && !defined( CATCH_CONFIG_COLOUR_ANSI ) +# ifdef CATCH_PLATFORM_WINDOWS +# define CATCH_CONFIG_COLOUR_WINDOWS +# else +# define CATCH_CONFIG_COLOUR_ANSI +# endif +#endif + +#if defined ( CATCH_CONFIG_COLOUR_WINDOWS ) ///////////////////////////////////////// #ifndef NOMINMAX #define NOMINMAX @@ -5946,7 +6280,7 @@ namespace Catch { namespace Detail { namespace Catch { namespace { - class Win32ColourImpl : public Detail::IColourImpl { + class Win32ColourImpl : public IColourImpl { public: Win32ColourImpl() : stdoutHandle( GetStdHandle(STD_OUTPUT_HANDLE) ) { @@ -5983,11 +6317,7 @@ namespace { WORD originalAttributes; }; - inline bool shouldUseColourForPlatform() { - return true; - } - - static Detail::IColourImpl* platformColourInstance() { + IColourImpl* platformColourInstance() { static Win32ColourImpl s_instance; return &s_instance; } @@ -5995,7 +6325,7 @@ namespace { } // end anon namespace } // end namespace Catch -#else // Not Windows - assumed to be POSIX compatible ////////////////////////// +#elif defined( CATCH_CONFIG_COLOUR_ANSI ) ////////////////////////////////////// #include @@ -6006,7 +6336,7 @@ namespace { // Thanks to Adam Strzelecki for original contribution // (http://github.com/nanoant) // https://github.com/philsquared/Catch/pull/131 - class PosixColourImpl : public Detail::IColourImpl { + class PosixColourImpl : public IColourImpl { public: virtual void use( Colour::Code _colourCode ) { switch( _colourCode ) { @@ -6027,53 +6357,48 @@ namespace { case Colour::Bright: throw std::logic_error( "not a colour" ); } } + static IColourImpl* instance() { + static PosixColourImpl s_instance; + return &s_instance; + } + private: void setColour( const char* _escapeCode ) { - std::cout << '\033' << _escapeCode; + Catch::cout() << '\033' << _escapeCode; } }; - inline bool shouldUseColourForPlatform() { - return isatty(STDOUT_FILENO); - } - - static Detail::IColourImpl* platformColourInstance() { - static PosixColourImpl s_instance; - return &s_instance; + IColourImpl* platformColourInstance() { + Ptr config = getCurrentContext().getConfig(); + return (config && config->forceColour()) || isatty(STDOUT_FILENO) + ? PosixColourImpl::instance() + : NoColourImpl::instance(); } } // end anon namespace } // end namespace Catch -#endif // not Windows +#else // not Windows or ANSI /////////////////////////////////////////////// namespace Catch { - namespace { - struct NoColourImpl : Detail::IColourImpl { - void use( Colour::Code ) {} + static IColourImpl* platformColourInstance() { return NoColourImpl::instance(); } - static IColourImpl* instance() { - static NoColourImpl s_instance; - return &s_instance; - } - }; - static bool shouldUseColour() { - return shouldUseColourForPlatform() && !isDebuggerActive(); - } - } +} // end namespace Catch + +#endif // Windows/ ANSI/ None + +namespace Catch { Colour::Colour( Code _colourCode ) : m_moved( false ) { use( _colourCode ); } Colour::Colour( Colour const& _other ) : m_moved( false ) { const_cast( _other ).m_moved = true; } Colour::~Colour(){ if( !m_moved ) use( None ); } - void Colour::use( Code _colourCode ) { - impl()->use( _colourCode ); - } - Detail::IColourImpl* Colour::impl() { - return shouldUseColour() - ? platformColourInstance() - : NoColourImpl::instance(); + void Colour::use( Code _colourCode ) { + static IColourImpl* impl = isDebuggerActive() + ? NoColourImpl::instance() + : platformColourInstance(); + impl->use( _colourCode ); } } // end namespace Catch @@ -6238,7 +6563,7 @@ namespace Catch { namespace Catch { inline TestCaseInfo::SpecialProperties parseSpecialTag( std::string const& tag ) { - if( tag == "." || + if( startsWith( tag, "." ) || tag == "hide" || tag == "!hide" ) return TestCaseInfo::IsHidden; @@ -6258,13 +6583,13 @@ namespace Catch { if( isReservedTag( tag ) ) { { Colour colourGuard( Colour::Red ); - std::cerr + Catch::cerr() << "Tag name [" << tag << "] not allowed.\n" << "Tag names starting with non alpha-numeric characters are reserved\n"; } { Colour colourGuard( Colour::FileName ); - std::cerr << _lineInfo << std::endl; + Catch::cerr() << _lineInfo << std::endl; } exit(1); } @@ -6292,14 +6617,15 @@ namespace Catch { } else { if( c == ']' ) { - enforceNotReservedTag( tag, _lineInfo ); - - inTag = false; - if( tag == "hide" || tag == "." ) + TestCaseInfo::SpecialProperties prop = parseSpecialTag( tag ); + if( prop == TestCaseInfo::IsHidden ) isHidden = true; - else - tags.insert( tag ); + else if( prop == TestCaseInfo::None ) + enforceNotReservedTag( tag, _lineInfo ); + + tags.insert( tag ); tag.clear(); + inTag = false; } else tag += c; @@ -6416,8 +6742,33 @@ namespace Catch { namespace Catch { - // These numbers are maintained by a script - Version libraryVersion( 1, 0, 53, "master" ); + Version::Version + ( unsigned int _majorVersion, + unsigned int _minorVersion, + unsigned int _patchNumber, + std::string const& _branchName, + unsigned int _buildNumber ) + : majorVersion( _majorVersion ), + minorVersion( _minorVersion ), + patchNumber( _patchNumber ), + branchName( _branchName ), + buildNumber( _buildNumber ) + {} + + std::ostream& operator << ( std::ostream& os, Version const& version ) { + os << version.majorVersion << "." + << version.minorVersion << "." + << version.patchNumber; + + if( !version.branchName.empty() ) { + os << "-" << version.branchName + << "." << version.buildNumber; + } + return os; + } + + Version libraryVersion( 1, 2, 1, "", 0 ); + } // #included from: catch_message.hpp @@ -6501,6 +6852,7 @@ namespace Catch virtual void testCaseEnded( TestCaseStats const& testCaseStats ); virtual void testGroupEnded( TestGroupStats const& testGroupStats ); virtual void testRunEnded( TestRunStats const& testRunStats ); + virtual void skipTest( TestCaseInfo const& ); private: Ptr m_legacyReporter; @@ -6574,6 +6926,8 @@ namespace Catch void LegacyReporterAdapter::testRunEnded( TestRunStats const& testRunStats ) { m_legacyReporter->EndTesting( testRunStats.totals ); } + void LegacyReporterAdapter::skipTest( TestCaseInfo const& ) { + } } // #included from: catch_timer.hpp @@ -6596,11 +6950,11 @@ namespace Catch { uint64_t getCurrentTicks() { static uint64_t hz=0, hzo=0; if (!hz) { - QueryPerformanceFrequency((LARGE_INTEGER*)&hz); - QueryPerformanceCounter((LARGE_INTEGER*)&hzo); + QueryPerformanceFrequency( reinterpret_cast( &hz ) ); + QueryPerformanceCounter( reinterpret_cast( &hzo ) ); } uint64_t t; - QueryPerformanceCounter((LARGE_INTEGER*)&t); + QueryPerformanceCounter( reinterpret_cast( &t ) ); return ((t-hzo)*1000000)/hz; } #else @@ -6615,14 +6969,14 @@ namespace Catch { void Timer::start() { m_ticks = getCurrentTicks(); } - unsigned int Timer::getElapsedNanoseconds() const { + unsigned int Timer::getElapsedMicroseconds() const { return static_cast(getCurrentTicks() - m_ticks); } unsigned int Timer::getElapsedMilliseconds() const { - return static_cast((getCurrentTicks() - m_ticks)/1000); + return static_cast(getElapsedMicroseconds()/1000); } double Timer::getElapsedSeconds() const { - return (getCurrentTicks() - m_ticks)/1000000.0; + return getElapsedMicroseconds()/1000000.0; } } // namespace Catch @@ -6660,6 +7014,20 @@ namespace Catch { return start != std::string::npos ? str.substr( start, 1+end-start ) : ""; } + bool replaceInPlace( std::string& str, std::string const& replaceThis, std::string const& withThis ) { + bool replaced = false; + std::size_t i = str.find( replaceThis ); + while( i != std::string::npos ) { + replaced = true; + str = str.substr( 0, i ) + withThis + str.substr( i+replaceThis.size() ); + if( i < str.size()-withThis.size() ) + i = str.find( replaceThis, i+withThis.size() ); + else + i = std::string::npos; + } + return replaced; + } + pluralise::pluralise( std::size_t count, std::string const& label ) : m_count( count ), m_label( label ) @@ -6687,6 +7055,9 @@ namespace Catch { bool SourceLineInfo::operator == ( SourceLineInfo const& other ) const { return line == other.line && file == other.file; } + bool SourceLineInfo::operator < ( SourceLineInfo const& other ) const { + return line < other.line || ( line == other.line && file < other.file ); + } std::ostream& operator << ( std::ostream& os, SourceLineInfo const& info ) { #ifndef __GNUG__ @@ -6781,7 +7152,7 @@ namespace Catch { size = sizeof(info); if( sysctl(mib, sizeof(mib) / sizeof(*mib), &info, &size, NULL, 0) != 0 ) { - std::cerr << "\n** Call to sysctl failed - unable to determine if debugger is active **\n" << std::endl; + Catch::cerr() << "\n** Call to sysctl failed - unable to determine if debugger is active **\n" << std::endl; return false; } @@ -6822,7 +7193,7 @@ namespace Catch { namespace Catch { void writeToDebugConsole( std::string const& text ) { // !TBD: Need a version for Mac/ XCode and other IDEs - std::cout << text; + Catch::cout() << text; } } #endif // Platform @@ -6834,6 +7205,8 @@ namespace Catch { namespace Detail { + std::string unprintableString = "{?}"; + namespace { struct Endianness { enum Arch { Big, Little }; @@ -6892,7 +7265,7 @@ std::string toString( std::wstring const& value ) { s.reserve( value.size() ); for(size_t i = 0; i < value.size(); ++i ) s += value[i] <= 0xff ? static_cast( value[i] ) : '?'; - return toString( s ); + return Catch::toString( s ); } std::string toString( const char* const value ) { @@ -6916,20 +7289,21 @@ std::string toString( wchar_t* const value ) std::string toString( int value ) { std::ostringstream oss; oss << value; + if( value >= 255 ) + oss << " (0x" << std::hex << value << ")"; return oss.str(); } std::string toString( unsigned long value ) { std::ostringstream oss; - if( value > 8192 ) - oss << "0x" << std::hex << value; - else - oss << value; + oss << value; + if( value >= 255 ) + oss << " (0x" << std::hex << value << ")"; return oss.str(); } std::string toString( unsigned int value ) { - return toString( static_cast( value ) ); + return Catch::toString( static_cast( value ) ); } template @@ -7055,7 +7429,7 @@ namespace Catch { if( !result.isOk() ) { if( getCurrentContext().getConfig()->shouldDebugBreak() ) m_shouldDebugBreak = true; - if( getCurrentContext().getRunner()->aborting() || m_assertionInfo.resultDisposition == ResultDisposition::Normal ) + if( getCurrentContext().getRunner()->aborting() || (m_assertionInfo.resultDisposition & ResultDisposition::Normal) ) m_shouldThrow = true; } } @@ -7195,7 +7569,7 @@ namespace Catch { } catch( std::exception& ex ) { Colour colourGuard( Colour::Red ); - std::cerr << ex.what() << std::endl; + Catch::cerr() << ex.what() << std::endl; exit(1); } } @@ -7208,6 +7582,8 @@ namespace Catch { // #included from: catch_reporter_bases.hpp #define TWOBLUECUBES_CATCH_REPORTER_BASES_HPP_INCLUDED +#include + namespace Catch { struct StreamingReporterBase : SharedImpl { @@ -7240,7 +7616,6 @@ namespace Catch { } virtual void testCaseEnded( TestCaseStats const& /* _testCaseStats */ ) { currentTestCaseInfo.reset(); - assert( m_sectionStack.empty() ); } virtual void testGroupEnded( TestGroupStats const& /* _testGroupStats */ ) { currentGroupInfo.reset(); @@ -7251,6 +7626,11 @@ namespace Catch { currentTestRunInfo.reset(); } + virtual void skipTest( TestCaseInfo const& ) { + // Don't do anything with this by default. + // It can optionally be overridden in the derived class. + } + Ptr m_config; std::ostream& stream; @@ -7380,6 +7760,8 @@ namespace Catch { } virtual void testRunEndedCumulative() = 0; + virtual void skipTest( TestCaseInfo const& ) {} + Ptr m_config; std::ostream& stream; std::vector m_assertions; @@ -7395,6 +7777,16 @@ namespace Catch { }; + template + char const* getLineOfChars() { + static char line[CATCH_CONFIG_CONSOLE_WIDTH] = {0}; + if( !*line ) { + memset( line, C, CATCH_CONFIG_CONSOLE_WIDTH-1 ); + line[CATCH_CONFIG_CONSOLE_WIDTH-1] = 0; + } + return line; + } + } // end namespace Catch // #included from: ../internal/catch_reporter_registrars.hpp @@ -7464,7 +7856,6 @@ namespace Catch { #define TWOBLUECUBES_CATCH_XMLWRITER_HPP_INCLUDED #include -#include #include #include @@ -7507,7 +7898,7 @@ namespace Catch { XmlWriter() : m_tagIsOpen( false ), m_needsNewline( false ), - m_os( &std::cout ) + m_os( &Catch::cout() ) {} XmlWriter( std::ostream& os ) @@ -7521,27 +7912,6 @@ namespace Catch { endElement(); } -//# ifndef CATCH_CPP11_OR_GREATER -// XmlWriter& operator = ( XmlWriter const& other ) { -// XmlWriter temp( other ); -// swap( temp ); -// return *this; -// } -//# else -// XmlWriter( XmlWriter const& ) = default; -// XmlWriter( XmlWriter && ) = default; -// XmlWriter& operator = ( XmlWriter const& ) = default; -// XmlWriter& operator = ( XmlWriter && ) = default; -//# endif -// -// void swap( XmlWriter& other ) { -// std::swap( m_tagIsOpen, other.m_tagIsOpen ); -// std::swap( m_needsNewline, other.m_needsNewline ); -// std::swap( m_tags, other.m_tags ); -// std::swap( m_indent, other.m_indent ); -// std::swap( m_os, other.m_os ); -// } - XmlWriter& startElement( std::string const& name ) { ensureTagClosed(); newlineIfNecessary(); @@ -7677,81 +8047,90 @@ namespace Catch { } namespace Catch { - class XmlReporter : public SharedImpl { + class XmlReporter : public StreamingReporterBase { public: - XmlReporter( ReporterConfig const& config ) : m_config( config ), m_sectionDepth( 0 ) {} + XmlReporter( ReporterConfig const& _config ) + : StreamingReporterBase( _config ), + m_sectionDepth( 0 ) + {} + + virtual ~XmlReporter(); static std::string getDescription() { return "Reports test results as an XML document"; } - virtual ~XmlReporter(); - private: // IReporter - - virtual bool shouldRedirectStdout() const { - return true; + public: // StreamingReporterBase + virtual ReporterPreferences getPreferences() const { + ReporterPreferences prefs; + prefs.shouldRedirectStdOut = true; + return prefs; } - virtual void StartTesting() { - m_xml.setStream( m_config.stream() ); - m_xml.startElement( "Catch" ); - if( !m_config.fullConfig()->name().empty() ) - m_xml.writeAttribute( "name", m_config.fullConfig()->name() ); + virtual void noMatchingTestCases( std::string const& s ) { + StreamingReporterBase::noMatchingTestCases( s ); } - virtual void EndTesting( const Totals& totals ) { - m_xml.scopedElement( "OverallResults" ) - .writeAttribute( "successes", totals.assertions.passed ) - .writeAttribute( "failures", totals.assertions.failed ) - .writeAttribute( "expectedFailures", totals.assertions.failedButOk ); - m_xml.endElement(); + virtual void testRunStarting( TestRunInfo const& testInfo ) { + StreamingReporterBase::testRunStarting( testInfo ); + m_xml.setStream( stream ); + m_xml.startElement( "Catch" ); + if( !m_config->name().empty() ) + m_xml.writeAttribute( "name", m_config->name() ); } - virtual void StartGroup( const std::string& groupName ) { + virtual void testGroupStarting( GroupInfo const& groupInfo ) { + StreamingReporterBase::testGroupStarting( groupInfo ); m_xml.startElement( "Group" ) - .writeAttribute( "name", groupName ); + .writeAttribute( "name", groupInfo.name ); } - virtual void EndGroup( const std::string&, const Totals& totals ) { - m_xml.scopedElement( "OverallResults" ) - .writeAttribute( "successes", totals.assertions.passed ) - .writeAttribute( "failures", totals.assertions.failed ) - .writeAttribute( "expectedFailures", totals.assertions.failedButOk ); - m_xml.endElement(); + virtual void testCaseStarting( TestCaseInfo const& testInfo ) { + StreamingReporterBase::testCaseStarting(testInfo); + m_xml.startElement( "TestCase" ).writeAttribute( "name", trim( testInfo.name ) ); + + if ( m_config->showDurations() == ShowDurations::Always ) + m_testCaseTimer.start(); } - virtual void StartSection( const std::string& sectionName, const std::string& description ) { + virtual void sectionStarting( SectionInfo const& sectionInfo ) { + StreamingReporterBase::sectionStarting( sectionInfo ); if( m_sectionDepth++ > 0 ) { m_xml.startElement( "Section" ) - .writeAttribute( "name", trim( sectionName ) ) - .writeAttribute( "description", description ); + .writeAttribute( "name", trim( sectionInfo.name ) ) + .writeAttribute( "description", sectionInfo.description ); } } - virtual void NoAssertionsInSection( const std::string& ) {} - virtual void NoAssertionsInTestCase( const std::string& ) {} - virtual void EndSection( const std::string& /*sectionName*/, const Counts& assertions ) { - if( --m_sectionDepth > 0 ) { - m_xml.scopedElement( "OverallResults" ) - .writeAttribute( "successes", assertions.passed ) - .writeAttribute( "failures", assertions.failed ) - .writeAttribute( "expectedFailures", assertions.failedButOk ); - m_xml.endElement(); - } - } + virtual void assertionStarting( AssertionInfo const& ) { } - virtual void StartTestCase( const Catch::TestCaseInfo& testInfo ) { - m_xml.startElement( "TestCase" ).writeAttribute( "name", trim( testInfo.name ) ); - m_currentTestSuccess = true; - } + virtual bool assertionEnded( AssertionStats const& assertionStats ) { + const AssertionResult& assertionResult = assertionStats.assertionResult; - virtual void Result( const Catch::AssertionResult& assertionResult ) { - if( !m_config.fullConfig()->includeSuccessfulResults() && assertionResult.getResultType() == ResultWas::Ok ) - return; + // Print any info messages in tags. + if( assertionStats.assertionResult.getResultType() != ResultWas::Ok ) { + for( std::vector::const_iterator it = assertionStats.infoMessages.begin(), itEnd = assertionStats.infoMessages.end(); + it != itEnd; + ++it ) { + if( it->type == ResultWas::Info ) { + m_xml.scopedElement( "Info" ) + .writeText( it->message ); + } else if ( it->type == ResultWas::Warning ) { + m_xml.scopedElement( "Warning" ) + .writeText( it->message ); + } + } + } + // Drop out if result was successful but we're not printing them. + if( !m_config->includeSuccessfulResults() && isOk(assertionResult.getResultType()) ) + return true; + + // Print the expression if there is one. if( assertionResult.hasExpression() ) { m_xml.startElement( "Expression" ) .writeAttribute( "success", assertionResult.succeeded() ) + .writeAttribute( "type", assertionResult.getTestMacroName() ) .writeAttribute( "filename", assertionResult.getSourceInfo().file ) .writeAttribute( "line", assertionResult.getSourceInfo().line ); @@ -7759,58 +8138,96 @@ namespace Catch { .writeText( assertionResult.getExpression() ); m_xml.scopedElement( "Expanded" ) .writeText( assertionResult.getExpandedExpression() ); - m_currentTestSuccess &= assertionResult.succeeded(); } + // And... Print a result applicable to each result type. switch( assertionResult.getResultType() ) { case ResultWas::ThrewException: m_xml.scopedElement( "Exception" ) .writeAttribute( "filename", assertionResult.getSourceInfo().file ) .writeAttribute( "line", assertionResult.getSourceInfo().line ) .writeText( assertionResult.getMessage() ); - m_currentTestSuccess = false; + break; + case ResultWas::FatalErrorCondition: + m_xml.scopedElement( "Fatal Error Condition" ) + .writeAttribute( "filename", assertionResult.getSourceInfo().file ) + .writeAttribute( "line", assertionResult.getSourceInfo().line ) + .writeText( assertionResult.getMessage() ); break; case ResultWas::Info: m_xml.scopedElement( "Info" ) .writeText( assertionResult.getMessage() ); break; case ResultWas::Warning: - m_xml.scopedElement( "Warning" ) - .writeText( assertionResult.getMessage() ); + // Warning will already have been written break; case ResultWas::ExplicitFailure: m_xml.scopedElement( "Failure" ) .writeText( assertionResult.getMessage() ); - m_currentTestSuccess = false; break; - case ResultWas::Unknown: - case ResultWas::Ok: - case ResultWas::FailureBit: - case ResultWas::ExpressionFailed: - case ResultWas::Exception: - case ResultWas::DidntThrowException: + default: break; } + if( assertionResult.hasExpression() ) m_xml.endElement(); + + return true; } - virtual void Aborted() { - // !TBD + virtual void sectionEnded( SectionStats const& sectionStats ) { + StreamingReporterBase::sectionEnded( sectionStats ); + if( --m_sectionDepth > 0 ) { + XmlWriter::ScopedElement e = m_xml.scopedElement( "OverallResults" ); + e.writeAttribute( "successes", sectionStats.assertions.passed ); + e.writeAttribute( "failures", sectionStats.assertions.failed ); + e.writeAttribute( "expectedFailures", sectionStats.assertions.failedButOk ); + + if ( m_config->showDurations() == ShowDurations::Always ) + e.writeAttribute( "durationInSeconds", sectionStats.durationInSeconds ); + + m_xml.endElement(); + } } - virtual void EndTestCase( const Catch::TestCaseInfo&, const Totals&, const std::string&, const std::string& ) { - m_xml.scopedElement( "OverallResult" ).writeAttribute( "success", m_currentTestSuccess ); + virtual void testCaseEnded( TestCaseStats const& testCaseStats ) { + StreamingReporterBase::testCaseEnded( testCaseStats ); + XmlWriter::ScopedElement e = m_xml.scopedElement( "OverallResult" ); + e.writeAttribute( "success", testCaseStats.totals.assertions.allOk() ); + + if ( m_config->showDurations() == ShowDurations::Always ) + e.writeAttribute( "durationInSeconds", m_testCaseTimer.getElapsedSeconds() ); + + m_xml.endElement(); + } + + virtual void testGroupEnded( TestGroupStats const& testGroupStats ) { + StreamingReporterBase::testGroupEnded( testGroupStats ); + // TODO: Check testGroupStats.aborting and act accordingly. + m_xml.scopedElement( "OverallResults" ) + .writeAttribute( "successes", testGroupStats.totals.assertions.passed ) + .writeAttribute( "failures", testGroupStats.totals.assertions.failed ) + .writeAttribute( "expectedFailures", testGroupStats.totals.assertions.failedButOk ); + m_xml.endElement(); + } + + virtual void testRunEnded( TestRunStats const& testRunStats ) { + StreamingReporterBase::testRunEnded( testRunStats ); + m_xml.scopedElement( "OverallResults" ) + .writeAttribute( "successes", testRunStats.totals.assertions.passed ) + .writeAttribute( "failures", testRunStats.totals.assertions.failed ) + .writeAttribute( "expectedFailures", testRunStats.totals.assertions.failedButOk ); m_xml.endElement(); } private: - ReporterConfig m_config; - bool m_currentTestSuccess; + Timer m_testCaseTimer; XmlWriter m_xml; int m_sectionDepth; }; + INTERNAL_CATCH_REGISTER_REPORTER( "xml", XmlReporter ) + } // end namespace Catch // #included from: ../reporters/catch_reporter_junit.hpp @@ -7937,7 +8354,7 @@ namespace Catch { xml.writeAttribute( "classname", className ); xml.writeAttribute( "name", name ); } - xml.writeAttribute( "time", toString( sectionNode.stats.durationInSeconds ) ); + xml.writeAttribute( "time", Catch::toString( sectionNode.stats.durationInSeconds ) ); writeAssertions( sectionNode ); @@ -7970,6 +8387,7 @@ namespace Catch { std::string elementName; switch( result.getResultType() ) { case ResultWas::ThrewException: + case ResultWas::FatalErrorCondition: elementName = "error"; break; case ResultWas::ExplicitFailure: @@ -8028,8 +8446,6 @@ namespace Catch { // #included from: ../reporters/catch_reporter_console.hpp #define TWOBLUECUBES_CATCH_REPORTER_CONSOLE_HPP_INCLUDED -#include - namespace Catch { struct ConsoleReporter : StreamingReporterBase { @@ -8164,6 +8580,11 @@ namespace Catch { passOrFail = "FAILED"; messageLabel = "due to unexpected exception with message"; break; + case ResultWas::FatalErrorCondition: + colour = Colour::Error; + passOrFail = "FAILED"; + messageLabel = "due to a fatal error condition"; + break; case ResultWas::DidntThrowException: colour = Colour::Error; passOrFail = "FAILED"; @@ -8273,14 +8694,12 @@ namespace Catch { stream << "\n" << getLineOfChars<'~'>() << "\n"; Colour colour( Colour::SecondaryText ); stream << currentTestRunInfo->name - << " is a Catch v" << libraryVersion.majorVersion << "." - << libraryVersion.minorVersion << " b" - << libraryVersion.buildNumber; - if( libraryVersion.branchName != std::string( "master" ) ) - stream << " (" << libraryVersion.branchName << ")"; - stream << " host application.\n" + << " is a Catch v" << libraryVersion << " host application.\n" << "Run with -? for options\n\n"; + if( m_config->rngSeed() != 0 ) + stream << "Randomness seeded to: " << m_config->rngSeed() << "\n\n"; + currentTestRunInfo.used = true; } void lazyPrintGroupInfo() { @@ -8452,15 +8871,6 @@ namespace Catch { void printSummaryDivider() { stream << getLineOfChars<'-'>() << "\n"; } - template - static char const* getLineOfChars() { - static char line[CATCH_CONFIG_CONSOLE_WIDTH] = {0}; - if( !*line ) { - memset( line, C, CATCH_CONFIG_CONSOLE_WIDTH-1 ); - line[CATCH_CONFIG_CONSOLE_WIDTH-1] = 0; - } - return line; - } private: bool m_headerPrinted; @@ -8569,6 +8979,13 @@ namespace Catch { printExpressionWas(); printRemainingMessages(); break; + case ResultWas::FatalErrorCondition: + printResultType( Colour::Error, failedString() ); + printIssue( "fatal error condition with message:" ); + printMessage(); + printExpressionWas(); + printRemainingMessages(); + break; case ResultWas::DidntThrowException: printResultType( Colour::Error, failedString() ); printIssue( "expected exception, got none" ); @@ -8798,8 +9215,6 @@ namespace Catch { Matchers::Impl::StdString::EndsWith::~EndsWith() {} void Config::dummy() {} - - INTERNAL_CATCH_REGISTER_LEGACY_REPORTER( "xml", XmlReporter ) } #ifdef __clang__ @@ -8988,9 +9403,13 @@ using Catch::Detail::Approx; #define TWOBLUECUBES_CATCH_REENABLE_WARNINGS_H_INCLUDED #ifdef __clang__ -#pragma clang diagnostic pop +# ifdef __ICC // icpc defines the __clang__ macro +# pragma warning(pop) +# else +# pragma clang diagnostic pop +# endif #elif defined __GNUC__ -#pragma GCC diagnostic pop +# pragma GCC diagnostic pop #endif #endif // TWOBLUECUBES_SINGLE_INCLUDE_CATCH_HPP_INCLUDED diff --git a/test/t/basic/helper.hpp b/test/t/basic/helper.hpp index 61e076934dc..5a2130e4eca 100644 --- a/test/t/basic/helper.hpp +++ b/test/t/basic/helper.hpp @@ -9,7 +9,7 @@ inline void add_tags(osmium::memory::Buffer& buffer, osmium::builder::Builder& builder, const std::vector>& tags) { osmium::builder::TagListBuilder tl_builder(buffer, &builder); - for (auto& tag : tags) { + for (const auto& tag : tags) { tl_builder.add_tag(tag.first, tag.second); } } @@ -26,9 +26,11 @@ inline osmium::Way& buffer_add_way(osmium::memory::Buffer& buffer, const char* u osmium::builder::WayBuilder builder(buffer); builder.add_user(user); add_tags(buffer, builder, tags); - osmium::builder::WayNodeListBuilder wnl_builder(buffer, &builder); - for (const osmium::object_id_type ref : nodes) { - wnl_builder.add_node_ref(ref); + { + osmium::builder::WayNodeListBuilder wnl_builder(buffer, &builder); + for (const osmium::object_id_type ref : nodes) { + wnl_builder.add_node_ref(ref); + } } buffer.commit(); return builder.object(); @@ -38,9 +40,11 @@ inline osmium::Way& buffer_add_way(osmium::memory::Buffer& buffer, const char* u osmium::builder::WayBuilder builder(buffer); builder.add_user(user); add_tags(buffer, builder, tags); - osmium::builder::WayNodeListBuilder wnl_builder(buffer, &builder); - for (auto& p : nodes) { - wnl_builder.add_node_ref(p.first, p.second); + { + osmium::builder::WayNodeListBuilder wnl_builder(buffer, &builder); + for (const auto& p : nodes) { + wnl_builder.add_node_ref(p.first, p.second); + } } buffer.commit(); return builder.object(); @@ -53,9 +57,11 @@ inline osmium::Relation& buffer_add_relation( osmium::builder::RelationBuilder builder(buffer); builder.add_user(user); add_tags(buffer, builder, tags); - osmium::builder::RelationMemberListBuilder rml_builder(buffer, &builder); - for (const auto& member : members) { - rml_builder.add_member(osmium::char_to_item_type(std::get<0>(member)), std::get<1>(member), std::get<2>(member)); + { + osmium::builder::RelationMemberListBuilder rml_builder(buffer, &builder); + for (const auto& member : members) { + rml_builder.add_member(osmium::char_to_item_type(std::get<0>(member)), std::get<1>(member), std::get<2>(member)); + } } buffer.commit(); return builder.object(); @@ -69,15 +75,15 @@ inline osmium::Area& buffer_add_area(osmium::memory::Buffer& buffer, const char* builder.add_user(user); add_tags(buffer, builder, tags); - for (auto& ring : rings) { + for (const auto& ring : rings) { if (ring.first) { osmium::builder::OuterRingBuilder ring_builder(buffer, &builder); - for (auto& p : ring.second) { + for (const auto& p : ring.second) { ring_builder.add_node_ref(p.first, p.second); } } else { osmium::builder::InnerRingBuilder ring_builder(buffer, &builder); - for (auto& p : ring.second) { + for (const auto& p : ring.second) { ring_builder.add_node_ref(p.first, p.second); } } diff --git a/test/t/basic/test_box.cpp b/test/t/basic/test_box.cpp index 8182fbf353c..768cf41f31a 100644 --- a/test/t/basic/test_box.cpp +++ b/test/t/basic/test_box.cpp @@ -2,7 +2,10 @@ #include +#include + #include +#include #include TEST_CASE("Box") { @@ -48,6 +51,10 @@ TEST_CASE("Box") { REQUIRE(b.contains(loc1)); REQUIRE(b.contains(loc2)); REQUIRE(b.contains(loc3)); + + osmium::CRC crc32; + crc32.update(b); + REQUIRE(crc32().checksum() == 0xd381a838); } SECTION("output_defined") { diff --git a/test/t/basic/test_changeset.cpp b/test/t/basic/test_changeset.cpp index 2549c1e6767..fc9f1bdee99 100644 --- a/test/t/basic/test_changeset.cpp +++ b/test/t/basic/test_changeset.cpp @@ -1,12 +1,16 @@ #include "catch.hpp" +#include + #include +#include #include "helper.hpp" -TEST_CASE("Basic_Changeset") { +TEST_CASE("Basic Changeset") { + + osmium::CRC crc32; -SECTION("changeset_builder") { osmium::memory::Buffer buffer(10 * 1000); osmium::Changeset& cs1 = buffer_add_changeset(buffer, @@ -28,6 +32,9 @@ SECTION("changeset_builder") { REQUIRE(1 == cs1.tags().size()); REQUIRE(std::string("user") == cs1.user()); + crc32.update(cs1); + REQUIRE(crc32().checksum() == 0xf44aff25); + osmium::Changeset& cs2 = buffer_add_changeset(buffer, "user", {{"comment", "foo"}, {"foo", "bar"}}); @@ -52,6 +59,5 @@ SECTION("changeset_builder") { REQUIRE(cs1 <= cs2); REQUIRE(false == (cs1 > cs2)); REQUIRE(false == (cs1 >= cs2)); -} } diff --git a/test/t/basic/test_crc.cpp b/test/t/basic/test_crc.cpp new file mode 100644 index 00000000000..aab1013f4cf --- /dev/null +++ b/test/t/basic/test_crc.cpp @@ -0,0 +1,49 @@ +#include "catch.hpp" + +#include + +#include + +#include "helper.hpp" + +TEST_CASE("CRC of basic datatypes") { + + osmium::CRC crc32; + + SECTION("Bool") { + crc32.update_bool(true); + crc32.update_bool(false); + + REQUIRE(crc32().checksum() == 0x58c223be); + } + + SECTION("Char") { + crc32.update_int8('x'); + crc32.update_int8('y'); + + REQUIRE(crc32().checksum() == 0x8fe62899); + } + + SECTION("String") { + const char* str = "foobar"; + crc32.update_string(str); + + REQUIRE(crc32().checksum() == 0x9ef61f95); + } + + SECTION("Timestamp") { + osmium::Timestamp t("2015-07-12T13:10:46Z"); + crc32.update(t); + + REQUIRE(crc32().checksum() == 0x58a29d7); + } + + SECTION("Location") { + osmium::Location loc { 3.46, 2.001 }; + crc32.update(loc); + + REQUIRE(crc32().checksum() == 0xddee042c); + } + +} + diff --git a/test/t/basic/test_node.cpp b/test/t/basic/test_node.cpp index 6c2c899bd25..db5b4cd53ce 100644 --- a/test/t/basic/test_node.cpp +++ b/test/t/basic/test_node.cpp @@ -1,11 +1,16 @@ #include "catch.hpp" +#include + +#include #include #include "helper.hpp" TEST_CASE("Basic_Node") { + osmium::CRC crc32; + SECTION("node_builder") { osmium::memory::Buffer buffer(10000); @@ -36,6 +41,9 @@ SECTION("node_builder") { REQUIRE(osmium::Location(3.5, 4.7) == node.location()); REQUIRE(2 == node.tags().size()); + crc32.update(node); + REQUIRE(crc32().checksum() == 0xc696802f); + node.set_visible(false); REQUIRE(false == node.visible()); REQUIRE(true == node.deleted()); diff --git a/test/t/basic/test_relation.cpp b/test/t/basic/test_relation.cpp index 4c62a4189c1..fd5c7b4ad29 100644 --- a/test/t/basic/test_relation.cpp +++ b/test/t/basic/test_relation.cpp @@ -1,12 +1,16 @@ #include "catch.hpp" +#include + +#include #include #include "helper.hpp" -TEST_CASE("Basic_Relation") { +TEST_CASE("Build relation") { + + osmium::CRC crc32; -SECTION("relation_builder") { osmium::memory::Buffer buffer(10000); osmium::Relation& relation = buffer_add_relation(buffer, @@ -55,6 +59,7 @@ SECTION("relation_builder") { } ++n; } -} + crc32.update(relation); + REQUIRE(crc32().checksum() == 0xebcd836d); } diff --git a/test/t/basic/test_timestamp.cpp b/test/t/basic/test_timestamp.cpp index 6a04a4d0957..f015730ea00 100644 --- a/test/t/basic/test_timestamp.cpp +++ b/test/t/basic/test_timestamp.cpp @@ -29,6 +29,10 @@ TEST_CASE("Timestamp") { REQUIRE("2000-01-01T00:00:00Z" == t.to_iso()); } + SECTION("throws if initialized from bad string") { + REQUIRE_THROWS_AS(osmium::Timestamp("x"), std::invalid_argument); + } + SECTION("can be implicitly cast to time_t") { osmium::Timestamp t(4242); time_t x = t; diff --git a/test/t/basic/test_types_from_string.cpp b/test/t/basic/test_types_from_string.cpp new file mode 100644 index 00000000000..2481ae87649 --- /dev/null +++ b/test/t/basic/test_types_from_string.cpp @@ -0,0 +1,90 @@ +#include "catch.hpp" + +#include +#include + +TEST_CASE("set ID from string") { + REQUIRE(osmium::string_to_object_id("0") == 0); + REQUIRE(osmium::string_to_object_id("17") == 17); + REQUIRE(osmium::string_to_object_id("-17") == -17); + REQUIRE(osmium::string_to_object_id("01") == 1); + + REQUIRE_THROWS_AS(osmium::string_to_object_id(""), std::range_error); + REQUIRE_THROWS_AS(osmium::string_to_object_id(" "), std::range_error); + REQUIRE_THROWS_AS(osmium::string_to_object_id(" 22"), std::range_error); + REQUIRE_THROWS_AS(osmium::string_to_object_id("x"), std::range_error); + REQUIRE_THROWS_AS(osmium::string_to_object_id("0x1"), std::range_error); + REQUIRE_THROWS_AS(osmium::string_to_object_id("12a"), std::range_error); + REQUIRE_THROWS_AS(osmium::string_to_object_id("12345678901234567890"), std::range_error); +} + +TEST_CASE("set type and ID from string") { + auto n17 = osmium::string_to_object_id("n17", osmium::osm_entity_bits::nwr); + REQUIRE(n17.first == osmium::item_type::node); + REQUIRE(n17.second == 17); + + auto w42 = osmium::string_to_object_id("w42", osmium::osm_entity_bits::nwr); + REQUIRE(w42.first == osmium::item_type::way); + REQUIRE(w42.second == 42); + + auto r_2 = osmium::string_to_object_id("r-2", osmium::osm_entity_bits::nwr); + REQUIRE(r_2.first == osmium::item_type::relation); + REQUIRE(r_2.second == -2); + + auto x3 = osmium::string_to_object_id("3", osmium::osm_entity_bits::nwr); + REQUIRE(x3.first == osmium::item_type::undefined); + REQUIRE(x3.second == 3); + + REQUIRE_THROWS_AS(osmium::string_to_object_id("", osmium::osm_entity_bits::nwr), std::range_error); + REQUIRE_THROWS_AS(osmium::string_to_object_id("n", osmium::osm_entity_bits::nwr), std::range_error); + REQUIRE_THROWS_AS(osmium::string_to_object_id("x3", osmium::osm_entity_bits::nwr), std::range_error); + REQUIRE_THROWS_AS(osmium::string_to_object_id("nx3", osmium::osm_entity_bits::nwr), std::range_error); + REQUIRE_THROWS_AS(osmium::string_to_object_id("n3", osmium::osm_entity_bits::way), std::range_error); + REQUIRE_THROWS_AS(osmium::string_to_object_id("n3a", osmium::osm_entity_bits::nwr), std::range_error); +} + +TEST_CASE("set object version from string") { + REQUIRE(osmium::string_to_object_version("0") == 0); + REQUIRE(osmium::string_to_object_version("1") == 1); + + REQUIRE_THROWS_AS(osmium::string_to_object_version("-1"), std::range_error); + REQUIRE_THROWS_AS(osmium::string_to_object_version(""), std::range_error); + REQUIRE_THROWS_AS(osmium::string_to_object_version(" "), std::range_error); + REQUIRE_THROWS_AS(osmium::string_to_object_version(" 22"), std::range_error); + REQUIRE_THROWS_AS(osmium::string_to_object_version("x"), std::range_error); +} + +TEST_CASE("set changeset id from string") { + REQUIRE(osmium::string_to_changeset_id("0") == 0); + REQUIRE(osmium::string_to_changeset_id("1") == 1); + + REQUIRE_THROWS_AS(osmium::string_to_changeset_id("-1"), std::range_error); + REQUIRE_THROWS_AS(osmium::string_to_changeset_id(""), std::range_error); + REQUIRE_THROWS_AS(osmium::string_to_changeset_id(" "), std::range_error); + REQUIRE_THROWS_AS(osmium::string_to_changeset_id(" 22"), std::range_error); + REQUIRE_THROWS_AS(osmium::string_to_changeset_id("x"), std::range_error); +} + +TEST_CASE("set user id from string") { + REQUIRE(osmium::string_to_user_id("0") == 0); + REQUIRE(osmium::string_to_user_id("1") == 1); + REQUIRE(osmium::string_to_user_id("-1") == -1); + + REQUIRE_THROWS_AS(osmium::string_to_user_id("-2"), std::range_error); + REQUIRE_THROWS_AS(osmium::string_to_user_id(""), std::range_error); + REQUIRE_THROWS_AS(osmium::string_to_user_id(" "), std::range_error); + REQUIRE_THROWS_AS(osmium::string_to_user_id(" 22"), std::range_error); + REQUIRE_THROWS_AS(osmium::string_to_user_id("x"), std::range_error); +} + +TEST_CASE("set num changes from string") { + REQUIRE(osmium::string_to_num_changes("0") == 0); + REQUIRE(osmium::string_to_num_changes("1") == 1); + + REQUIRE_THROWS_AS(osmium::string_to_num_changes("-1"), std::range_error); + REQUIRE_THROWS_AS(osmium::string_to_num_changes(""), std::range_error); + REQUIRE_THROWS_AS(osmium::string_to_num_changes(" "), std::range_error); + REQUIRE_THROWS_AS(osmium::string_to_num_changes(" 22"), std::range_error); + REQUIRE_THROWS_AS(osmium::string_to_num_changes("x"), std::range_error); +} + diff --git a/test/t/basic/test_way.cpp b/test/t/basic/test_way.cpp index 9d2ba0691ef..7c7bc214841 100644 --- a/test/t/basic/test_way.cpp +++ b/test/t/basic/test_way.cpp @@ -1,11 +1,16 @@ #include "catch.hpp" +#include + #include +#include #include #include "helper.hpp" -TEST_CASE("Basic_Way") { +TEST_CASE("Build way") { + + osmium::CRC crc32; SECTION("way_builder") { osmium::memory::Buffer buffer(10000); @@ -38,6 +43,9 @@ SECTION("way_builder") { REQUIRE(3 == way.nodes()[1].ref()); REQUIRE(2 == way.nodes()[2].ref()); REQUIRE(! way.is_closed()); + + crc32.update(way); + REQUIRE(crc32().checksum() == 0x20fe7a30); } SECTION("closed_way") { diff --git a/test/t/buffer/test_buffer_purge.cpp b/test/t/buffer/test_buffer_purge.cpp index 10cdfe72cea..a72db1b5c1e 100644 --- a/test/t/buffer/test_buffer_purge.cpp +++ b/test/t/buffer/test_buffer_purge.cpp @@ -108,7 +108,7 @@ TEST_CASE("Purge data from buffer") { node_builder.object().set_removed(true); } buffer.commit(); - size_t size2 = buffer.committed() - size1; + REQUIRE(std::distance(buffer.begin(), buffer.end()) == 2); CallbackClass callback; @@ -127,20 +127,20 @@ TEST_CASE("Purge data from buffer") { node_builder.add_user("testuser_longer_name"); } buffer.commit(); - size_t size1 = buffer.committed(); + { osmium::builder::NodeBuilder node_builder(buffer); node_builder.add_user("testuser"); node_builder.object().set_removed(true); } buffer.commit(); - size_t size2 = buffer.committed() - size1; + { osmium::builder::NodeBuilder node_builder(buffer); node_builder.add_user("sn"); } buffer.commit(); - size_t size3 = buffer.committed() - (size1 + size2); + REQUIRE(std::distance(buffer.begin(), buffer.end()) == 3); CallbackClass callback; @@ -159,21 +159,21 @@ TEST_CASE("Purge data from buffer") { node_builder.object().set_removed(true); } buffer.commit(); - size_t size1 = buffer.committed(); + { osmium::builder::NodeBuilder node_builder(buffer); node_builder.add_user("testuser"); node_builder.object().set_removed(true); } buffer.commit(); - size_t size2 = buffer.committed() - size1; + { osmium::builder::NodeBuilder node_builder(buffer); node_builder.add_user("sn"); node_builder.object().set_removed(true); } buffer.commit(); - size_t size3 = buffer.committed() - (size1 + size2); + REQUIRE(std::distance(buffer.begin(), buffer.end()) == 3); CallbackClass callback; diff --git a/test/t/geom/test_exception.cpp b/test/t/geom/test_exception.cpp new file mode 100644 index 00000000000..fe950434b54 --- /dev/null +++ b/test/t/geom/test_exception.cpp @@ -0,0 +1,16 @@ +#include "catch.hpp" + +#include + +#include + +TEST_CASE("Geometry exception") { + + SECTION("geometry_error") { + osmium::geometry_error e("some error message", "node", 17); + REQUIRE(e.id() == 17); + REQUIRE(std::string(e.what()) == "some error message (node_id=17)"); + } + +} + diff --git a/test/t/geom/test_geos.cpp b/test/t/geom/test_geos.cpp index e93228b4cc7..d849e3b1603 100644 --- a/test/t/geom/test_geos.cpp +++ b/test/t/geom/test_geos.cpp @@ -5,9 +5,7 @@ #include "../basic/helper.hpp" -TEST_CASE("GEOS_Geometry") { - -SECTION("point") { +TEST_CASE("GEOS geometry factory - create point") { osmium::geom::GEOSFactory<> factory; std::unique_ptr point {factory.create_point(osmium::Location(3.2, 4.2))}; @@ -16,7 +14,7 @@ SECTION("point") { REQUIRE(-1 == point->getSRID()); } -SECTION("non_default_srid") { +TEST_CASE("GEOS geometry factory - create point with non-default srid") { osmium::geom::GEOSFactory<> factory(4326); std::unique_ptr point {factory.create_point(osmium::Location(3.2, 4.2))}; @@ -25,13 +23,23 @@ SECTION("non_default_srid") { REQUIRE(4326 == point->getSRID()); } -SECTION("empty_point") { +TEST_CASE("GEOS geometry factory - create point with externally created GEOS factory") { + geos::geom::GeometryFactory geos_factory; + osmium::geom::GEOSFactory<> factory(geos_factory); + + std::unique_ptr point {factory.create_point(osmium::Location(3.2, 4.2))}; + REQUIRE(3.2 == point->getX()); + REQUIRE(4.2 == point->getY()); + REQUIRE(0 == point->getSRID()); +} + +TEST_CASE("GEOS geometry factory - can not create from invalid location") { osmium::geom::GEOSFactory<> factory; REQUIRE_THROWS_AS(factory.create_point(osmium::Location()), osmium::invalid_location); } -SECTION("linestring") { +TEST_CASE("GEOS geometry factory - create linestring") { osmium::geom::GEOSFactory<> factory; osmium::memory::Buffer buffer(10000); @@ -42,7 +50,7 @@ SECTION("linestring") { {2, {3.6, 4.9}} }); - { + SECTION("from way node list") { std::unique_ptr linestring {factory.create_linestring(wnl)}; REQUIRE(3 == linestring->getNumPoints()); @@ -52,7 +60,7 @@ SECTION("linestring") { REQUIRE(3.6 == p2->getX()); } - { + SECTION("without duplicates and backwards") { std::unique_ptr linestring {factory.create_linestring(wnl, osmium::geom::use_nodes::unique, osmium::geom::direction::backward)}; REQUIRE(3 == linestring->getNumPoints()); std::unique_ptr p0 = std::unique_ptr(linestring->getPointN(0)); @@ -61,14 +69,14 @@ SECTION("linestring") { REQUIRE(3.2 == p2->getX()); } - { + SECTION("with duplicates") { std::unique_ptr linestring {factory.create_linestring(wnl, osmium::geom::use_nodes::all)}; REQUIRE(4 == linestring->getNumPoints()); std::unique_ptr p0 = std::unique_ptr(linestring->getPointN(0)); REQUIRE(3.2 == p0->getX()); } - { + SECTION("with duplicates and backwards") { std::unique_ptr linestring {factory.create_linestring(wnl, osmium::geom::use_nodes::all, osmium::geom::direction::backward)}; REQUIRE(4 == linestring->getNumPoints()); std::unique_ptr p0 = std::unique_ptr(linestring->getPointN(0)); @@ -76,7 +84,7 @@ SECTION("linestring") { } } -SECTION("area_1outer_0inner") { +TEST_CASE("GEOS geometry factory - create area with one outer and no inner rings") { osmium::geom::GEOSFactory<> factory; osmium::memory::Buffer buffer(10000); @@ -105,7 +113,7 @@ SECTION("area_1outer_0inner") { REQUIRE(3.5 == l0e_p0->getX()); } -SECTION("area_1outer_1inner") { +TEST_CASE("GEOS geometry factory - create area with one outer and one inner ring") { osmium::geom::GEOSFactory<> factory; osmium::memory::Buffer buffer(10000); @@ -142,7 +150,7 @@ SECTION("area_1outer_1inner") { REQUIRE(5 == l0i0->getNumPoints()); } -SECTION("area_2outer_2inner") { +TEST_CASE("GEOS geometry factory - create area with two outer and two inner rings") { osmium::geom::GEOSFactory<> factory; osmium::memory::Buffer buffer(10000); @@ -195,4 +203,3 @@ SECTION("area_2outer_2inner") { REQUIRE(5 == l1e->getNumPoints()); } -} diff --git a/test/t/geom/test_projection.cpp b/test/t/geom/test_projection.cpp index 2257d7f0669..58854109e42 100644 --- a/test/t/geom/test_projection.cpp +++ b/test/t/geom/test_projection.cpp @@ -1,5 +1,7 @@ #include "catch.hpp" +#include + #include #include #include @@ -128,4 +130,20 @@ SECTION("compare_mercators") { } } +SECTION("compare_mercators") { + osmium::geom::MercatorProjection projection_merc; + osmium::geom::Projection projection_3857(3857); + + std::random_device rd; + std::mt19937 gen(rd()); + std::uniform_real_distribution<> dis_x(-180.0, 180.0); + std::uniform_real_distribution<> dis_y(-90.0, 90.0); + + for (int n = 0; n < 100000; ++n) { + const osmium::Location loc(dis_x(gen), dis_y(gen)); + REQUIRE(projection_merc(loc).x == Approx(projection_3857(loc).x).epsilon(0.1)); + REQUIRE(projection_merc(loc).y == Approx(projection_3857(loc).y).epsilon(0.1)); + } +} + } diff --git a/test/t/geom/test_tile.cpp b/test/t/geom/test_tile.cpp new file mode 100644 index 00000000000..e80cb960410 --- /dev/null +++ b/test/t/geom/test_tile.cpp @@ -0,0 +1,93 @@ +#include "catch.hpp" + +#include + +#include + +#include "helper.hpp" + +#include "test_tile_data.hpp" + +TEST_CASE("Tile") { + + SECTION("x0.0 y0.0 zoom 0") { + osmium::Location l(0.0, 0.0); + + osmium::geom::Tile t(0, l); + + REQUIRE(t.x == 0); + REQUIRE(t.y == 0); + REQUIRE(t.z == 0); + } + + SECTION("x180.0 y90.0 zoom 0") { + osmium::Location l(180.0, 90.0); + + osmium::geom::Tile t(0, l); + + REQUIRE(t.x == 0); + REQUIRE(t.y == 0); + REQUIRE(t.z == 0); + } + + SECTION("x180.0 y90.0 zoom 4") { + osmium::Location l(180.0, 90.0); + + osmium::geom::Tile t(4, l); + + REQUIRE(t.x == (1 << 4) - 1); + REQUIRE(t.y == 0); + REQUIRE(t.z == 4); + } + + SECTION("x0.0 y0.0 zoom 4") { + osmium::Location l(0.0, 0.0); + + osmium::geom::Tile t(4, l); + + auto n = 1 << (4-1); + REQUIRE(t.x == n); + REQUIRE(t.y == n); + REQUIRE(t.z == 4); + } + + SECTION("equality") { + osmium::geom::Tile a(4, 3, 4); + osmium::geom::Tile b(4, 3, 4); + osmium::geom::Tile c(4, 4, 3); + REQUIRE(a == b); + REQUIRE(a != c); + REQUIRE(b != c); + } + + SECTION("order") { + osmium::geom::Tile a(2, 3, 4); + osmium::geom::Tile b(4, 3, 4); + osmium::geom::Tile c(4, 4, 3); + osmium::geom::Tile d(4, 4, 2); + REQUIRE(a < b); + REQUIRE(a < c); + REQUIRE(b < c); + REQUIRE(d < c); + } + + SECTION("tilelist") { + std::istringstream input_data(s); + while (input_data) { + double lon, lat; + uint32_t x, y, zoom; + input_data >> lon; + input_data >> lat; + input_data >> x; + input_data >> y; + input_data >> zoom; + + osmium::Location l(lon, lat); + osmium::geom::Tile t(zoom, l); + REQUIRE(t.x == x); + REQUIRE(t.y == y); + } + } + +} + diff --git a/test/t/geom/test_tile_data.hpp b/test/t/geom/test_tile_data.hpp new file mode 100644 index 00000000000..e5c0953c050 --- /dev/null +++ b/test/t/geom/test_tile_data.hpp @@ -0,0 +1,475 @@ + +std::string s = R"(127.4864358 16.8380041 223904 118630 18 +163.1103174 39.4760232 121 48 7 +-4.1372725 -22.5105386 31 36 6 +98.7193066 -36.2312406 1 1 1 +63.5773661 -13.47636 21 17 5 +-88.4518148 37.9805485 260 395 10 +-14.5903133 -28.2989812 3763 4767 13 +-13.4971239 -34.4080035 14 19 5 +-169.156223 -64.0900356 3 93 7 +27.1473191 -4.1993125 4713 4191 13 +-160.9733129 54.3684314 13 81 8 +129.0194139 14.2576156 439 235 9 +-69.5085993 -56.8253221 10057 22700 15 +-77.8387486 18.1961517 1162 1837 12 +-76.2695325 -18.2494296 147 282 9 +-91.594905 7.6698071 1 3 3 +-116.7926741 -20.6060813 179 571 10 +109.0552776 -1.9947569 52620 33131 16 +-156.1846426 -79.3817554 33 449 9 +-95.3403755 -27.8978407 1 4 3 +-55.1827573 -73.2293796 44 103 7 +-108.5207885 -48.0099293 50 167 8 +23.7540108 -15.3395164 9273 8898 14 +-155.6662842 -68.3295899 0 0 0 +75.8139119 30.9914252 363 209 9 +-135.8034544 64.7242469 0 1 2 +-48.743352 70.9392876 23 13 6 +-38.6968026 7.7867812 0 0 0 +-18.5234838 11.8557704 29395 30594 16 +-152.5632568 19.4069834 78 455 10 +-63.2089431 -80.5909713 0 0 0 +-94.1255611 -81.2028822 244 930 10 +175.0862205 -33.0865142 3 2 2 +-179.6241926 -37.0256609 1 625 10 +135.6783824 -38.6011643 459739 323170 19 +-139.6407533 -83.2495209 0 7 3 +-14.1336447 -56.5465949 58 88 7 +-30.7414568 -32.9543731 26 38 6 +10.3306861 73.2444693 1082 399 11 +-84.8379263 29.2363222 16 26 6 +-94.0685822 -39.5503996 7821 20309 15 +-86.5944356 -41.7491891 265 642 10 +11.9182172 -80.5613703 34937 58784 16 +91.8773752 -32.1741317 0 0 0 +126.2879157 20.5759564 1742 904 11 +-160.7743029 -47.3192128 27999 340565 19 +-4.2449045 -50.3288332 31 42 6 +-66.6857158 61.4380757 10 9 5 +169.7372317 -74.3365704 3 3 2 +87.4815328 75.6218888 95 21 7 +60.3544927 28.3165267 0 0 0 +-48.9614619 -59.3292497 2 5 3 +-123.2935018 -59.5454886 80 362 9 +-31.5909316 -14.8985476 13 17 5 +58.1862173 59.0957666 2710 1209 12 +-72.8881665 -2.2648849 1218 2073 12 +-33.7267461 8.6006817 106512 124785 18 +175.723181 46.4929742 7 2 3 +-127.1963326 76.0328786 0 0 0 +-161.7444367 -26.7634497 13293 151310 18 +-163.976298 -8.1169845 91 1070 11 +63.7757109 5.6049418 2 1 2 +12.7012434 22.1157713 70160 57276 17 +19.5832849 -25.0284049 1135 1171 11 +-22.6619642 54.8831276 111 81 8 +78.7736907 24.0919863 5888 3530 13 +121.9003045 -64.6256685 107 94 7 +-64.8766793 61.6655916 81 71 8 +113.0498445 -70.0016518 416 397 9 +-51.5116259 68.1532424 46781 31217 17 +-89.9005747 87.9523248 512 0 11 +59.2536822 -75.1520493 10 13 4 +17.375372 74.9259262 287448 93371 19 +75.6426945 8.4808632 186153 124873 18 +144.89589 75.1647661 14786 2875 14 +174.4350898 -29.268169 258091 153376 18 +-91.8773113 50.6182166 0 0 0 +5.3822308 45.1391794 134991 94156 18 +43.0978373 -10.0764237 324909 276895 19 +55.6682917 -40.9015342 21451 20470 15 +-37.584032 52.253723 6 5 4 +131.0141997 29.0271798 28309 13621 15 +69.8830721 -86.8548645 363918 524287 19 +-107.7917548 -76.9626654 26290 110787 17 +-57.1736642 49.9345991 2 2 3 +165.5170226 -84.9735363 982 1021 10 +-139.7208984 -73.8754873 1 12 4 +-154.5959463 -10.596718 4 33 6 +-106.4164918 36.5042686 3348 6405 14 +-92.9688259 -11.2262505 0 2 2 +138.2722283 8.1109779 7 3 3 +123.1389319 -40.3505677 862 637 10 +-171.4780435 24.9112482 0 13 5 +89.3668763 -63.1809434 392293 381781 19 +-79.6906176 -13.376312 9130 17612 15 +133.1329522 -28.9032634 445 298 9 +115.0369496 70.6965823 6 1 3 +-74.8926697 -78.9770185 2391 7144 13 +51.9175766 73.0184277 164 50 8 +178.1319784 43.7111421 509 186 9 +162.4098389 61.1595443 0 0 0 +7.6241126 -75.6182944 2 3 2 +172.8902767 28.5068027 125 53 7 +156.4030739 -76.7309238 478 431 9 +-131.3963189 62.2039684 1 2 3 +34.6057088 -36.2933264 2441 2491 12 +-3.3896413 -48.2734347 15 20 5 +83.528842 -48.219886 2998 2675 12 +16.6000512 -31.6322244 17894 19421 15 +-18.2425749 21.0749052 14 14 5 +35.6035336 46.9916228 78498 46105 17 +-109.3453091 -34.2312523 12 38 6 +88.4909962 58.6104749 47 19 6 +-129.7372783 21.7408241 571 1794 12 +25.7269392 -40.1240139 1 1 1 +26.3829579 5.248998 37570 31811 16 +141.1768247 -41.4653038 14617 10269 14 +151.3788752 -38.0160512 241302 161042 18 +-92.7471483 -35.391745 15883 39664 16 +142.5902623 -14.1023743 28 17 5 +179.3461485 -86.7573357 3 3 2 +-40.9746194 61.5689546 790 576 11 +128.6399735 -54.8895009 56186 44772 16 +-141.391583 -63.8272 0 2 2 +-3.5004218 19.3089998 4016 3648 13 +16.138208 -42.7868627 1 1 1 +78.502315 -18.91667 2 2 2 +-44.6311826 -15.7483106 98572 142686 18 +-120.431941 -39.3431529 0 0 0 +-70.4915024 25.4763412 9967 13984 15 +118.0711114 -66.5691073 211 192 8 +-114.945538 38.1764389 0 0 0 +142.991315 -46.3378741 14699 10577 14 +34.2770562 -84.7173756 312063 518834 19 +109.5563415 -85.9138266 105424 131071 17 +-171.8032643 70.1948223 0 3 4 +-90.8434294 89.7252122 126 0 9 +163.5677082 -53.9885419 3 2 2 +128.884016 -63.0732584 112461 95358 17 +116.5348575 -56.1616143 843 705 10 +-171.5770602 37.9225943 11 197 9 +48.1396653 -49.3932234 5191 5392 13 +-157.6786731 15.3895763 32507 239456 19 +15.7385145 -37.1364397 1113 1251 11 +137.5179466 61.2025671 1 0 1 +-70.4767722 42.9798424 19938 24086 16 +27.8867901 43.4642562 9461 5991 14 +68.5677462 -84.9560937 11312 16334 14 +-56.790151 -67.124147 179437 395476 19 +108.850832 43.3493384 26291 11996 15 +-139.0655153 8.2673118 1 7 4 +59.3726661 83.3857699 21788 1515 15 +-158.4718709 -12.2313178 1 17 5 +30.358316 -83.020501 2393 3871 12 +-169.9667863 71.1152107 1 13 6 +-89.9418297 -7.7258969 131156 273429 19 +-16.5050312 47.1843923 116 89 8 +-151.9641886 -36.3579042 5103 39881 16 +169.7406916 60.0950674 62 18 6 +32.6694543 -1.8435981 2 2 2 +97.7880811 77.5295169 50569 9674 16 +87.7554889 -85.6741368 389947 524287 19 +-19.0174909 29.1940122 0 0 1 +-86.6180019 82.6445919 1 0 2 +-50.1997802 -21.1913243 1476 2294 12 +73.8710121 75.2201385 5 1 3 +41.7434624 79.1410045 322937 65770 19 +-122.4312562 -68.4513916 10 48 6 +-54.6116448 -23.1761137 1 2 2 +-35.8774263 -51.5317442 102 170 8 +179.2976644 -76.2729885 127 107 7 +-111.6934402 -85.2696836 1 7 3 +-24.2137947 28.4102025 3 3 3 +-168.0816395 -64.0317696 16 375 9 +43.2514876 6.4266793 2540 1974 12 +25.5877932 49.3486828 4 2 3 +-133.3671657 -25.0795973 8 36 6 +-140.2002274 -37.3713396 452 2507 12 +73.6326557 -21.9898207 360 288 9 +-83.0901448 -69.1893461 2205 6305 13 +-32.83227 -11.1061854 3348 4350 13 +-151.8897244 14.6891958 0 0 0 +-118.1125151 82.3085937 704 288 12 +119.9903922 1.4884267 53 31 6 +-116.9455865 -48.1971821 0 2 2 +111.4279587 -52.6024326 828 688 10 +-78.9207774 28.6183104 147207 218615 19 +45.1367631 24.1416251 40 27 6 +115.0905685 7.2971256 6714 3929 13 +-58.0837371 -55.4033522 43 87 7 +-44.6785779 83.0751777 6158 877 14 +80.2002966 84.3087089 2960 91 12 +-167.3118039 -59.2698371 72 1445 11 +139.9974902 -74.5994693 455 419 9 +-27.5858357 -36.1890547 6936 9960 14 +68.8572424 20.2096749 353 226 9 +-168.7172825 69.1607462 2053 15104 16 +62.1361934 74.6007777 44079 11896 16 +-102.9645124 65.3735325 112191 135144 19 +178.8434887 18.6231394 65325 29316 16 +-138.0494072 -80.3786289 29 228 8 +33.3858934 62.9029909 155382 71699 18 +-1.6915643 74.737228 1 0 2 +18.2680756 6.6441482 1 0 1 +34.0511738 -28.6862516 2 2 2 +-24.0705994 -0.6026568 28386 32877 16 +-176.8765092 -14.3252022 568 35403 16 +154.6553417 -79.7390666 1903 1809 11 +151.6055263 59.552346 14 4 4 +-101.7681729 45.3498263 113933 187876 19 +-52.3703658 -89.8260406 181 511 9 +-85.7937259 16.5632413 2143 3713 13 +114.7030999 6.6846014 104 61 7 +13.6027861 88.6863877 2 0 2 +60.5726928 38.256536 43794 25219 16 +141.0903381 -35.4468007 7306 4959 13 +-38.8182454 -55.2332792 200 350 9 +179.7818018 -79.6600052 523970 462627 19 +-45.2102175 82.7381225 196301 32058 19 +-36.5811826 -35.3991346 0 0 0 +-26.7719575 -75.7014102 223154 435372 19 +77.3451937 -65.4613594 1 1 1 +-37.3286225 38.3250599 51945 50407 17 +70.8235495 35.3870419 365288 206979 19 +99.8381373 -83.4101655 1 1 1 +25.8851923 67.415543 9370 3991 14 +-12.3393758 68.5972027 7 3 4 +-142.9400273 -36.2904852 1 9 4 +110.085689 -50.9514972 422467 348655 19 +121.5093657 53.9939447 54888 21044 16 +151.1307841 -33.0798435 58 38 6 +100.4346499 -57.0129759 6 5 3 +114.0564882 63.4987339 428250 141474 19 +-17.3978586 19.2981593 57 57 7 +-129.101039 -66.3423574 579 3067 12 +117.6639917 14.0568892 433504 241463 19 +97.7014577 69.1261732 202216 60490 18 +-174.7931333 81.4174709 7583 46045 19 +47.205341 -48.8608019 20680 21495 15 +167.416796 -7.5063098 247 133 8 +63.6744589 21.3392888 0 0 0 +-106.0719817 33.8353547 53832 104862 18 +-48.4051268 27.0438152 191648 221209 19 +64.5675545 -27.0683253 5 4 3 +110.7800249 -35.8217841 0 0 0 +164.8954138 81.9558801 31393 2538 15 +-54.4595565 -23.0128432 357 579 10 +43.2542709 67.694011 2540 989 12 +137.6537233 -78.7052092 28913 28450 15 +76.9271563 -60.0777348 0 0 0 +-145.0240216 47.3869213 6367 22947 16 +136.2328853 -73.4534547 3598 3304 12 +48.8047148 74.3001023 81 23 7 +-144.6978756 33.6270048 200 820 11 +64.6723407 -37.196427 21 19 5 +-58.3356482 -19.0812366 22148 36307 16 +19.611535 -31.3013119 0 0 0 +-63.8923439 -25.9560287 1 2 2 +-152.5288047 -30.9747477 0 1 1 +-99.8509683 -69.0258965 7295 25181 15 +-22.9015207 -21.0915082 13 17 5 +161.4402438 61.1430241 15539 4652 14 +-102.9499976 -1.2629414 1 4 3 +-159.7995801 51.1570142 14 85 8 +10.5265485 -86.8526122 67 127 7 +47.1581041 -73.1628608 646 823 10 +2.7577906 -66.3257104 66540 98133 17 +96.9653593 -9.8258675 6 4 3 +-55.4199846 25.3325116 2834 3499 13 +-47.3590106 59.5085133 2 2 3 +179.1441501 22.1324479 7 3 3 +16.2649007 47.8480398 4 2 3 +-27.8373963 -0.3926597 27700 32839 16 +-99.7938802 -48.8013068 912 2685 12 +133.4858024 28.457146 3 1 2 +-174.7621207 -25.8318084 238 9409 14 +106.856378 19.640472 203 113 8 +31.4361995 -80.2488406 76981 116886 17 +148.6539554 -70.0347611 116 99 7 +45.1821773 17.5961595 80 57 7 +-30.3630114 71.3238998 54481 27877 17 +106.547704 44.9731358 6520 2947 13 +-132.8057564 23.3928795 67 221 9 +116.6455816 -10.1097592 52 33 6 +172.5512559 -53.2307289 3 2 2 +-114.2084519 42.834332 2994 6030 14 +91.0172087 87.6025 1 0 1 +-101.666812 -84.7858729 7130 32495 15 +-14.8074931 68.3277393 0 0 0 +140.3354277 -7.8709618 14 8 4 +-130.4961987 -69.4777523 9011 50594 16 +-121.239479 1.0896367 1 3 3 +-141.2241052 3.4495348 56471 257117 19 +42.6041377 -19.1328846 2532 2269 12 +141.3750885 -1.8110503 468036 264781 19 +-26.545021 -30.9641274 218 302 9 +87.6154866 10.1978751 5 3 3 +88.002559 -33.4108714 762 612 10 +170.9910642 -7.1925019 3993 2130 12 +-66.5680487 -88.1144993 165197 524287 19 +-71.5925378 45.720316 0 0 0 +-16.457642 57.1688038 930 625 11 +20.8379624 -11.1485568 35 33 6 +-72.9399538 -16.6504502 1 2 2 +52.8746845 -71.5213577 10598 12927 14 +93.2392918 12.5073156 48 29 6 +-3.7803834 73.7748237 128319 49794 18 +93.5713795 -38.8308882 24 19 5 +87.3474619 -6.9970464 23 16 5 +161.2199467 48.5612779 30 11 5 +87.343969 -31.3153618 47 37 6 +60.5318627 58.0869948 42 19 6 +161.4388458 42.7612385 1 0 1 +14.1262999 -26.2101142 4 4 3 +-135.776346 -21.4248586 503 2297 12 +-100.3980364 82.0236388 1 0 3 +-55.8093833 -87.2455194 0 0 0 +-15.6397352 64.454076 935 540 11 +131.5974423 39.1251372 110 48 7 +56.8045509 27.5658629 168 107 8 +146.3634996 14.9287416 3 1 2 +51.5699041 74.0370231 82 23 7 +29.3100901 33.7208834 152414 104963 18 +19.0622442 1.6781772 141 126 8 +-22.2575472 73.6467471 114864 50126 18 +125.0208495 47.8533914 3470 1426 12 +-92.4804503 29.8409387 995 1691 12 +113.6362202 86.1158625 0 0 0 +113.6203756 28.4267778 26 13 5 +124.472202 25.8347062 13856 6974 14 +79.3654306 -33.7864728 46 38 6 +-121.169233 -66.7642843 2 12 4 +179.5816311 1.0182064 0 0 0 +179.1123794 53.367624 0 0 0 +-25.6611689 32.6575586 14048 13236 15 +92.3370554 58.8306651 12 4 4 +142.2756448 63.6603101 3 1 2 +-162.8914282 -84.54406 6229 129034 17 +-54.0759935 53.1086102 0 0 0 +9.1647045 -39.4876873 34436 40604 16 +59.3708822 -43.9789068 10894 10425 14 +-16.6397572 46.5985252 929 723 11 +-36.1053642 -67.6882841 0 1 1 +-82.5851985 -69.5277766 4 12 4 +117.0059969 -63.2697116 432546 382068 19 +127.9740251 85.5341901 112129 0 17 +-13.9255265 59.7720207 120931 76457 18 +-167.7778382 41.7974194 69 761 11 +-100.1217856 -70.5624949 454 1599 11 +-42.4790343 -2.0016191 25034 33132 16 +-8.0782317 65.9680074 30 16 6 +-37.0481466 -55.2856125 203 350 9 +149.074382 16.7121888 58 28 6 +-53.1010518 -15.8492068 0 0 0 +50.8517919 -30.8366257 42025 38674 16 +-175.0355994 77.1929422 0 4 5 +-33.3221217 72.5636809 6 3 4 +-139.9201802 52.3400823 7296 21546 16 +86.0299659 23.4535286 1513 886 11 +105.6297238 -50.5521342 25998 21733 15 +104.2700533 36.5417507 206999 102450 18 +58.5238253 84.0320186 86843 3912 17 +-121.9177826 65.4393267 1321 2108 13 +-152.689211 57.4112922 39774 159515 19 +94.2583831 80.8625455 6240 801 13 +118.8199874 -37.6116567 53 39 6 +5.787144 7.0599251 4227 3934 13 +152.1682213 11.778732 236 119 8 +37.4503636 64.311498 2474 1084 12 +35.2616139 -59.8196291 38 45 6 +-20.2808572 -26.9983008 29075 37875 16 +-88.8541607 -20.4896703 66370 146320 18 +125.7726709 41.2603793 27 11 5 +-55.4322696 22.1767236 11338 14313 15 +-38.2393439 -56.4681037 6 11 4 +-139.4517643 -81.3621632 3 29 5 +-146.7954207 -74.6050826 6044 53642 16 +161.6654898 -71.7313805 15549 12957 14 +-85.4720514 -73.177675 2 6 3 +-25.7457381 -42.9842376 13 20 5 +-2.0766311 51.0142455 0 0 1 +-82.9179334 1.4865326 8836 16248 15 +140.5661302 61.5056993 28 9 5 +-30.2614099 35.5786378 54518 51659 17 +-49.2072142 -38.6965571 0 1 1 +124.6587534 9.5039576 433 242 9 +-113.4516741 81.4585593 24229 11410 17 +-4.5134723 68.229217 3 1 3 +75.4838529 -44.997406 2906 2622 12 +6.4715239 28.6900832 8486 6828 14 +91.9187555 -24.8192643 6187 4679 13 +-28.2510181 -42.6238777 26 40 6 +-151.1042237 -21.4553189 2630 18384 15 +-149.6272551 0.4235911 1382 8172 14 +-51.9171488 74.630423 0 0 0 +-36.5660117 37.242858 101 99 8 +91.7136677 -30.9077617 772 604 10 +61.6846009 -85.9378164 5 7 3 +11.3772008 69.40183 34839 14980 16 +82.6825938 9.4496443 11954 7759 14 +61.8446231 -40.0114106 2751 2545 12 +-51.6223196 -11.7880324 5 8 4 +113.0309076 -73.4376173 13336 13217 14 +-169.3808275 -72.7209175 0 25 5 +-98.5653414 -80.0893122 231 910 10 +-20.4653707 29.9801495 3 3 3 +78.9156686 2.599349 0 0 0 +76.0635255 -71.0823347 2913 3216 12 +-26.1185551 22.3616029 28013 28589 16 +177.8803853 -56.3662368 4071 2828 12 +-157.7926463 78.4998022 15 34 8 +168.6834344 -34.5535211 7934 4934 13 +-77.208013 -44.0964079 0 1 1 +-56.6162078 28.1240365 10 13 5 +8.6548899 72.178831 137374 53767 18 +-27.9342497 8.2525327 1 1 2 +91.6356971 -13.5230128 6181 4406 13 +-161.9980398 -75.4443511 0 13 4 +46.8556576 -27.1078679 5162 4737 13 +147.2806954 -48.1491071 465 334 9 +-168.2679875 -29.0171568 0 2 2 +10.0251187 -3.144812 8 8 4 +109.0281873 81.9713348 26307 2528 15 +-129.6281276 -36.9614028 73359 320148 19 +7.3831244 -18.3270273 2132 2260 12 +-34.4625217 53.2837646 52988 42524 17 +129.8855275 -26.30807 3525 2358 12 +-69.6374195 -45.7769025 10045 21081 15 +59.9868336 50.3716565 43688 22120 16 +126.4653338 -75.607391 1743 1698 11 +-34.6459616 53.2502443 51 41 7 +152.2415169 -71.7528837 1 1 1 +-83.2126752 32.6685119 35239 52939 17 +178.6388805 70.5727365 31 7 5 +-153.5646223 -81.9491561 2 29 5 +178.2668159 11.8222247 0 0 0 +-27.136902 18.4287365 6 7 4 +-104.7744923 -64.7769211 54777 193540 18 +58.2318889 9.5897974 2710 1938 12 +138.9707487 -65.2734433 14516 12149 14 +-115.29331 -84.3402223 2944 16033 14 +-66.3487123 -81.8340211 20 58 6 +76.1802855 40.4007156 364 193 9 +-77.5026834 -30.653231 145 301 9 +-52.6095007 -79.7155889 11595 28942 15 +-2.5594899 -31.6276806 7 9 4 +-58.6267217 41.4851391 2 2 3 +-0.7245205 -70.7329433 509 801 10 +161.2822996 -79.0311957 124257 114418 17 +144.8720197 80.1059269 14785 1811 14 +159.3764075 -23.5100054 0 0 0 +82.3109493 -7.0673699 95504 68115 17 +94.3931949 63.0474034 97 34 7 +87.4523391 -73.2690527 3043 3297 12 +101.4256455 -0.8267694 12 8 4 +-112.7666152 -82.3670281 12239 61007 16 +-82.0948447 -38.0810449 8 19 5 +113.2906728 -19.193243 104 70 7 +16.953131 78.402684 35 8 6 +-81.7974685 -53.9596787 34 86 7 +69.8051889 58.1416894 181902 78760 18 +-9.2435464 -53.4296594 3 5 3 +30.0415066 11.4884737 4 3 3 +125.2704157 -69.6324197 54 49 6 +-41.2060435 63.8501787 789 548 11 +120.407662 -4.9889504 6835 4209 13 +92.0345558 -8.0809262 24761 17121 15 +127.1061722 83.1311204 6988 428 13 +-178.9414629 64.0086678 770 69897 18 +49.0743035 -4.3000419 10425 8387 14 +-45.109002 -55.1435498 1534 2803 12 +3.2795498 75.95918 32 10 6 +)"; + diff --git a/test/t/geom/test_wkt.cpp b/test/t/geom/test_wkt.cpp index ff1417c72e9..53830740938 100644 --- a/test/t/geom/test_wkt.cpp +++ b/test/t/geom/test_wkt.cpp @@ -74,6 +74,14 @@ SECTION("linestring_with_two_same_locations") { }); REQUIRE_THROWS_AS(factory.create_linestring(wnl), osmium::geometry_error); + + try { + factory.create_linestring(wnl); + } catch (osmium::geometry_error& e) { + REQUIRE(e.id() == 0); + REQUIRE(std::string(e.what()) == "need at least two points for linestring"); + } + REQUIRE_THROWS_AS(factory.create_linestring(wnl, osmium::geom::use_nodes::unique, osmium::geom::direction::backward), osmium::geometry_error); { diff --git a/test/t/index/test_typed_mmap.cpp b/test/t/index/test_typed_mmap.cpp deleted file mode 100644 index bcc17bd2d24..00000000000 --- a/test/t/index/test_typed_mmap.cpp +++ /dev/null @@ -1,76 +0,0 @@ -#include "catch.hpp" - -#include - -#if defined(_MSC_VER) || (defined(__GNUC__) && defined(_WIN32)) -#include "win_mkstemp.hpp" -#endif - -TEST_CASE("TypedMmap") { - - SECTION("Mmap") { - uint64_t* data = osmium::detail::typed_mmap::map(10); - - data[0] = 4ul; - data[3] = 9ul; - data[9] = 25ul; - - REQUIRE(4ul == data[0]); - REQUIRE(9ul == data[3]); - REQUIRE(25ul == data[9]); - - osmium::detail::typed_mmap::unmap(data, 10); - } - - SECTION("MmapSizeZero") { - REQUIRE_THROWS_AS(osmium::detail::typed_mmap::map(0), std::system_error); - } - - SECTION("MmapHugeSize") { - // this is a horrible hack to only run the test on 64bit machines. - if (sizeof(size_t) >= 8) { - REQUIRE_THROWS_AS(osmium::detail::typed_mmap::map(1ULL << (sizeof(size_t) * 6)), std::system_error); - } - } - -#ifdef __linux__ - SECTION("Remap") { - uint64_t* data = osmium::detail::typed_mmap::map(10); - - data[0] = 4ul; - data[3] = 9ul; - data[9] = 25ul; - - uint64_t* new_data = osmium::detail::typed_mmap::remap(data, 10, 1000); - - REQUIRE(4ul == new_data[0]); - REQUIRE(9ul == new_data[3]); - REQUIRE(25ul == new_data[9]); - } -#else -# pragma message("not running 'Remap' test case on this machine") -#endif - - SECTION("FileSize") { - const int size = 100; - char filename[] = "test_mmap_file_size_XXXXXX"; - const int fd = mkstemp(filename); - REQUIRE(fd > 0); - REQUIRE(0 == osmium::detail::typed_mmap::file_size(fd)); - REQUIRE(0 == ftruncate(fd, size * sizeof(uint64_t))); - REQUIRE(size == osmium::detail::typed_mmap::file_size(fd)); - - osmium::detail::typed_mmap::grow_file(size / 2, fd); - REQUIRE(size == osmium::detail::typed_mmap::file_size(fd)); - - osmium::detail::typed_mmap::grow_file(size, fd); - REQUIRE(size == osmium::detail::typed_mmap::file_size(fd)); - - osmium::detail::typed_mmap::grow_file(size * 2, fd); - REQUIRE((size * 2) == osmium::detail::typed_mmap::file_size(fd)); - - REQUIRE(0 == close(fd)); - REQUIRE(0 == unlink(filename)); - } - -} diff --git a/test/t/index/test_typed_mmap_grow.cpp b/test/t/index/test_typed_mmap_grow.cpp deleted file mode 100644 index 92ee0b40ea4..00000000000 --- a/test/t/index/test_typed_mmap_grow.cpp +++ /dev/null @@ -1,34 +0,0 @@ -#include "catch.hpp" - -#include - -#if defined(_MSC_VER) || (defined(__GNUC__) && defined(_WIN32)) -#include "win_mkstemp.hpp" -#endif - -TEST_CASE("TypedMmapGrow") { - - SECTION("GrowAndMap") { - const int size = 100; - char filename[] = "test_mmap_grow_and_map_XXXXXX"; - const int fd = mkstemp(filename); - REQUIRE(fd > 0); - - uint64_t* data = osmium::detail::typed_mmap::grow_and_map(size, fd); - REQUIRE(size == osmium::detail::typed_mmap::file_size(fd)); - - data[0] = 1ul; - data[1] = 8ul; - data[99] = 27ul; - - REQUIRE(1ul == data[0]); - REQUIRE(8ul == data[1]); - REQUIRE(27ul == data[99]); - - osmium::detail::typed_mmap::unmap(data, size); - - REQUIRE(0 == close(fd)); - REQUIRE(0 == unlink(filename)); - } - -} diff --git a/test/t/io/test_file_formats.cpp b/test/t/io/test_file_formats.cpp index e8785d6a411..f0ba0c66f9d 100644 --- a/test/t/io/test_file_formats.cpp +++ b/test/t/io/test_file_formats.cpp @@ -247,5 +247,29 @@ TEST_CASE("FileFormats") { REQUIRE_THROWS_AS(f.check(), std::runtime_error); } + SECTION("url without format") { + osmium::io::File f {"http://www.example.com/api"}; + REQUIRE(osmium::io::file_format::xml == f.format()); + REQUIRE(osmium::io::file_compression::none == f.compression()); + REQUIRE(false == f.has_multiple_object_versions()); + f.check(); + } + + SECTION("url without format and filename") { + osmium::io::File f {"http://planet.osm.org/pbf/planet-latest.osm.pbf"}; + REQUIRE(osmium::io::file_format::pbf == f.format()); + REQUIRE(osmium::io::file_compression::none == f.compression()); + REQUIRE(false == f.has_multiple_object_versions()); + f.check(); + } + + SECTION("url with format") { + osmium::io::File f {"http://www.example.com/api", "osh"}; + REQUIRE(osmium::io::file_format::xml == f.format()); + REQUIRE(osmium::io::file_compression::none == f.compression()); + REQUIRE(true == f.has_multiple_object_versions()); + f.check(); + } + } diff --git a/test/t/io/test_string_table.cpp b/test/t/io/test_string_table.cpp new file mode 100644 index 00000000000..7fedfcfcac0 --- /dev/null +++ b/test/t/io/test_string_table.cpp @@ -0,0 +1,94 @@ +#include "catch.hpp" + +#include + +TEST_CASE("String store") { + osmium::io::detail::StringStore ss(100); + + SECTION("empty") { + REQUIRE(ss.begin() == ss.end()); + } + + SECTION("add zero-length string") { + const char* s1 = ss.add(""); + REQUIRE(std::string(s1) == ""); + + auto it = ss.begin(); + REQUIRE(s1 == *it); + REQUIRE(std::string(*it) == ""); + REQUIRE(++it == ss.end()); + } + + SECTION("add strings") { + const char* s1 = ss.add("foo"); + const char* s2 = ss.add("bar"); + REQUIRE(s1 != s2); + REQUIRE(std::string(s1) == "foo"); + REQUIRE(std::string(s2) == "bar"); + + auto it = ss.begin(); + REQUIRE(s1 == *it++); + REQUIRE(s2 == *it++); + REQUIRE(it == ss.end()); + } + + SECTION("add zero-length string and longer strings") { + const char* s1 = ss.add(""); + const char* s2 = ss.add("xxx"); + const char* s3 = ss.add("yyyyy"); + + auto it = ss.begin(); + REQUIRE(std::string(*it++) == ""); + REQUIRE(std::string(*it++) == "xxx"); + REQUIRE(std::string(*it++) == "yyyyy"); + REQUIRE(it == ss.end()); + } + + SECTION("add many strings") { + for (const char* teststring : {"a", "abc", "abcd", "abcde"}) { + int i = 0; + for (; i < 100; ++i) { + ss.add(teststring); + } + + for (const char* s : ss) { + REQUIRE(std::string(s) == teststring); + --i; + } + + REQUIRE(i == 0); + ss.clear(); + } + } + +} + +TEST_CASE("String table") { + osmium::io::detail::StringTable st; + + SECTION("empty") { + REQUIRE(st.size() == 1); + REQUIRE(std::next(st.begin()) == st.end()); + } + + SECTION("add strings") { + REQUIRE(st.add("foo") == 1); + REQUIRE(st.add("bar") == 2); + REQUIRE(st.add("bar") == 2); + REQUIRE(st.add("baz") == 3); + REQUIRE(st.add("foo") == 1); + REQUIRE(st.size() == 4); + + auto it = st.begin(); + REQUIRE(std::string("") == *it++); + REQUIRE(std::string("foo") == *it++); + REQUIRE(std::string("bar") == *it++); + REQUIRE(std::string("baz") == *it++); + REQUIRE(it == st.end()); + + st.clear(); + REQUIRE(st.size() == 1); + } + +} + diff --git a/test/t/tags/test_tag_list.cpp b/test/t/tags/test_tag_list.cpp index c2512d12510..77523e7965d 100644 --- a/test/t/tags/test_tag_list.cpp +++ b/test/t/tags/test_tag_list.cpp @@ -4,73 +4,99 @@ #include #include -TEST_CASE("tag_list") { +TEST_CASE("create tag list") { + osmium::memory::Buffer buffer(10240); + + SECTION("with TagListBuilder from char*") { + { + osmium::builder::TagListBuilder builder(buffer); + builder.add_tag("highway", "primary"); + builder.add_tag("name", "Main Street"); + } + buffer.commit(); + } - SECTION("can_be_created_from_initializer_list") { - osmium::memory::Buffer buffer(10240); + SECTION("with TagListBuilder from char* with length") { + { + osmium::builder::TagListBuilder builder(buffer); + builder.add_tag("highway", strlen("highway"), "primary", strlen("primary")); + builder.add_tag("nameXX", 4, "Main Street", 11); + } + buffer.commit(); + } - const osmium::TagList& tl = osmium::builder::build_tag_list(buffer, { + SECTION("with TagListBuilder from std::string") { + { + osmium::builder::TagListBuilder builder(buffer); + builder.add_tag(std::string("highway"), std::string("primary")); + const std::string source = "name"; + std::string gps = "Main Street"; + builder.add_tag(source, gps); + } + buffer.commit(); + } + + SECTION("with build_tag_list from initializer list") { + osmium::builder::build_tag_list(buffer, { { "highway", "primary" }, - { "name", "Main Street" }, - { "source", "GPS" } + { "name", "Main Street" } }); - - REQUIRE(osmium::item_type::tag_list == tl.type()); - REQUIRE(3 == tl.size()); - REQUIRE(std::string("highway") == tl.begin()->key()); - REQUIRE(std::string("primary") == tl.begin()->value()); } - SECTION("can_be_created_from_map") { - osmium::memory::Buffer buffer(10240); - - const osmium::TagList& tl = osmium::builder::build_tag_list_from_map(buffer, std::map({ + SECTION("with build_tag_list_from_map") { + osmium::builder::build_tag_list_from_map(buffer, std::map({ { "highway", "primary" }, { "name", "Main Street" } })); - - REQUIRE(osmium::item_type::tag_list == tl.type()); - REQUIRE(2 == tl.size()); - - if (std::string("highway") == tl.begin()->key()) { - REQUIRE(std::string("primary") == tl.begin()->value()); - REQUIRE(std::string("name") == std::next(tl.begin(), 1)->key()); - REQUIRE(std::string("Main Street") == std::next(tl.begin(), 1)->value()); - } else { - REQUIRE(std::string("highway") == std::next(tl.begin(), 1)->key()); - REQUIRE(std::string("primary") == std::next(tl.begin(), 1)->value()); - REQUIRE(std::string("name") == tl.begin()->key()); - REQUIRE(std::string("Main Street") == tl.begin()->value()); - } } - SECTION("can_be_created_with_callback") { - osmium::memory::Buffer buffer(10240); - - const osmium::TagList& tl = osmium::builder::build_tag_list_from_func(buffer, [](osmium::builder::TagListBuilder& tlb) { + SECTION("with build_tag_list_from_func") { + osmium::builder::build_tag_list_from_func(buffer, [](osmium::builder::TagListBuilder& tlb) { tlb.add_tag("highway", "primary"); - tlb.add_tag("bridge", "true"); + tlb.add_tag("name", "Main Street"); }); - - REQUIRE(osmium::item_type::tag_list == tl.type()); - REQUIRE(2 == tl.size()); - REQUIRE(std::string("bridge") == std::next(tl.begin(), 1)->key()); - REQUIRE(std::string("true") == std::next(tl.begin(), 1)->value()); } - SECTION("returns_value_by_key") { - osmium::memory::Buffer buffer(10240); + const osmium::TagList& tl = *buffer.begin(); + REQUIRE(osmium::item_type::tag_list == tl.type()); + REQUIRE(2 == tl.size()); - const osmium::TagList& tl = osmium::builder::build_tag_list_from_func(buffer, [](osmium::builder::TagListBuilder& tlb) { - tlb.add_tag("highway", "primary"); - tlb.add_tag("bridge", "true"); - }); + auto it = tl.begin(); + REQUIRE(std::string("highway") == it->key()); + REQUIRE(std::string("primary") == it->value()); + ++it; + REQUIRE(std::string("name") == it->key()); + REQUIRE(std::string("Main Street") == it->value()); + ++it; + REQUIRE(it == tl.end()); - REQUIRE(std::string("primary") == tl.get_value_by_key("highway")); - REQUIRE(nullptr == tl.get_value_by_key("name")); - REQUIRE(std::string("foo") == tl.get_value_by_key("name", "foo")); + REQUIRE(std::string("primary") == tl.get_value_by_key("highway")); + REQUIRE(nullptr == tl.get_value_by_key("foo")); + REQUIRE(std::string("default") == tl.get_value_by_key("foo", "default")); - REQUIRE(std::string("true") == tl["bridge"]); - } + REQUIRE(std::string("Main Street") == tl["name"]); +} + +TEST_CASE("empty keys and values are okay") { + osmium::memory::Buffer buffer(10240); + + const osmium::TagList& tl = osmium::builder::build_tag_list(buffer, { + { "empty value", "" }, + { "", "empty key" } + }); + + REQUIRE(osmium::item_type::tag_list == tl.type()); + REQUIRE(2 == tl.size()); + + auto it = tl.begin(); + REQUIRE(std::string("empty value") == it->key()); + REQUIRE(std::string("") == it->value()); + ++it; + REQUIRE(std::string("") == it->key()); + REQUIRE(std::string("empty key") == it->value()); + ++it; + REQUIRE(it == tl.end()); + REQUIRE(std::string("") == tl.get_value_by_key("empty value")); + REQUIRE(std::string("empty key") == tl.get_value_by_key("")); } diff --git a/test/t/thread/test_pool.cpp b/test/t/thread/test_pool.cpp index 66bb3734a8f..5fa6bbab5ae 100644 --- a/test/t/thread/test_pool.cpp +++ b/test/t/thread/test_pool.cpp @@ -59,7 +59,6 @@ TEST_CASE("thread") { auto& pool = osmium::thread::Pool::instance(); result = 0; - bool got_exception = false; auto future = pool.submit(test_job_throw {}); REQUIRE_THROWS_AS(future.get(), std::runtime_error); diff --git a/test/t/util/test_data_file.cpp b/test/t/util/test_data_file.cpp new file mode 100644 index 00000000000..3f432f94376 --- /dev/null +++ b/test/t/util/test_data_file.cpp @@ -0,0 +1,81 @@ +#include "catch.hpp" + +#include + +#include + +TEST_CASE("temporary file") { + + SECTION("create/open") { + osmium::util::DataFile file; + + REQUIRE(!!file); + int fd = file.fd(); + + REQUIRE(fd > 0); + + const char buf[] = "foobar"; + REQUIRE(::write(fd, buf, sizeof(buf)) == sizeof(buf)); + + file.close(); + + REQUIRE(!file); + } + +} + +TEST_CASE("named file") { + + SECTION("create/open") { + { + osmium::util::DataFile file("test.data", true); + + REQUIRE(!!file); + int fd = file.fd(); + + REQUIRE(fd > 0); + + REQUIRE(file.size() == 0); + + const char buf[] = "foobar"; + REQUIRE(::write(fd, buf, sizeof(buf) - 1) == sizeof(buf) - 1); + + file.close(); + + REQUIRE(!file); + } + { + osmium::util::DataFile file("test.data", false); + + REQUIRE(!!file); + int fd = file.fd(); + + REQUIRE(fd > 0); + + REQUIRE(file.size() == 6); + + char buf[10]; + int len = ::read(fd, buf, sizeof(buf)); + + REQUIRE(len == 6); + REQUIRE(!strncmp(buf, "foobar", 6)); + + file.close(); + + REQUIRE(!file); + REQUIRE(unlink("test.data") == 0); + } + } + + SECTION("grow file") { + osmium::util::DataFile file("test.data", true); + + REQUIRE(!!file); + + REQUIRE(file.size() == 0); + file.grow(10); + REQUIRE(file.size() == 10); + } + +} + diff --git a/test/t/util/test_delta.cpp b/test/t/util/test_delta.cpp new file mode 100644 index 00000000000..cebcca8ed06 --- /dev/null +++ b/test/t/util/test_delta.cpp @@ -0,0 +1,68 @@ +#include "catch.hpp" + +#include + +#include + +TEST_CASE("delta encode") { + + osmium::util::DeltaEncode x; + + SECTION("int") { + REQUIRE(x.update(17) == 17); + REQUIRE(x.update(10) == -7); + } + +} + +TEST_CASE("delta decode") { + + osmium::util::DeltaDecode x; + + SECTION("int") { + REQUIRE(x.update(17) == 17); + REQUIRE(x.update(10) == 27); + } + +} + +TEST_CASE("delta encode and decode") { + + std::vector a = { 5, -9, 22, 13, 0, 23 }; + + osmium::util::DeltaEncode de; + std::vector b; + for (int x : a) { + b.push_back(de.update(x)); + } + + osmium::util::DeltaDecode dd; + std::vector c; + for (int x : b) { + c.push_back(dd.update(x)); + } + +} + +TEST_CASE("delta encode iterator") { + std::vector data = { 4, 5, 13, 22, 12 }; + + auto l = [](std::vector::const_iterator it) -> int { + return *it; + }; + + typedef osmium::util::DeltaEncodeIterator::const_iterator, decltype(l), int> it_type; + it_type it(data.begin(), data.end(), l); + it_type end(data.end(), data.end(), l); + + REQUIRE(*it == 4); + ++it; + REQUIRE(*it++ == 1); + REQUIRE(*it == 8); + ++it; + REQUIRE(*it++ == 9); + REQUIRE(*it == -10); + ++it; + REQUIRE(it == end); +} + diff --git a/test/t/util/test_file.cpp b/test/t/util/test_file.cpp new file mode 100644 index 00000000000..2787261a54d --- /dev/null +++ b/test/t/util/test_file.cpp @@ -0,0 +1,69 @@ +#include "catch.hpp" + +#include + +#ifdef _WIN32 +// https://msdn.microsoft.com/en-us/library/ksazx244.aspx +// https://msdn.microsoft.com/en-us/library/a9yf33zb.aspx +class DoNothingInvalidParameterHandler { + + static void invalid_parameter_handler( + const wchar_t* expression, + const wchar_t* function, + const wchar_t* file, + unsigned int line, + uintptr_t pReserved + ) { + // do nothing + } + + _invalid_parameter_handler old_handler; + +public: + + DoNothingInvalidParameterHandler() : + old_handler(_set_invalid_parameter_handler(invalid_parameter_handler)) { + } + + ~DoNothingInvalidParameterHandler() { + _set_invalid_parameter_handler(old_handler); + } + +}; // class InvalidParameterHandler +#endif + + +TEST_CASE("file_size") { + +#ifdef _WIN32 + DoNothingInvalidParameterHandler handler; +#endif + + SECTION("illegal fd should throw") { + REQUIRE_THROWS_AS(osmium::util::file_size(-1), std::system_error); + } + + SECTION("unused fd should throw") { + // its unlikely that fd 1000 is open... + REQUIRE_THROWS_AS(osmium::util::file_size(1000), std::system_error); + } + +} + +TEST_CASE("resize_file") { + +#ifdef _WIN32 + DoNothingInvalidParameterHandler handler; +#endif + + SECTION("illegal fd should throw") { + REQUIRE_THROWS_AS(osmium::util::resize_file(-1, 10), std::system_error); + } + + SECTION("unused fd should throw") { + // its unlikely that fd 1000 is open... + REQUIRE_THROWS_AS(osmium::util::resize_file(1000, 10), std::system_error); + } + +} + diff --git a/test/t/util/test_memory_mapping.cpp b/test/t/util/test_memory_mapping.cpp new file mode 100644 index 00000000000..29893f7c70b --- /dev/null +++ b/test/t/util/test_memory_mapping.cpp @@ -0,0 +1,419 @@ +#include "catch.hpp" + +#include +#include + +#include +#include + +#if defined(_MSC_VER) || (defined(__GNUC__) && defined(_WIN32)) +#include "win_mkstemp.hpp" +#endif + +static const size_t huge = std::numeric_limits::max(); + +TEST_CASE("anonymous mapping") { + + SECTION("simple memory mapping should work") { + osmium::util::MemoryMapping mapping(1000, osmium::util::MemoryMapping::mapping_mode::write_private); + REQUIRE(mapping.get_addr() != nullptr); + + REQUIRE(mapping.size() >= 1000); + + volatile int* addr = mapping.get_addr(); + + REQUIRE(mapping.writable()); + + *addr = 42; + REQUIRE(*addr == 42); + + REQUIRE(!!mapping); + mapping.unmap(); + REQUIRE(!mapping); + mapping.unmap(); // second unmap is okay + } + + SECTION("memory mapping of zero length should work") { + osmium::util::MemoryMapping mapping(0, osmium::util::MemoryMapping::mapping_mode::write_private); + REQUIRE(mapping.get_addr() != nullptr); + + REQUIRE(mapping.size() == osmium::util::get_pagesize()); + + REQUIRE(!!mapping); + mapping.unmap(); + REQUIRE(!mapping); + } + + SECTION("moving a memory mapping should work") { + osmium::util::MemoryMapping mapping1(1000, osmium::util::MemoryMapping::mapping_mode::write_private); + int* addr1 = mapping1.get_addr(); + *addr1 = 42; + + REQUIRE(!!mapping1); + osmium::util::MemoryMapping mapping2(std::move(mapping1)); + REQUIRE(!!mapping2); + REQUIRE(!mapping1); + mapping1.unmap(); + + int* addr2 = mapping2.get_addr(); + REQUIRE(*addr2 == 42); + + mapping2.unmap(); + REQUIRE(!mapping2); + } + + SECTION("move assignment should work") { + osmium::util::MemoryMapping mapping1(1000, osmium::util::MemoryMapping::mapping_mode::write_private); + osmium::util::MemoryMapping mapping2(1000, osmium::util::MemoryMapping::mapping_mode::write_private); + + REQUIRE(!!mapping1); + REQUIRE(!!mapping2); + + int* addr1 = mapping1.get_addr(); + *addr1 = 42; + + mapping2 = std::move(mapping1); + REQUIRE(!!mapping2); + REQUIRE(!mapping1); + + int* addr2 = mapping2.get_addr(); + REQUIRE(*addr2 == 42); + + mapping2.unmap(); + REQUIRE(!mapping2); + } + +#ifdef __linux__ + SECTION("remapping to larger size should work") { + osmium::util::MemoryMapping mapping(1000, osmium::util::MemoryMapping::mapping_mode::write_private); + REQUIRE(mapping.size() >= 1000); + + size_t size1 = mapping.size(); + + int* addr1 = mapping.get_addr(); + *addr1 = 42; + + mapping.resize(8000); + REQUIRE(mapping.size() > size1); + + int* addr2 = mapping.get_addr(); + REQUIRE(*addr2 == 42); + } + + SECTION("remapping to smaller size should work") { + osmium::util::MemoryMapping mapping(8000, osmium::util::MemoryMapping::mapping_mode::write_private); + REQUIRE(mapping.size() >= 1000); + + size_t size1 = mapping.size(); + + int* addr1 = mapping.get_addr(); + *addr1 = 42; + + mapping.resize(500); + REQUIRE(mapping.size() < size1); + + int* addr2 = mapping.get_addr(); + REQUIRE(*addr2 == 42); + } +#endif + +} + +TEST_CASE("file-based mapping") { + + SECTION("writing to a mapped file should work") { + char filename[] = "test_mmap_write_XXXXXX"; + const int fd = mkstemp(filename); + REQUIRE(fd > 0); + + osmium::util::resize_file(fd, 100); + + { + osmium::util::MemoryMapping mapping(100, osmium::util::MemoryMapping::mapping_mode::write_shared, fd); + REQUIRE(mapping.writable()); + + REQUIRE(!!mapping); + REQUIRE(mapping.size() >= 100); + + *mapping.get_addr() = 1234; + + mapping.unmap(); + } + + REQUIRE(osmium::util::file_size(fd) == 100); + + { + osmium::util::MemoryMapping mapping(100, osmium::util::MemoryMapping::mapping_mode::readonly, fd); + REQUIRE(!mapping.writable()); + + REQUIRE(!!mapping); + REQUIRE(mapping.size() >= 100); + REQUIRE(*mapping.get_addr() == 1234); + + mapping.unmap(); + } + + REQUIRE(0 == close(fd)); + REQUIRE(0 == unlink(filename)); + } + + SECTION("writing to a privately mapped file should work") { + char filename[] = "test_mmap_write_XXXXXX"; + const int fd = mkstemp(filename); + REQUIRE(fd > 0); + + osmium::util::resize_file(fd, 100); + + { + osmium::util::MemoryMapping mapping(100, osmium::util::MemoryMapping::mapping_mode::write_private, fd); + REQUIRE(mapping.writable()); + + REQUIRE(!!mapping); + REQUIRE(mapping.size() >= 100); + + *mapping.get_addr() = 1234; + + mapping.unmap(); + } + + REQUIRE(osmium::util::file_size(fd) == 100); + + { + osmium::util::MemoryMapping mapping(100, osmium::util::MemoryMapping::mapping_mode::readonly, fd); + REQUIRE(!mapping.writable()); + + REQUIRE(!!mapping); + REQUIRE(mapping.size() >= 100); + REQUIRE(*mapping.get_addr() == 0); // should not see the value set above + + mapping.unmap(); + } + + REQUIRE(0 == close(fd)); + REQUIRE(0 == unlink(filename)); + } + + SECTION("remapping to larger size should work") { + char filename[] = "test_mmap_grow_XXXXXX"; + const int fd = mkstemp(filename); + REQUIRE(fd > 0); + + osmium::util::MemoryMapping mapping(100, osmium::util::MemoryMapping::mapping_mode::write_shared, fd); + REQUIRE(mapping.size() >= 100); + size_t size1 = mapping.size(); + + int* addr1 = mapping.get_addr(); + *addr1 = 42; + + mapping.resize(8000); + REQUIRE(mapping.size() >= 8000); + REQUIRE(mapping.size() > size1); + + int* addr2 = mapping.get_addr(); + REQUIRE(*addr2 == 42); + + mapping.unmap(); + + REQUIRE(0 == close(fd)); + REQUIRE(0 == unlink(filename)); + } + + SECTION("remapping to smaller size should work") { + char filename[] = "test_mmap_shrink_XXXXXX"; + const int fd = mkstemp(filename); + REQUIRE(fd > 0); + + { + osmium::util::MemoryMapping mapping(8000, osmium::util::MemoryMapping::mapping_mode::write_shared, fd); + REQUIRE(mapping.size() >= 8000); + size_t size1 = mapping.size(); + + int* addr1 = mapping.get_addr(); + *addr1 = 42; + + mapping.resize(50); + REQUIRE(mapping.size() >= 50); + REQUIRE(mapping.size() < size1); + + int* addr2 = mapping.get_addr(); + REQUIRE(*addr2 == 42); + } + + REQUIRE(0 == close(fd)); + REQUIRE(0 == unlink(filename)); + } +} + +TEST_CASE("typed anonymous mapping") { + + SECTION("simple memory mapping should work") { + osmium::util::TypedMemoryMapping mapping(1000); + volatile uint32_t* addr = mapping.begin(); + + REQUIRE(mapping.writable()); + + *addr = 42; + REQUIRE(*addr == 42); + + REQUIRE(!!mapping); + mapping.unmap(); + REQUIRE(!mapping); + mapping.unmap(); // second unmap is okay + } + + SECTION("moving a memory mapping should work") { + osmium::util::TypedMemoryMapping mapping1(1000); + uint32_t* addr1 = mapping1.begin(); + *addr1 = 42; + + REQUIRE(!!mapping1); + osmium::util::TypedMemoryMapping mapping2(std::move(mapping1)); + REQUIRE(!!mapping2); + REQUIRE(!mapping1); + mapping1.unmap(); + + auto addr2 = mapping2.begin(); + REQUIRE(*addr2 == 42); + + mapping2.unmap(); + REQUIRE(!mapping2); + } + + SECTION("move assignment should work") { + osmium::util::TypedMemoryMapping mapping1(1000); + osmium::util::TypedMemoryMapping mapping2(1000); + + REQUIRE(!!mapping1); + REQUIRE(!!mapping2); + + auto addr1 = mapping1.begin(); + *addr1 = 42; + + mapping2 = std::move(mapping1); + REQUIRE(!!mapping2); + REQUIRE(!mapping1); + + auto addr2 = mapping2.begin(); + REQUIRE(*addr2 == 42); + + mapping2.unmap(); + REQUIRE(!mapping2); + } + +#ifdef __linux__ + SECTION("remapping to larger size should work") { + osmium::util::TypedMemoryMapping mapping(1000); + REQUIRE(mapping.size() >= 1000); + + auto addr1 = mapping.begin(); + *addr1 = 42; + + mapping.resize(8000); + + auto addr2 = mapping.begin(); + REQUIRE(*addr2 == 42); + } + + SECTION("remapping to smaller size should work") { + osmium::util::TypedMemoryMapping mapping(8000); + REQUIRE(mapping.size() >= 8000); + + auto addr1 = mapping.begin(); + *addr1 = 42; + + mapping.resize(500); + + auto addr2 = mapping.begin(); + REQUIRE(*addr2 == 42); + } +#endif + +} + +TEST_CASE("typed file-based mapping") { + + SECTION("writing to a mapped file should work") { + char filename[] = "test_mmap_file_size_XXXXXX"; + const int fd = mkstemp(filename); + REQUIRE(fd > 0); + + osmium::util::resize_file(fd, 100); + + { + osmium::util::TypedMemoryMapping mapping(100, osmium::util::MemoryMapping::mapping_mode::write_shared, fd); + REQUIRE(mapping.writable()); + + REQUIRE(!!mapping); + REQUIRE(mapping.size() >= 100); + + *mapping.begin() = 1234; + + mapping.unmap(); + } + + { + osmium::util::TypedMemoryMapping mapping(100, osmium::util::MemoryMapping::mapping_mode::readonly, fd); + REQUIRE(!mapping.writable()); + + REQUIRE(!!mapping); + REQUIRE(mapping.size() >= 100); + REQUIRE(*mapping.begin() == 1234); + + mapping.unmap(); + } + + REQUIRE(0 == close(fd)); + REQUIRE(0 == unlink(filename)); + } + +} + +TEST_CASE("anonymous memory mapping class") { + + SECTION("simple memory mapping should work") { + osmium::util::AnonymousMemoryMapping mapping(1000); + REQUIRE(mapping.get_addr() != nullptr); + + volatile int* addr = mapping.get_addr(); + + REQUIRE(mapping.writable()); + + *addr = 42; + REQUIRE(*addr == 42); + + REQUIRE(!!mapping); + mapping.unmap(); + REQUIRE(!mapping); + mapping.unmap(); // second unmap is okay + } + +#ifdef __linux__ + SECTION("remapping to larger size should work") { + osmium::util::AnonymousMemoryMapping mapping(1000); + REQUIRE(mapping.size() >= 1000); + + int* addr1 = mapping.get_addr(); + *addr1 = 42; + + mapping.resize(2000); + + int* addr2 = mapping.get_addr(); + REQUIRE(*addr2 == 42); + } + + SECTION("remapping to smaller size should work") { + osmium::util::AnonymousMemoryMapping mapping(2000); + REQUIRE(mapping.size() >= 2000); + + int* addr1 = mapping.get_addr(); + *addr1 = 42; + + mapping.resize(500); + + int* addr2 = mapping.get_addr(); + REQUIRE(*addr2 == 42); + } +#endif + +} + diff --git a/test/t/util/test_minmax.cpp b/test/t/util/test_minmax.cpp new file mode 100644 index 00000000000..8b40f856e84 --- /dev/null +++ b/test/t/util/test_minmax.cpp @@ -0,0 +1,68 @@ +#include "catch.hpp" + +#include +#include + +TEST_CASE("minmax numeric") { + + SECTION("min") { + osmium::min_op x; + REQUIRE(x() == std::numeric_limits::max()); + + x.update(17); + REQUIRE(x() == 17); + + x.update(10); + REQUIRE(x() == 10); + + x.update(22); + REQUIRE(x() == 10); + } + + SECTION("max") { + osmium::max_op x; + REQUIRE(x() == 0); + + x.update(17); + REQUIRE(x() == 17); + + x.update(10); + REQUIRE(x() == 17); + + x.update(22); + REQUIRE(x() == 22); + } + +} + +TEST_CASE("minmax timestamp") { + + SECTION("min") { + osmium::min_op x; + + x.update(osmium::Timestamp("2010-01-01T00:00:00Z")); + REQUIRE(x().to_iso() == "2010-01-01T00:00:00Z"); + + x.update(osmium::Timestamp("2015-01-01T00:00:00Z")); + REQUIRE(x().to_iso() == "2010-01-01T00:00:00Z"); + + x.update(osmium::Timestamp("2000-01-01T00:00:00Z")); + REQUIRE(x().to_iso() == "2000-01-01T00:00:00Z"); + } + + SECTION("max") { + osmium::max_op x; + + x.update(osmium::Timestamp("2010-01-01T00:00:00Z")); + REQUIRE(x().to_iso() == "2010-01-01T00:00:00Z"); + + x.update(osmium::Timestamp("2015-01-01T00:00:00Z")); + REQUIRE(x().to_iso() == "2015-01-01T00:00:00Z"); + + x.update(osmium::Timestamp("2000-01-01T00:00:00Z")); + REQUIRE(x().to_iso() == "2015-01-01T00:00:00Z"); + + } + +} + diff --git a/test/t/util/test_string.cpp b/test/t/util/test_string.cpp index fa49787791f..0960afe9a3b 100644 --- a/test/t/util/test_string.cpp +++ b/test/t/util/test_string.cpp @@ -9,6 +9,7 @@ TEST_CASE("split_string") { std::vector result = {"foo", "baramba", "baz"}; REQUIRE(result == osmium::split_string(str, ',')); + REQUIRE(result == osmium::split_string(str, ',', true)); } SECTION("split_string string without sep") { @@ -16,34 +17,43 @@ TEST_CASE("split_string") { std::vector result = {"foo"}; REQUIRE(result == osmium::split_string(str, ',')); + REQUIRE(result == osmium::split_string(str, ',', true)); } SECTION("split_string string with empty at end") { std::string str { "foo,bar," }; std::vector result = {"foo", "bar", ""}; + std::vector resultc = {"foo", "bar"}; REQUIRE(result == osmium::split_string(str, ',')); + REQUIRE(resultc == osmium::split_string(str, ',', true)); } SECTION("split_string string with empty in middle") { std::string str { "foo,,bar" }; std::vector result = {"foo", "", "bar"}; + std::vector resultc = {"foo", "bar"}; REQUIRE(result == osmium::split_string(str, ',')); + REQUIRE(resultc == osmium::split_string(str, ',', true)); } SECTION("split_string string with empty at start") { std::string str { ",bar,baz" }; std::vector result = {"", "bar", "baz"}; + std::vector resultc = {"bar", "baz"}; REQUIRE(result == osmium::split_string(str, ',')); + REQUIRE(resultc == osmium::split_string(str, ',', true)); } SECTION("split_string sep") { std::string str { "," }; std::vector result = {"", ""}; + std::vector resultc; REQUIRE(result == osmium::split_string(str, ',')); + REQUIRE(resultc == osmium::split_string(str, ',', true)); } SECTION("split_string empty string") { @@ -51,6 +61,7 @@ TEST_CASE("split_string") { std::vector result; REQUIRE(result == osmium::split_string(str, ',')); + REQUIRE(result == osmium::split_string(str, ',', true)); } } From 03c8fdd30adb15ee7d35cd39569d25341da67896 Mon Sep 17 00:00:00 2001 From: "Daniel J. Hofmann" Date: Mon, 31 Aug 2015 16:48:27 +0200 Subject: [PATCH 056/122] Remove protobuf dependencies from docker setup --- docker/Dockerfile | 3 --- 1 file changed, 3 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index d7c2f9e71ef..4535949a1bb 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -4,7 +4,6 @@ RUN apt-get update -y RUN apt-get install -y build-essential git-core python-pip python-software-properties software-properties-common RUN apt-get -y install gcc-4.8 g++-4.8 libboost1.55-all-dev llvm-3.4 -RUN apt-get -y install protobuf-compiler libprotoc-dev libprotobuf-dev RUN apt-get -y install libbz2-dev libstxxl-dev libstxxl1 libxml2-dev RUN apt-get -y install libzip-dev lua5.1 liblua5.1-0-dev libtbb-dev libgdal-dev ruby1.9 RUN apt-get -y install curl cmake cmake-curses-gui @@ -15,8 +14,6 @@ RUN pip install awscli RUN curl https://gist.githubusercontent.com/DennisOSRM/f2eb7b948e6fe1ae319e/raw/install-luabind.sh | sudo bash # osmosis RUN curl -s https://gist.githubusercontent.com/DennisOSRM/803a64a9178ec375069f/raw/ | sudo bash -# osmpbf library -RUN curl -s https://gist.githubusercontent.com/DennisOSRM/13b1b4fe38a57ead850e/raw/install_osmpbf.sh | sudo bash RUN useradd -ms /bin/bash mapbox USER mapbox From c39ca7189b685f251c6e7db3ce14133feeac10a5 Mon Sep 17 00:00:00 2001 From: "Daniel J. Hofmann" Date: Mon, 31 Aug 2015 16:52:39 +0200 Subject: [PATCH 057/122] Remove protobuf dependencies from build system --- CMakeLists.txt | 10 ------- cmake/CPackDebianConfig.cmake | 2 +- cmake/FindOSMPBF.cmake | 54 ----------------------------------- 3 files changed, 1 insertion(+), 65 deletions(-) delete mode 100644 cmake/FindOSMPBF.cmake diff --git a/CMakeLists.txt b/CMakeLists.txt index 77799a31cef..89c2c58f887 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -281,16 +281,6 @@ if(OPENMP_FOUND) set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${OpenMP_CXX_FLAGS}") endif() -find_package(OSMPBF REQUIRED) -include_directories(${OSMPBF_INCLUDE_DIR}) -target_link_libraries(osrm-extract ${OSMPBF_LIBRARY}) -target_link_libraries(osrm-prepare ${OSMPBF_LIBRARY}) - -find_package(Protobuf REQUIRED) -include_directories(${PROTOBUF_INCLUDE_DIRS}) -target_link_libraries(osrm-extract ${PROTOBUF_LIBRARY}) -target_link_libraries(osrm-prepare ${PROTOBUF_LIBRARY}) - find_package(BZip2 REQUIRED) include_directories(${BZIP_INCLUDE_DIRS}) target_link_libraries(osrm-extract ${BZIP2_LIBRARIES}) diff --git a/cmake/CPackDebianConfig.cmake b/cmake/CPackDebianConfig.cmake index 065c4290015..bd434ee9e33 100644 --- a/cmake/CPackDebianConfig.cmake +++ b/cmake/CPackDebianConfig.cmake @@ -34,7 +34,7 @@ SET(CPACK_DEBIAN_PACKAGE_SECTION "devel") SET(CPACK_DEBIAN_PACKAGE_DESCRIPTION "Open Source Routing Machine (OSRM) is a high-performance routing engine. It combines sophisticated routing algorithms with the open and free data of the OpenStreetMap." ) -SET(CPACK_DEBIAN_PACKAGE_DEPENDS "libc6-dev, libprotobuf-dev, libosmpbf-dev, libbz2-1.0, libstxxl1, libxml2, libzip2, liblua5.1-0, libtbb2, libboost-all-dev") +SET(CPACK_DEBIAN_PACKAGE_DEPENDS "libc6-dev, libbz2-1.0, libstxxl1, libxml2, libzip2, liblua5.1-0, libtbb2, libboost-all-dev") file(GLOB_RECURSE ProfileGlob ${CMAKE_SOURCE_DIR}/profiles/*) install(FILES ${ProfileGlob} DESTINATION "share/doc/${LOWER_PROJECT_NAME}/profiles") diff --git a/cmake/FindOSMPBF.cmake b/cmake/FindOSMPBF.cmake deleted file mode 100644 index 78b1d9dc827..00000000000 --- a/cmake/FindOSMPBF.cmake +++ /dev/null @@ -1,54 +0,0 @@ -# Locate OSMPBF library -# This module defines -# OSMPBF_FOUND, if false, do not try to link to OSMPBF -# OSMPBF_LIBRARIES -# OSMPBF_INCLUDE_DIR, where to find OSMPBF.hpp -# -# Note that the expected include convention is -# #include -# and not -# #include - -IF( NOT OSMPBF_FIND_QUIETLY ) - MESSAGE(STATUS "Looking for OSMPBF...") -ENDIF() - -FIND_PATH(OSMPBF_INCLUDE_DIR osmpbf.h - HINTS - $ENV{OSMPBF_DIR} - PATH_SUFFIXES OSMPBF include/osmpbf include - PATHS - ~/Library/Frameworks - /Library/Frameworks - /usr/local - /usr - /opt/local # DarwinPorts - /opt -) - -FIND_LIBRARY(OSMPBF_LIBRARY - NAMES osmpbf - HINTS - $ENV{OSMPBF_DIR} - PATH_SUFFIXES lib64 lib - PATHS - ~/Library/Frameworks - /Library/Frameworks - /usr/local - /usr - /opt/local - /opt -) - -INCLUDE(FindPackageHandleStandardArgs) -# handle the QUIETLY and REQUIRED arguments and set OSMPBF_FOUND to TRUE if -# all listed variables are TRUE -FIND_PACKAGE_HANDLE_STANDARD_ARGS(OSMPBF DEFAULT_MSG OSMPBF_LIBRARY OSMPBF_INCLUDE_DIR) - -IF( NOT OSMPBF_FIND_QUIETLY ) - IF( OSMPBF_FOUND ) - MESSAGE(STATUS "Found OSMPBF: ${OSMPBF_LIBRARY}" ) - ENDIF() -ENDIF() - -#MARK_AS_ADVANCED(OSMPBF_INCLUDE_DIR OSMPBF_LIBRARIES OSMPBF_LIBRARY OSMPBF_LIBRARY_DBG) From ac64e8b15ee63e1e3019aebb6f9faead8c8ece81 Mon Sep 17 00:00:00 2001 From: "Daniel J. Hofmann" Date: Mon, 31 Aug 2015 16:57:42 +0200 Subject: [PATCH 058/122] Remove protobuf dependencies from travis config --- .travis.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 55e719035ad..830599fe47b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,7 +7,7 @@ install: - sudo apt-add-repository -y ppa:ubuntu-toolchain-r/test - sudo add-apt-repository -y ppa:boost-latest/ppa - sudo apt-get update >/dev/null - - sudo apt-get -q install protobuf-compiler libprotoc-dev libprotobuf7 libprotobuf-dev libbz2-dev libstxxl-dev libstxxl1 libxml2-dev libzip-dev lua5.1 liblua5.1-0-dev rubygems libtbb-dev + - sudo apt-get -q install libbz2-dev libstxxl-dev libstxxl1 libxml2-dev libzip-dev lua5.1 liblua5.1-0-dev rubygems libtbb-dev - sudo apt-get -q install g++-4.8 - sudo apt-get install libboost1.54-all-dev - sudo apt-get install libgdal-dev @@ -17,8 +17,6 @@ install: - curl -s https://gist.githubusercontent.com/DennisOSRM/803a64a9178ec375069f/raw/ | sudo bash # cmake - curl -s https://gist.githubusercontent.com/DennisOSRM/5fad9bee5c7f09fd7fc9/raw/ | sudo bash - # osmpbf library - - curl -s https://gist.githubusercontent.com/DennisOSRM/13b1b4fe38a57ead850e/raw/install_osmpbf.sh | sudo bash before_script: - rvm use 1.9.3 - gem install bundler From e3757fbbfa93f5efe57e77f139e15d82d48d5cc6 Mon Sep 17 00:00:00 2001 From: chaupow Date: Wed, 22 Apr 2015 17:19:54 +0200 Subject: [PATCH 059/122] add round trip plugin with greedy approximation --- library/osrm_impl.cpp | 2 + plugins/round_trip.hpp | 187 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 189 insertions(+) create mode 100644 plugins/round_trip.hpp diff --git a/library/osrm_impl.cpp b/library/osrm_impl.cpp index 5bb58d1749a..8057069eb88 100644 --- a/library/osrm_impl.cpp +++ b/library/osrm_impl.cpp @@ -41,6 +41,7 @@ class named_mutex; #include "../plugins/locate.hpp" #include "../plugins/nearest.hpp" #include "../plugins/timestamp.hpp" +#include "../plugins/round_trip.hpp" #include "../plugins/viaroute.hpp" #include "../plugins/match.hpp" #include "../server/data_structures/datafacade_base.hpp" @@ -86,6 +87,7 @@ OSRM_impl::OSRM_impl(libosrm_config &lib_config) query_data_facade, lib_config.max_locations_map_matching)); RegisterPlugin(new TimestampPlugin>(query_data_facade)); RegisterPlugin(new ViaRoutePlugin>(query_data_facade)); + RegisterPlugin(new RoundTripPlugin>(query_data_facade)); } OSRM_impl::~OSRM_impl() diff --git a/plugins/round_trip.hpp b/plugins/round_trip.hpp new file mode 100644 index 00000000000..2afe74ace81 --- /dev/null +++ b/plugins/round_trip.hpp @@ -0,0 +1,187 @@ +/* + +Copyright (c) 2015, Project OSRM contributors +All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + +Redistributions of source code must retain the above copyright notice, this list +of conditions and the following disclaimer. +Redistributions in binary form must reproduce the above copyright notice, this +list of conditions and the following disclaimer in the documentation and/or +other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +*/ + +#ifndef ROUND_TRIP_HPP +#define ROUND_TRIP_HPP + +#include "plugin_base.hpp" + +#include "../algorithms/object_encoder.hpp" +#include "../data_structures/query_edge.hpp" +#include "../data_structures/search_engine.hpp" +#include "../descriptors/descriptor_base.hpp" +#include "../util/json_renderer.hpp" +#include "../util/make_unique.hpp" +#include "../util/string_util.hpp" +#include "../util/timing_util.hpp" + +#include + +#include + +#include +#include +#include +#include +#include +#include + +template class RoundTripPlugin final : public BasePlugin +{ + private: + std::string descriptor_string; + DataFacadeT *facade; + std::unique_ptr> search_engine_ptr; + + public: + explicit RoundTripPlugin(DataFacadeT *facade) + : descriptor_string("trip"), facade(facade) + { + search_engine_ptr = osrm::make_unique>(facade); + } + + const std::string GetDescriptor() const override final { return descriptor_string; } + + int HandleRequest(const RouteParameters &route_parameters, + osrm::json::Object &json_result) override final + { + // check if all inputs are coordinates + if (!check_all_coordinates(route_parameters.coordinates)) + { + return 400; + } + const bool checksum_OK = (route_parameters.check_sum == facade->GetCheckSum()); + + // find phantom nodes for all input coords + PhantomNodeArray phantom_node_vector(route_parameters.coordinates.size()); + for (const auto i : osrm::irange(0, route_parameters.coordinates.size())) + { + // if client hints are helpful, encode hints + if (checksum_OK && i < route_parameters.hints.size() && + !route_parameters.hints[i].empty()) + { + PhantomNode current_phantom_node; + ObjectEncoder::DecodeFromBase64(route_parameters.hints[i], current_phantom_node); + if (current_phantom_node.is_valid(facade->GetNumberOfNodes())) + { + phantom_node_vector[i].emplace_back(std::move(current_phantom_node)); + continue; + } + } + facade->IncrementalFindPhantomNodeForCoordinate(route_parameters.coordinates[i], + phantom_node_vector[i], 1); + if (phantom_node_vector[i].size() > 1) + { + phantom_node_vector[i].pop_back(); + } + + BOOST_ASSERT(phantom_node_vector[i].front().is_valid(facade->GetNumberOfNodes())); + } + + // compute the distance table of all phantom nodes + std::shared_ptr> result_table = + search_engine_ptr->distance_table(phantom_node_vector); + + if (!result_table) + { + return 400; + } + + SimpleLogger().Write() << "Distance Table Computed"; + + + ////////////////////////////////////////////////////////////////////////////////////////////////// + // START GREEDY NEAREST NEIGHBOUR HERE + // 1. grab a random location and mark as starting point + // 2. find the nearest unvisited neighbour, set it as the current location and mark as visited + // 3. repeat 2 until there is no unvisited location + // 4. return route back to starting point + // 5. compute route + // 6. DONE! + ////////////////////////////////////////////////////////////////////////////////////////////////// + + const auto number_of_locations = phantom_node_vector.size(); + InternalRouteResult raw_route; + // 1. START WITH LOCATION 0 AS START POINT + int curr_node = 0; + std::vector visited(number_of_locations, false); + visited[0] = true; + + SimpleLogger().Write() << "Added an initial via"; + SimpleLogger().Write() << "Started from location 0"; + SimpleLogger().Write() << "Number of locs: " << number_of_locations; + + PhantomNodes subroute; + // 3. REPEAT FOR EVERY UNVISITED NODE + for(int stopover = 1; stopover < number_of_locations; ++stopover) + { + auto row_begin_iterator = result_table->begin() + (curr_node * number_of_locations); + auto row_end_iterator = result_table->begin() + ((curr_node + 1) * number_of_locations); + int min_dist = std::numeric_limits::max(); + int min_id = -1; + + // 2. FIND NEAREST NEIGHBOUR + for (auto it = row_begin_iterator; it != row_end_iterator; ++it) { + auto index = std::distance(row_begin_iterator, it); + if (!visited[index] && *it < min_dist) + { + min_dist = *it; + min_id = index; + } + } + visited[min_id] = true; + subroute = PhantomNodes{phantom_node_vector[curr_node][0], phantom_node_vector[min_id][0]} + raw_route.segment_end_coordinates.emplace_back(subroute); + + SimpleLogger().Write() << "Found location " << curr_node; + SimpleLogger().Write() << "Added a looped via" << curr_node << " " << min_id; + + curr_node = min_id; + } + + // 4. ROUTE BACK TO STARTING POINT + // SimpleLogger().Write() << "Added a final via"; + // subroute = PhantomNodes{raw_route.segment_end_coordinates.back().target_phantom, phantom_node_vector[0][0]}; + // raw_route.segment_end_coordinates.emplace_back(subroute); + + // 5. COMPUTE ROUTE + search_engine_ptr->shortest_path(raw_route.segment_end_coordinates, + route_parameters.uturns, raw_route); + + // return result to json + std::unique_ptr> descriptor; + descriptor = osrm::make_unique>(facade); + + descriptor->SetConfig(route_parameters); + descriptor->Run(raw_route, json_result); + + return 200; + } + +}; + +#endif // ROUND_TRIP_HPP From 00146ae87ca69f6f166b89d59b5a4633bad7ad91 Mon Sep 17 00:00:00 2001 From: chaupow Date: Thu, 23 Apr 2015 16:30:55 +0200 Subject: [PATCH 060/122] add support for locations that are not reachable as well as information about location permutaton --- plugins/round_trip.hpp | 54 ++++++++++++++++++++++++++++++++---------- 1 file changed, 41 insertions(+), 13 deletions(-) diff --git a/plugins/round_trip.hpp b/plugins/round_trip.hpp index 2afe74ace81..be55d4dfc6b 100644 --- a/plugins/round_trip.hpp +++ b/plugins/round_trip.hpp @@ -34,10 +34,12 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "../data_structures/query_edge.hpp" #include "../data_structures/search_engine.hpp" #include "../descriptors/descriptor_base.hpp" +#include "../descriptors/json_descriptor.hpp" #include "../util/json_renderer.hpp" #include "../util/make_unique.hpp" #include "../util/string_util.hpp" #include "../util/timing_util.hpp" +#include "../util/simple_logger.hpp" #include @@ -100,6 +102,7 @@ template class RoundTripPlugin final : public BasePlugin } BOOST_ASSERT(phantom_node_vector[i].front().is_valid(facade->GetNumberOfNodes())); + // SimpleLogger().Write() << "In loop 1"; } // compute the distance table of all phantom nodes @@ -111,7 +114,7 @@ template class RoundTripPlugin final : public BasePlugin return 400; } - SimpleLogger().Write() << "Distance Table Computed"; + // SimpleLogger().Write() << "Distance Table Computed"; ////////////////////////////////////////////////////////////////////////////////////////////////// @@ -128,12 +131,14 @@ template class RoundTripPlugin final : public BasePlugin InternalRouteResult raw_route; // 1. START WITH LOCATION 0 AS START POINT int curr_node = 0; + std::vector loc_permutation(number_of_locations, -1); + loc_permutation[0] = 0; std::vector visited(number_of_locations, false); visited[0] = true; - SimpleLogger().Write() << "Added an initial via"; - SimpleLogger().Write() << "Started from location 0"; - SimpleLogger().Write() << "Number of locs: " << number_of_locations; + // SimpleLogger().Write() << "Added an initial via"; + // SimpleLogger().Write() << "Started from location 0"; + // SimpleLogger().Write() << "Number of locs: " << number_of_locations; PhantomNodes subroute; // 3. REPEAT FOR EVERY UNVISITED NODE @@ -147,26 +152,45 @@ template class RoundTripPlugin final : public BasePlugin // 2. FIND NEAREST NEIGHBOUR for (auto it = row_begin_iterator; it != row_end_iterator; ++it) { auto index = std::distance(row_begin_iterator, it); + if (!visited[index] && *it < min_dist) { min_dist = *it; min_id = index; } + + // SimpleLogger().Write() << "In loop 2"; } - visited[min_id] = true; - subroute = PhantomNodes{phantom_node_vector[curr_node][0], phantom_node_vector[min_id][0]} - raw_route.segment_end_coordinates.emplace_back(subroute); - - SimpleLogger().Write() << "Found location " << curr_node; - SimpleLogger().Write() << "Added a looped via" << curr_node << " " << min_id; + // SimpleLogger().Write() << "After loop 2"; + - curr_node = min_id; + // SimpleLogger().Write() << "visited size is " << visited.size(); + + if (min_id == -1) + { + SimpleLogger().Write() << "ALARM: NO ROUTE!"; + break; + } + else + { + loc_permutation[min_id] = stopover; + visited[min_id] = true; + subroute = PhantomNodes{phantom_node_vector[curr_node][0], phantom_node_vector[min_id][0]}; + raw_route.segment_end_coordinates.emplace_back(subroute); + + // SimpleLogger().Write() << "Found location " << curr_node; + // SimpleLogger().Write() << "Added a looped via" << curr_node << " " << min_id; + + curr_node = min_id; + + // SimpleLogger().Write() << "In loop 3"; + } } // 4. ROUTE BACK TO STARTING POINT // SimpleLogger().Write() << "Added a final via"; - // subroute = PhantomNodes{raw_route.segment_end_coordinates.back().target_phantom, phantom_node_vector[0][0]}; - // raw_route.segment_end_coordinates.emplace_back(subroute); + subroute = PhantomNodes{raw_route.segment_end_coordinates.back().target_phantom, phantom_node_vector[0][0]}; + raw_route.segment_end_coordinates.emplace_back(subroute); // 5. COMPUTE ROUTE search_engine_ptr->shortest_path(raw_route.segment_end_coordinates, @@ -179,6 +203,10 @@ template class RoundTripPlugin final : public BasePlugin descriptor->SetConfig(route_parameters); descriptor->Run(raw_route, json_result); + osrm::json::Array json_loc_permutation; + json_loc_permutation.values.insert(json_loc_permutation.values.end(), loc_permutation.begin(), loc_permutation.end()); + json_result.values["loc_permutation"] = json_loc_permutation; + return 200; } From 108f87678afe182d6cf899f6a48dfa98460dbed4 Mon Sep 17 00:00:00 2001 From: chaupow Date: Thu, 23 Apr 2015 19:35:11 +0200 Subject: [PATCH 061/122] fix bugs and add comments rename subroute to via_point merge is_lonely_island and is_connected to make code easier to understand --- plugins/round_trip.hpp | 163 ++++++++++++++++++++++++----------------- 1 file changed, 97 insertions(+), 66 deletions(-) diff --git a/plugins/round_trip.hpp b/plugins/round_trip.hpp index be55d4dfc6b..3b984623b1b 100644 --- a/plugins/round_trip.hpp +++ b/plugins/round_trip.hpp @@ -98,11 +98,9 @@ template class RoundTripPlugin final : public BasePlugin phantom_node_vector[i], 1); if (phantom_node_vector[i].size() > 1) { - phantom_node_vector[i].pop_back(); + phantom_node_vector[i].erase(phantom_node_vector[i].begin()); } - BOOST_ASSERT(phantom_node_vector[i].front().is_valid(facade->GetNumberOfNodes())); - // SimpleLogger().Write() << "In loop 1"; } // compute the distance table of all phantom nodes @@ -114,9 +112,6 @@ template class RoundTripPlugin final : public BasePlugin return 400; } - // SimpleLogger().Write() << "Distance Table Computed"; - - ////////////////////////////////////////////////////////////////////////////////////////////////// // START GREEDY NEAREST NEIGHBOUR HERE // 1. grab a random location and mark as starting point @@ -124,87 +119,123 @@ template class RoundTripPlugin final : public BasePlugin // 3. repeat 2 until there is no unvisited location // 4. return route back to starting point // 5. compute route + // 6. repeat 1-5 with different starting points and choose iteration with shortest trip // 6. DONE! ////////////////////////////////////////////////////////////////////////////////////////////////// const auto number_of_locations = phantom_node_vector.size(); - InternalRouteResult raw_route; - // 1. START WITH LOCATION 0 AS START POINT - int curr_node = 0; - std::vector loc_permutation(number_of_locations, -1); - loc_permutation[0] = 0; - std::vector visited(number_of_locations, false); - visited[0] = true; - - // SimpleLogger().Write() << "Added an initial via"; - // SimpleLogger().Write() << "Started from location 0"; - // SimpleLogger().Write() << "Number of locs: " << number_of_locations; - - PhantomNodes subroute; - // 3. REPEAT FOR EVERY UNVISITED NODE - for(int stopover = 1; stopover < number_of_locations; ++stopover) + // min_route is the shortest route found + InternalRouteResult min_route; + min_route.shortest_path_length = std::numeric_limits::max(); + // min_loc_permutation stores the order of visited locations of the shortest route + std::vector min_loc_permutation; + + // is_lonely_island[i] indicates whether node i is a node that cannot be reached from other nodes + // 1 means that node i is a lonely island + // 0 means that it is not known for node i + // -1 means that node i is not a lonely island but a reachable, connected node + std::vector is_lonely_island(number_of_locations, 0); + int count_unreachables; + + // ALWAYS START AT ANOTHER STARTING POINT + for(int start_node = 0; start_node < number_of_locations; ++start_node) { - auto row_begin_iterator = result_table->begin() + (curr_node * number_of_locations); - auto row_end_iterator = result_table->begin() + ((curr_node + 1) * number_of_locations); - int min_dist = std::numeric_limits::max(); - int min_id = -1; - - // 2. FIND NEAREST NEIGHBOUR - for (auto it = row_begin_iterator; it != row_end_iterator; ++it) { - auto index = std::distance(row_begin_iterator, it); - - if (!visited[index] && *it < min_dist) - { - min_dist = *it; - min_id = index; + + if (is_lonely_island[start_node] >= 0) + { + // if node is a lonely island it is an unsuitable node to start from and shall be skipped + if (is_lonely_island[start_node]) + continue; + count_unreachables = 0; + auto start_dist_begin = result_table->begin() + (start_node * number_of_locations); + auto start_dist_end = result_table->begin() + ((start_node + 1) * number_of_locations); + for (auto it2 = start_dist_begin; it2 != start_dist_end; ++it2) { + if (*it2 == 0 || *it2 == std::numeric_limits::max()) { + ++count_unreachables; + } + } + if (count_unreachables >= number_of_locations) { + is_lonely_island[start_node] = 1; + continue; } - - // SimpleLogger().Write() << "In loop 2"; } - // SimpleLogger().Write() << "After loop 2"; - - - // SimpleLogger().Write() << "visited size is " << visited.size(); - if (min_id == -1) + int curr_node = start_node; + is_lonely_island[curr_node] = -1; + InternalRouteResult raw_route; + //TODO: Should we always use the same vector or does it not matter at all because of loop scope? + std::vector loc_permutation(number_of_locations, -1); + loc_permutation[start_node] = 0; + // visited[i] indicates whether node i was already visited by the salesman + std::vector visited(number_of_locations, false); + visited[start_node] = true; + + PhantomNodes viapoint; + // 3. REPEAT FOR EVERY UNVISITED NODE + for(int via_point = 1; via_point < number_of_locations; ++via_point) { - SimpleLogger().Write() << "ALARM: NO ROUTE!"; - break; + int min_dist = std::numeric_limits::max(); + int min_id = -1; + + // 2. FIND NEAREST NEIGHBOUR + auto row_begin_iterator = result_table->begin() + (curr_node * number_of_locations); + auto row_end_iterator = result_table->begin() + ((curr_node + 1) * number_of_locations); + for (auto it = row_begin_iterator; it != row_end_iterator; ++it) { + auto index = std::distance(row_begin_iterator, it); + if (is_lonely_island[index] < 1 && !visited[index] && *it < min_dist) + { + min_dist = *it; + min_id = index; + } + } + // in case there was no unvisited and reachable node found, it means that all remaining (unvisited) nodes must be lonely islands + if (min_id == -1) + { + for(int loc = 0; loc < visited.size(); ++loc) { + if (!visited[loc]) { + is_lonely_island[loc] = 1; + } + } + break; + } + // set the nearest unvisited location as the next via_point + else + { + is_lonely_island[min_id] = -1; + loc_permutation[min_id] = via_point; + visited[min_id] = true; + viapoint = PhantomNodes{phantom_node_vector[curr_node][0], phantom_node_vector[min_id][0]}; + raw_route.segment_end_coordinates.emplace_back(viapoint); + curr_node = min_id; + } } - else - { - loc_permutation[min_id] = stopover; - visited[min_id] = true; - subroute = PhantomNodes{phantom_node_vector[curr_node][0], phantom_node_vector[min_id][0]}; - raw_route.segment_end_coordinates.emplace_back(subroute); - - // SimpleLogger().Write() << "Found location " << curr_node; - // SimpleLogger().Write() << "Added a looped via" << curr_node << " " << min_id; - - curr_node = min_id; - - // SimpleLogger().Write() << "In loop 3"; + + // 4. ROUTE BACK TO STARTING POINT + viapoint = PhantomNodes{raw_route.segment_end_coordinates.back().target_phantom, phantom_node_vector[start_node][0]}; + raw_route.segment_end_coordinates.emplace_back(viapoint); + + // 5. COMPUTE ROUTE + search_engine_ptr->shortest_path(raw_route.segment_end_coordinates, route_parameters.uturns, raw_route); + // SimpleLogger().Write() << "Route starting at " << start_node << " with length " << raw_route.shortest_path_length; + + // check round trip with this starting point is shorter than the shortest round trip found till now + if (raw_route.shortest_path_length < min_route.shortest_path_length) { + min_route = raw_route; + min_loc_permutation = loc_permutation; } } - // 4. ROUTE BACK TO STARTING POINT - // SimpleLogger().Write() << "Added a final via"; - subroute = PhantomNodes{raw_route.segment_end_coordinates.back().target_phantom, phantom_node_vector[0][0]}; - raw_route.segment_end_coordinates.emplace_back(subroute); - - // 5. COMPUTE ROUTE - search_engine_ptr->shortest_path(raw_route.segment_end_coordinates, - route_parameters.uturns, raw_route); + SimpleLogger().Write() << "Shortest route " << min_route.shortest_path_length; // return result to json std::unique_ptr> descriptor; descriptor = osrm::make_unique>(facade); descriptor->SetConfig(route_parameters); - descriptor->Run(raw_route, json_result); + descriptor->Run(min_route, json_result); osrm::json::Array json_loc_permutation; - json_loc_permutation.values.insert(json_loc_permutation.values.end(), loc_permutation.begin(), loc_permutation.end()); + json_loc_permutation.values.insert(json_loc_permutation.values.end(), min_loc_permutation.begin(), min_loc_permutation.end()); json_result.values["loc_permutation"] = json_loc_permutation; return 200; From b570e89dbd91752c6e68b65e6f08ef7cbdc8bf01 Mon Sep 17 00:00:00 2001 From: chaupow Date: Wed, 27 May 2015 01:32:59 +0200 Subject: [PATCH 062/122] capsule tsp round trip computation in a private method --- plugins/round_trip.hpp | 130 +++++++++++++++++++++-------------------- 1 file changed, 68 insertions(+), 62 deletions(-) diff --git a/plugins/round_trip.hpp b/plugins/round_trip.hpp index 3b984623b1b..e084007f1cb 100644 --- a/plugins/round_trip.hpp +++ b/plugins/round_trip.hpp @@ -59,59 +59,11 @@ template class RoundTripPlugin final : public BasePlugin DataFacadeT *facade; std::unique_ptr> search_engine_ptr; - public: - explicit RoundTripPlugin(DataFacadeT *facade) - : descriptor_string("trip"), facade(facade) - { - search_engine_ptr = osrm::make_unique>(facade); - } - - const std::string GetDescriptor() const override final { return descriptor_string; } - - int HandleRequest(const RouteParameters &route_parameters, - osrm::json::Object &json_result) override final - { - // check if all inputs are coordinates - if (!check_all_coordinates(route_parameters.coordinates)) - { - return 400; - } - const bool checksum_OK = (route_parameters.check_sum == facade->GetCheckSum()); - - // find phantom nodes for all input coords - PhantomNodeArray phantom_node_vector(route_parameters.coordinates.size()); - for (const auto i : osrm::irange(0, route_parameters.coordinates.size())) - { - // if client hints are helpful, encode hints - if (checksum_OK && i < route_parameters.hints.size() && - !route_parameters.hints[i].empty()) - { - PhantomNode current_phantom_node; - ObjectEncoder::DecodeFromBase64(route_parameters.hints[i], current_phantom_node); - if (current_phantom_node.is_valid(facade->GetNumberOfNodes())) - { - phantom_node_vector[i].emplace_back(std::move(current_phantom_node)); - continue; - } - } - facade->IncrementalFindPhantomNodeForCoordinate(route_parameters.coordinates[i], - phantom_node_vector[i], 1); - if (phantom_node_vector[i].size() > 1) - { - phantom_node_vector[i].erase(phantom_node_vector[i].begin()); - } - BOOST_ASSERT(phantom_node_vector[i].front().is_valid(facade->GetNumberOfNodes())); - } - - // compute the distance table of all phantom nodes - std::shared_ptr> result_table = - search_engine_ptr->distance_table(phantom_node_vector); - - if (!result_table) - { - return 400; - } - + void NearestNeighbour(const RouteParameters & route_parameters, + const PhantomNodeArray & phantom_node_vector, + std::vector & result_table, + InternalRouteResult & min_route, + std::vector & min_loc_permutation) { ////////////////////////////////////////////////////////////////////////////////////////////////// // START GREEDY NEAREST NEIGHBOUR HERE // 1. grab a random location and mark as starting point @@ -124,11 +76,7 @@ template class RoundTripPlugin final : public BasePlugin ////////////////////////////////////////////////////////////////////////////////////////////////// const auto number_of_locations = phantom_node_vector.size(); - // min_route is the shortest route found - InternalRouteResult min_route; min_route.shortest_path_length = std::numeric_limits::max(); - // min_loc_permutation stores the order of visited locations of the shortest route - std::vector min_loc_permutation; // is_lonely_island[i] indicates whether node i is a node that cannot be reached from other nodes // 1 means that node i is a lonely island @@ -147,8 +95,8 @@ template class RoundTripPlugin final : public BasePlugin if (is_lonely_island[start_node]) continue; count_unreachables = 0; - auto start_dist_begin = result_table->begin() + (start_node * number_of_locations); - auto start_dist_end = result_table->begin() + ((start_node + 1) * number_of_locations); + auto start_dist_begin = result_table.begin() + (start_node * number_of_locations); + auto start_dist_end = result_table.begin() + ((start_node + 1) * number_of_locations); for (auto it2 = start_dist_begin; it2 != start_dist_end; ++it2) { if (*it2 == 0 || *it2 == std::numeric_limits::max()) { ++count_unreachables; @@ -178,8 +126,8 @@ template class RoundTripPlugin final : public BasePlugin int min_id = -1; // 2. FIND NEAREST NEIGHBOUR - auto row_begin_iterator = result_table->begin() + (curr_node * number_of_locations); - auto row_end_iterator = result_table->begin() + ((curr_node + 1) * number_of_locations); + auto row_begin_iterator = result_table.begin() + (curr_node * number_of_locations); + auto row_end_iterator = result_table.begin() + ((curr_node + 1) * number_of_locations); for (auto it = row_begin_iterator; it != row_end_iterator; ++it) { auto index = std::distance(row_begin_iterator, it); if (is_lonely_island[index] < 1 && !visited[index] && *it < min_dist) @@ -216,7 +164,6 @@ template class RoundTripPlugin final : public BasePlugin // 5. COMPUTE ROUTE search_engine_ptr->shortest_path(raw_route.segment_end_coordinates, route_parameters.uturns, raw_route); - // SimpleLogger().Write() << "Route starting at " << start_node << " with length " << raw_route.shortest_path_length; // check round trip with this starting point is shorter than the shortest round trip found till now if (raw_route.shortest_path_length < min_route.shortest_path_length) { @@ -226,6 +173,65 @@ template class RoundTripPlugin final : public BasePlugin } SimpleLogger().Write() << "Shortest route " << min_route.shortest_path_length; + } + + public: + explicit RoundTripPlugin(DataFacadeT *facade) + : descriptor_string("trip"), facade(facade) + { + search_engine_ptr = osrm::make_unique>(facade); + } + + const std::string GetDescriptor() const override final { return descriptor_string; } + + int HandleRequest(const RouteParameters &route_parameters, + osrm::json::Object &json_result) override final + { + // check if all inputs are coordinates + if (!check_all_coordinates(route_parameters.coordinates)) + { + return 400; + } + const bool checksum_OK = (route_parameters.check_sum == facade->GetCheckSum()); + + // find phantom nodes for all input coords + PhantomNodeArray phantom_node_vector(route_parameters.coordinates.size()); + for (const auto i : osrm::irange(0, route_parameters.coordinates.size())) + { + // if client hints are helpful, encode hints + if (checksum_OK && i < route_parameters.hints.size() && + !route_parameters.hints[i].empty()) + { + PhantomNode current_phantom_node; + ObjectEncoder::DecodeFromBase64(route_parameters.hints[i], current_phantom_node); + if (current_phantom_node.is_valid(facade->GetNumberOfNodes())) + { + phantom_node_vector[i].emplace_back(std::move(current_phantom_node)); + continue; + } + } + facade->IncrementalFindPhantomNodeForCoordinate(route_parameters.coordinates[i], + phantom_node_vector[i], 1); + if (phantom_node_vector[i].size() > 1) + { + phantom_node_vector[i].erase(phantom_node_vector[i].begin()); + } + BOOST_ASSERT(phantom_node_vector[i].front().is_valid(facade->GetNumberOfNodes())); + } + + // compute the distance table of all phantom nodes + const std::shared_ptr> result_table = + search_engine_ptr->distance_table(phantom_node_vector); + + if (!result_table) + { + return 400; + } + + // compute TSP round trip + InternalRouteResult min_route; + std::vector min_loc_permutation; + NearestNeighbour(route_parameters, phantom_node_vector, *result_table, min_route, min_loc_permutation); // return result to json std::unique_ptr> descriptor; From a2dc8378f52249dea48f4ebaad3888f862447e40 Mon Sep 17 00:00:00 2001 From: chaupow Date: Wed, 27 May 2015 01:36:40 +0200 Subject: [PATCH 063/122] rename result_table to dist_table --- plugins/round_trip.hpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/plugins/round_trip.hpp b/plugins/round_trip.hpp index e084007f1cb..ba83397dba0 100644 --- a/plugins/round_trip.hpp +++ b/plugins/round_trip.hpp @@ -61,7 +61,7 @@ template class RoundTripPlugin final : public BasePlugin void NearestNeighbour(const RouteParameters & route_parameters, const PhantomNodeArray & phantom_node_vector, - std::vector & result_table, + std::vector & dist_table, InternalRouteResult & min_route, std::vector & min_loc_permutation) { ////////////////////////////////////////////////////////////////////////////////////////////////// @@ -95,8 +95,8 @@ template class RoundTripPlugin final : public BasePlugin if (is_lonely_island[start_node]) continue; count_unreachables = 0; - auto start_dist_begin = result_table.begin() + (start_node * number_of_locations); - auto start_dist_end = result_table.begin() + ((start_node + 1) * number_of_locations); + auto start_dist_begin = dist_table.begin() + (start_node * number_of_locations); + auto start_dist_end = dist_table.begin() + ((start_node + 1) * number_of_locations); for (auto it2 = start_dist_begin; it2 != start_dist_end; ++it2) { if (*it2 == 0 || *it2 == std::numeric_limits::max()) { ++count_unreachables; @@ -126,8 +126,8 @@ template class RoundTripPlugin final : public BasePlugin int min_id = -1; // 2. FIND NEAREST NEIGHBOUR - auto row_begin_iterator = result_table.begin() + (curr_node * number_of_locations); - auto row_end_iterator = result_table.begin() + ((curr_node + 1) * number_of_locations); + auto row_begin_iterator = dist_table.begin() + (curr_node * number_of_locations); + auto row_end_iterator = dist_table.begin() + ((curr_node + 1) * number_of_locations); for (auto it = row_begin_iterator; it != row_end_iterator; ++it) { auto index = std::distance(row_begin_iterator, it); if (is_lonely_island[index] < 1 && !visited[index] && *it < min_dist) From ca7d4067871fdccb5e6c1b6615a0c6f296bb02e1 Mon Sep 17 00:00:00 2001 From: chaupow Date: Wed, 27 May 2015 01:46:12 +0200 Subject: [PATCH 064/122] add timer to check runtime of round trip algorithm --- plugins/round_trip.hpp | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/plugins/round_trip.hpp b/plugins/round_trip.hpp index ba83397dba0..94b6089fe40 100644 --- a/plugins/round_trip.hpp +++ b/plugins/round_trip.hpp @@ -171,8 +171,6 @@ template class RoundTripPlugin final : public BasePlugin min_loc_permutation = loc_permutation; } } - - SimpleLogger().Write() << "Shortest route " << min_route.shortest_path_length; } public: @@ -231,7 +229,12 @@ template class RoundTripPlugin final : public BasePlugin // compute TSP round trip InternalRouteResult min_route; std::vector min_loc_permutation; + TIMER_START(tsp_nn); NearestNeighbour(route_parameters, phantom_node_vector, *result_table, min_route, min_loc_permutation); + TIMER_STOP(tsp_nn); + + SimpleLogger().Write() << "Distance " << min_route.shortest_path_length; + SimpleLogger().Write() << "Time " << TIMER_MSEC(tsp_nn); // return result to json std::unique_ptr> descriptor; @@ -242,7 +245,9 @@ template class RoundTripPlugin final : public BasePlugin osrm::json::Array json_loc_permutation; json_loc_permutation.values.insert(json_loc_permutation.values.end(), min_loc_permutation.begin(), min_loc_permutation.end()); - json_result.values["loc_permutation"] = json_loc_permutation; + json_result.values["nn_loc_permutation"] = json_loc_permutation; + json_result.values["nn_distance"] = min_route.shortest_path_length; + json_result.values["nn_runtime"] = TIMER_MSEC(tsp_nn); return 200; } From ebbe1692c81549930e3b48a064ca0c98d8658c79 Mon Sep 17 00:00:00 2001 From: chaupow Date: Wed, 27 May 2015 11:47:48 +0200 Subject: [PATCH 065/122] add description of farthest insertion algorithm add farthest insertion algorithm for round trip farthest insertion: always add the node that add the biggest distance to the total route farthest insertion: remove total distance computation and compute only diff instead --- plugins/round_trip.hpp | 167 ++++++++++++++++++++++++++++++++++++----- 1 file changed, 148 insertions(+), 19 deletions(-) diff --git a/plugins/round_trip.hpp b/plugins/round_trip.hpp index 94b6089fe40..85d1f2057b8 100644 --- a/plugins/round_trip.hpp +++ b/plugins/round_trip.hpp @@ -50,7 +50,7 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include #include #include -#include +#include template class RoundTripPlugin final : public BasePlugin { @@ -59,9 +59,112 @@ template class RoundTripPlugin final : public BasePlugin DataFacadeT *facade; std::unique_ptr> search_engine_ptr; + void FarthestInsertion(const RouteParameters & route_parameters, + const PhantomNodeArray & phantom_node_vector, + const std::vector & dist_table, + InternalRouteResult & min_route, + std::vector & min_loc_permutation) { + ////////////////////////////////////////////////////////////////////////////////////////////////// + // START FARTHEST INSERTION HERE + // 1. start at a random round trip of 2 locations + // 2. find the location that is the farthest away from the visited locations + // 3. add the found location to the current round trip such that round trip is the shortest + // 4. repeat 2-3 until all locations are visited + // 5. DONE! + ////////////////////////////////////////////////////////////////////////////////////////////////// + + const auto number_of_locations = phantom_node_vector.size(); + std::list current_trip; + std::vector visited(number_of_locations, false); + + // find two locations that have max distance + auto max_dist = -1; + int max_from = -1; + int max_to = -1; + + auto i = 0; + for (auto it = dist_table.begin(); it != dist_table.end(); ++it) { + if (*it > max_dist) { + max_dist = *it; + max_from = i / number_of_locations; + max_to = i % number_of_locations; + } + ++i; + } + + visited[max_from] = true; + visited[max_to] = true; + + // SimpleLogger().Write() << "Start with " << max_from << " " << max_to; + + current_trip.push_back(max_from); + current_trip.push_back(max_to); + + for (int j = 2; j < number_of_locations; ++j) { + auto max_min_dist = -1; + int next_node = -1; + auto min_max_insert = current_trip.begin(); + + // look for loc i that is the farthest away from all other visited locs + for (int i = 0; i < number_of_locations; ++i) { + if (!visited[i]) { + // SimpleLogger().Write() << "- node " << i << " is not visited yet"; + + auto min_insert = std::numeric_limits::max(); + auto min_to = current_trip.begin(); + + for (auto from_node = current_trip.begin(); from_node != current_trip.end(); ++from_node) { + + auto to_node = std::next(from_node); + if (std::next(from_node) == current_trip.end()) { + to_node = current_trip.begin(); + } + + auto dist_from = *(dist_table.begin() + (*from_node * number_of_locations) + i); + auto dist_to = *(dist_table.begin() + (i * number_of_locations) + *to_node); + auto trip_dist = dist_from + dist_to - *(dist_table.begin() + (*from_node * number_of_locations) + *to_node); + + // SimpleLogger().Write() << " From " << *from_node << " to " << i << " to " << *to_node << " is " << trip_dist; + + if (trip_dist < min_insert) { + min_insert = trip_dist; + min_to = to_node; + } + } + if (min_insert > max_min_dist) { + max_min_dist = min_insert; + next_node = i; + min_max_insert = min_to; + } + } + } + // SimpleLogger().Write() << "- Insert new node " << next_node; + visited[next_node] = true; + + current_trip.insert(min_max_insert, next_node); + } + + int perm = 0; + for (auto it = current_trip.begin(); it != current_trip.end(); ++it) { + // SimpleLogger().Write() << "- Visit location " << *it; + + auto from_node = *it; + auto to_node = *std::next(it); + if (std::next(it) == current_trip.end()) { + to_node = current_trip.front(); + } + PhantomNodes viapoint; + viapoint = PhantomNodes{phantom_node_vector[from_node][0], phantom_node_vector[to_node][0]}; + min_route.segment_end_coordinates.emplace_back(viapoint); + min_loc_permutation[from_node] = perm; + ++perm; + } + search_engine_ptr->shortest_path(min_route.segment_end_coordinates, route_parameters.uturns, min_route); + } + void NearestNeighbour(const RouteParameters & route_parameters, const PhantomNodeArray & phantom_node_vector, - std::vector & dist_table, + const std::vector & dist_table, InternalRouteResult & min_route, std::vector & min_loc_permutation) { ////////////////////////////////////////////////////////////////////////////////////////////////// @@ -72,7 +175,7 @@ template class RoundTripPlugin final : public BasePlugin // 4. return route back to starting point // 5. compute route // 6. repeat 1-5 with different starting points and choose iteration with shortest trip - // 6. DONE! + // 7. DONE! ////////////////////////////////////////////////////////////////////////////////////////////////// const auto number_of_locations = phantom_node_vector.size(); @@ -88,7 +191,7 @@ template class RoundTripPlugin final : public BasePlugin // ALWAYS START AT ANOTHER STARTING POINT for(int start_node = 0; start_node < number_of_locations; ++start_node) { - + if (is_lonely_island[start_node] >= 0) { // if node is a lonely island it is an unsuitable node to start from and shall be skipped @@ -108,7 +211,7 @@ template class RoundTripPlugin final : public BasePlugin } } - int curr_node = start_node; + int curr_node = start_node; is_lonely_island[curr_node] = -1; InternalRouteResult raw_route; //TODO: Should we always use the same vector or does it not matter at all because of loop scope? @@ -129,7 +232,7 @@ template class RoundTripPlugin final : public BasePlugin auto row_begin_iterator = dist_table.begin() + (curr_node * number_of_locations); auto row_end_iterator = dist_table.begin() + ((curr_node + 1) * number_of_locations); for (auto it = row_begin_iterator; it != row_end_iterator; ++it) { - auto index = std::distance(row_begin_iterator, it); + auto index = std::distance(row_begin_iterator, it); if (is_lonely_island[index] < 1 && !visited[index] && *it < min_dist) { min_dist = *it; @@ -164,7 +267,7 @@ template class RoundTripPlugin final : public BasePlugin // 5. COMPUTE ROUTE search_engine_ptr->shortest_path(raw_route.segment_end_coordinates, route_parameters.uturns, raw_route); - + // check round trip with this starting point is shorter than the shortest round trip found till now if (raw_route.shortest_path_length < min_route.shortest_path_length) { min_route = raw_route; @@ -185,6 +288,7 @@ template class RoundTripPlugin final : public BasePlugin int HandleRequest(const RouteParameters &route_parameters, osrm::json::Object &json_result) override final { + TIMER_START(tsp_pre); // check if all inputs are coordinates if (!check_all_coordinates(route_parameters.coordinates)) { @@ -227,27 +331,52 @@ template class RoundTripPlugin final : public BasePlugin } // compute TSP round trip - InternalRouteResult min_route; - std::vector min_loc_permutation; + InternalRouteResult min_route_nn; + InternalRouteResult min_route_fi; + std::vector min_loc_permutation_nn(phantom_node_vector.size(), -1); + std::vector min_loc_permutation_fi(phantom_node_vector.size(), -1); + TIMER_STOP(tsp_pre); + TIMER_START(tsp_nn); - NearestNeighbour(route_parameters, phantom_node_vector, *result_table, min_route, min_loc_permutation); + NearestNeighbour(route_parameters, phantom_node_vector, *result_table, min_route_nn, min_loc_permutation_nn); TIMER_STOP(tsp_nn); + SimpleLogger().Write() << "Distance " << min_route_nn.shortest_path_length; + SimpleLogger().Write() << "Time " << TIMER_MSEC(tsp_nn) + TIMER_MSEC(tsp_pre); - SimpleLogger().Write() << "Distance " << min_route.shortest_path_length; - SimpleLogger().Write() << "Time " << TIMER_MSEC(tsp_nn); + // std::unique_ptr> descriptor; + // descriptor = osrm::make_unique>(facade); + + // descriptor->SetConfig(route_parameters); + // descriptor->Run(min_route_nn, json_result); + + osrm::json::Array json_loc_permutation_nn; + json_loc_permutation_nn.values.insert(json_loc_permutation_nn.values.end(), min_loc_permutation_nn.begin(), min_loc_permutation_nn.end()); + json_result.values["nn_loc_permutation"] = json_loc_permutation_nn; + json_result.values["nn_distance"] = min_route_nn.shortest_path_length; + json_result.values["nn_runtime"] = TIMER_MSEC(tsp_nn) + TIMER_MSEC(tsp_pre); + + TIMER_START(tsp_fi); + FarthestInsertion(route_parameters, phantom_node_vector, *result_table, min_route_fi, min_loc_permutation_fi); + TIMER_STOP(tsp_fi); + SimpleLogger().Write() << "Distance " << min_route_fi.shortest_path_length; + SimpleLogger().Write() << "Time " << TIMER_MSEC(tsp_fi) + TIMER_MSEC(tsp_pre); // return result to json std::unique_ptr> descriptor; descriptor = osrm::make_unique>(facade); - + descriptor->SetConfig(route_parameters); - descriptor->Run(min_route, json_result); + descriptor->Run(min_route_fi, json_result); + + osrm::json::Array json_loc_permutation_fi; + json_loc_permutation_fi.values.insert(json_loc_permutation_fi.values.end(), min_loc_permutation_fi.begin(), min_loc_permutation_fi.end()); + json_result.values["fi_loc_permutation"] = json_loc_permutation_fi; + json_result.values["fi_distance"] = min_route_fi.shortest_path_length; + json_result.values["fi_runtime"] = TIMER_MSEC(tsp_fi) + TIMER_MSEC(tsp_pre); - osrm::json::Array json_loc_permutation; - json_loc_permutation.values.insert(json_loc_permutation.values.end(), min_loc_permutation.begin(), min_loc_permutation.end()); - json_result.values["nn_loc_permutation"] = json_loc_permutation; - json_result.values["nn_distance"] = min_route.shortest_path_length; - json_result.values["nn_runtime"] = TIMER_MSEC(tsp_nn); + // for (int i = 0; i < min_loc_permutation_fi.size(); ++i) { + // SimpleLogger().Write() << min_loc_permutation_nn[i] << " " << min_loc_permutation_fi[i]; + // } return 200; } From d3ebd360b2d8903034dc79ef25fd7431730bf924 Mon Sep 17 00:00:00 2001 From: Chau Nguyen Date: Tue, 16 Jun 2015 23:19:51 +0200 Subject: [PATCH 066/122] add brute force algorithm for tsp for small tests --- routing_algorithms/tsp_brute_force.hpp | 89 ++++++++++++++++++++++++++ 1 file changed, 89 insertions(+) create mode 100644 routing_algorithms/tsp_brute_force.hpp diff --git a/routing_algorithms/tsp_brute_force.hpp b/routing_algorithms/tsp_brute_force.hpp new file mode 100644 index 00000000000..099ed4e4a8c --- /dev/null +++ b/routing_algorithms/tsp_brute_force.hpp @@ -0,0 +1,89 @@ +/* + +Copyright (c) 2015, Project OSRM contributors +All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + +Redistributions of source code must retain the above copyright notice, this list +of conditions and the following disclaimer. +Redistributions in binary form must reproduce the above copyright notice, this +list of conditions and the following disclaimer in the documentation and/or +other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +*/ + +#ifndef TSP_BRUTE_FORCE_HPP +#define TSP_BRUTE_FORCE_HPP + + +#include "../data_structures/search_engine.hpp" +#include "../util/string_util.hpp" + +#include + +#include + +#include +#include +#include +#include +#include "../util/simple_logger.hpp" + + +// HAHAHA. NICE TRY, CHAU. + + +namespace osrm +{ +namespace tsp +{ +void BruteForce(const RouteParameters & route_parameters, + const PhantomNodeArray & phantom_node_vector, + const std::vector & dist_table, + InternalRouteResult & min_route, + std::vector & min_loc_permutation) { + + const auto number_of_locations = phantom_node_vector.size(); + std::vector location_ids(number_of_locations); + std::iota(location_ids.begin(), location_ids.end(), 0); + + int min_route_dist = std::numeric_limits::max(); + do { + int route_dist = 0; + for (int i = 0; i < number_of_locations - 1; ++i) { + route_dist += *(dist_table.begin() + (location_ids[i] * number_of_locations) + location_ids[i+1]); + } + route_dist += *(dist_table.begin() + (location_ids[number_of_locations-1] * number_of_locations) + location_ids[0]); + + if (route_dist < min_route_dist) { + min_route_dist = route_dist; + //TODO: this gets copied right? fix this + min_loc_permutation = location_ids; + } + } while(std::next_permutation(location_ids.begin(), location_ids.end())); + + PhantomNodes viapoint; + for (int i = 0; i < number_of_locations - 1; ++i) { + viapoint = PhantomNodes{phantom_node_vector[i][0], phantom_node_vector[i + 1][0]}; + min_route.segment_end_coordinates.emplace_back(viapoint); + } + viapoint = PhantomNodes{phantom_node_vector[number_of_locations - 1][0], phantom_node_vector[0][0]}; + min_route.segment_end_coordinates.emplace_back(viapoint); +} + +} +} +#endif // TSP_BRUTE_FORCE_HPP \ No newline at end of file From f0d66ff0fb184b0b848a41a6d13a0d5d8404037a Mon Sep 17 00:00:00 2001 From: Chau Nguyen Date: Tue, 16 Jun 2015 23:20:38 +0200 Subject: [PATCH 067/122] move implementation of algorithms to own hpp in routing_algorithms folder add changes to improve readability --- plugins/round_trip.hpp | 275 +++--------------- routing_algorithms/tsp_farthest_insertion.hpp | 159 ++++++++++ routing_algorithms/tsp_nearest_neighbour.hpp | 168 +++++++++++ 3 files changed, 367 insertions(+), 235 deletions(-) create mode 100644 routing_algorithms/tsp_farthest_insertion.hpp create mode 100644 routing_algorithms/tsp_nearest_neighbour.hpp diff --git a/plugins/round_trip.hpp b/plugins/round_trip.hpp index 85d1f2057b8..652e82f3b84 100644 --- a/plugins/round_trip.hpp +++ b/plugins/round_trip.hpp @@ -31,6 +31,9 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "plugin_base.hpp" #include "../algorithms/object_encoder.hpp" +#include "../routing_algorithms/tsp_nearest_neighbour.hpp" +#include "../routing_algorithms/tsp_farthest_insertion.hpp" +#include "../routing_algorithms/tsp_brute_force.hpp" #include "../data_structures/query_edge.hpp" #include "../data_structures/search_engine.hpp" #include "../descriptors/descriptor_base.hpp" @@ -44,7 +47,6 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include #include - #include #include #include @@ -59,223 +61,6 @@ template class RoundTripPlugin final : public BasePlugin DataFacadeT *facade; std::unique_ptr> search_engine_ptr; - void FarthestInsertion(const RouteParameters & route_parameters, - const PhantomNodeArray & phantom_node_vector, - const std::vector & dist_table, - InternalRouteResult & min_route, - std::vector & min_loc_permutation) { - ////////////////////////////////////////////////////////////////////////////////////////////////// - // START FARTHEST INSERTION HERE - // 1. start at a random round trip of 2 locations - // 2. find the location that is the farthest away from the visited locations - // 3. add the found location to the current round trip such that round trip is the shortest - // 4. repeat 2-3 until all locations are visited - // 5. DONE! - ////////////////////////////////////////////////////////////////////////////////////////////////// - - const auto number_of_locations = phantom_node_vector.size(); - std::list current_trip; - std::vector visited(number_of_locations, false); - - // find two locations that have max distance - auto max_dist = -1; - int max_from = -1; - int max_to = -1; - - auto i = 0; - for (auto it = dist_table.begin(); it != dist_table.end(); ++it) { - if (*it > max_dist) { - max_dist = *it; - max_from = i / number_of_locations; - max_to = i % number_of_locations; - } - ++i; - } - - visited[max_from] = true; - visited[max_to] = true; - - // SimpleLogger().Write() << "Start with " << max_from << " " << max_to; - - current_trip.push_back(max_from); - current_trip.push_back(max_to); - - for (int j = 2; j < number_of_locations; ++j) { - auto max_min_dist = -1; - int next_node = -1; - auto min_max_insert = current_trip.begin(); - - // look for loc i that is the farthest away from all other visited locs - for (int i = 0; i < number_of_locations; ++i) { - if (!visited[i]) { - // SimpleLogger().Write() << "- node " << i << " is not visited yet"; - - auto min_insert = std::numeric_limits::max(); - auto min_to = current_trip.begin(); - - for (auto from_node = current_trip.begin(); from_node != current_trip.end(); ++from_node) { - - auto to_node = std::next(from_node); - if (std::next(from_node) == current_trip.end()) { - to_node = current_trip.begin(); - } - - auto dist_from = *(dist_table.begin() + (*from_node * number_of_locations) + i); - auto dist_to = *(dist_table.begin() + (i * number_of_locations) + *to_node); - auto trip_dist = dist_from + dist_to - *(dist_table.begin() + (*from_node * number_of_locations) + *to_node); - - // SimpleLogger().Write() << " From " << *from_node << " to " << i << " to " << *to_node << " is " << trip_dist; - - if (trip_dist < min_insert) { - min_insert = trip_dist; - min_to = to_node; - } - } - if (min_insert > max_min_dist) { - max_min_dist = min_insert; - next_node = i; - min_max_insert = min_to; - } - } - } - // SimpleLogger().Write() << "- Insert new node " << next_node; - visited[next_node] = true; - - current_trip.insert(min_max_insert, next_node); - } - - int perm = 0; - for (auto it = current_trip.begin(); it != current_trip.end(); ++it) { - // SimpleLogger().Write() << "- Visit location " << *it; - - auto from_node = *it; - auto to_node = *std::next(it); - if (std::next(it) == current_trip.end()) { - to_node = current_trip.front(); - } - PhantomNodes viapoint; - viapoint = PhantomNodes{phantom_node_vector[from_node][0], phantom_node_vector[to_node][0]}; - min_route.segment_end_coordinates.emplace_back(viapoint); - min_loc_permutation[from_node] = perm; - ++perm; - } - search_engine_ptr->shortest_path(min_route.segment_end_coordinates, route_parameters.uturns, min_route); - } - - void NearestNeighbour(const RouteParameters & route_parameters, - const PhantomNodeArray & phantom_node_vector, - const std::vector & dist_table, - InternalRouteResult & min_route, - std::vector & min_loc_permutation) { - ////////////////////////////////////////////////////////////////////////////////////////////////// - // START GREEDY NEAREST NEIGHBOUR HERE - // 1. grab a random location and mark as starting point - // 2. find the nearest unvisited neighbour, set it as the current location and mark as visited - // 3. repeat 2 until there is no unvisited location - // 4. return route back to starting point - // 5. compute route - // 6. repeat 1-5 with different starting points and choose iteration with shortest trip - // 7. DONE! - ////////////////////////////////////////////////////////////////////////////////////////////////// - - const auto number_of_locations = phantom_node_vector.size(); - min_route.shortest_path_length = std::numeric_limits::max(); - - // is_lonely_island[i] indicates whether node i is a node that cannot be reached from other nodes - // 1 means that node i is a lonely island - // 0 means that it is not known for node i - // -1 means that node i is not a lonely island but a reachable, connected node - std::vector is_lonely_island(number_of_locations, 0); - int count_unreachables; - - // ALWAYS START AT ANOTHER STARTING POINT - for(int start_node = 0; start_node < number_of_locations; ++start_node) - { - - if (is_lonely_island[start_node] >= 0) - { - // if node is a lonely island it is an unsuitable node to start from and shall be skipped - if (is_lonely_island[start_node]) - continue; - count_unreachables = 0; - auto start_dist_begin = dist_table.begin() + (start_node * number_of_locations); - auto start_dist_end = dist_table.begin() + ((start_node + 1) * number_of_locations); - for (auto it2 = start_dist_begin; it2 != start_dist_end; ++it2) { - if (*it2 == 0 || *it2 == std::numeric_limits::max()) { - ++count_unreachables; - } - } - if (count_unreachables >= number_of_locations) { - is_lonely_island[start_node] = 1; - continue; - } - } - - int curr_node = start_node; - is_lonely_island[curr_node] = -1; - InternalRouteResult raw_route; - //TODO: Should we always use the same vector or does it not matter at all because of loop scope? - std::vector loc_permutation(number_of_locations, -1); - loc_permutation[start_node] = 0; - // visited[i] indicates whether node i was already visited by the salesman - std::vector visited(number_of_locations, false); - visited[start_node] = true; - - PhantomNodes viapoint; - // 3. REPEAT FOR EVERY UNVISITED NODE - for(int via_point = 1; via_point < number_of_locations; ++via_point) - { - int min_dist = std::numeric_limits::max(); - int min_id = -1; - - // 2. FIND NEAREST NEIGHBOUR - auto row_begin_iterator = dist_table.begin() + (curr_node * number_of_locations); - auto row_end_iterator = dist_table.begin() + ((curr_node + 1) * number_of_locations); - for (auto it = row_begin_iterator; it != row_end_iterator; ++it) { - auto index = std::distance(row_begin_iterator, it); - if (is_lonely_island[index] < 1 && !visited[index] && *it < min_dist) - { - min_dist = *it; - min_id = index; - } - } - // in case there was no unvisited and reachable node found, it means that all remaining (unvisited) nodes must be lonely islands - if (min_id == -1) - { - for(int loc = 0; loc < visited.size(); ++loc) { - if (!visited[loc]) { - is_lonely_island[loc] = 1; - } - } - break; - } - // set the nearest unvisited location as the next via_point - else - { - is_lonely_island[min_id] = -1; - loc_permutation[min_id] = via_point; - visited[min_id] = true; - viapoint = PhantomNodes{phantom_node_vector[curr_node][0], phantom_node_vector[min_id][0]}; - raw_route.segment_end_coordinates.emplace_back(viapoint); - curr_node = min_id; - } - } - - // 4. ROUTE BACK TO STARTING POINT - viapoint = PhantomNodes{raw_route.segment_end_coordinates.back().target_phantom, phantom_node_vector[start_node][0]}; - raw_route.segment_end_coordinates.emplace_back(viapoint); - - // 5. COMPUTE ROUTE - search_engine_ptr->shortest_path(raw_route.segment_end_coordinates, route_parameters.uturns, raw_route); - - // check round trip with this starting point is shorter than the shortest round trip found till now - if (raw_route.shortest_path_length < min_route.shortest_path_length) { - min_route = raw_route; - min_loc_permutation = loc_permutation; - } - } - } - public: explicit RoundTripPlugin(DataFacadeT *facade) : descriptor_string("trip"), facade(facade) @@ -333,50 +118,70 @@ template class RoundTripPlugin final : public BasePlugin // compute TSP round trip InternalRouteResult min_route_nn; InternalRouteResult min_route_fi; + InternalRouteResult min_route_bf; std::vector min_loc_permutation_nn(phantom_node_vector.size(), -1); std::vector min_loc_permutation_fi(phantom_node_vector.size(), -1); + std::vector min_loc_permutation_bf(phantom_node_vector.size(), -1); TIMER_STOP(tsp_pre); + + //######################### NEAREST NEIGHBOUR ###############################// TIMER_START(tsp_nn); - NearestNeighbour(route_parameters, phantom_node_vector, *result_table, min_route_nn, min_loc_permutation_nn); + osrm::tsp::NearestNeighbour(route_parameters, phantom_node_vector, *result_table, min_route_nn, min_loc_permutation_nn); + search_engine_ptr->shortest_path(min_route_nn.segment_end_coordinates, route_parameters.uturns, min_route_nn); TIMER_STOP(tsp_nn); SimpleLogger().Write() << "Distance " << min_route_nn.shortest_path_length; SimpleLogger().Write() << "Time " << TIMER_MSEC(tsp_nn) + TIMER_MSEC(tsp_pre); - // std::unique_ptr> descriptor; - // descriptor = osrm::make_unique>(facade); - - // descriptor->SetConfig(route_parameters); - // descriptor->Run(min_route_nn, json_result); - osrm::json::Array json_loc_permutation_nn; json_loc_permutation_nn.values.insert(json_loc_permutation_nn.values.end(), min_loc_permutation_nn.begin(), min_loc_permutation_nn.end()); json_result.values["nn_loc_permutation"] = json_loc_permutation_nn; json_result.values["nn_distance"] = min_route_nn.shortest_path_length; json_result.values["nn_runtime"] = TIMER_MSEC(tsp_nn) + TIMER_MSEC(tsp_pre); + + //########################### BRUTE FORCE ####################################// + if (route_parameters.coordinates.size() < 12) { + TIMER_START(tsp_bf); + osrm::tsp::BruteForce(route_parameters, phantom_node_vector, *result_table, min_route_bf, min_loc_permutation_bf); + search_engine_ptr->shortest_path(min_route_bf.segment_end_coordinates, route_parameters.uturns, min_route_bf); + TIMER_STOP(tsp_bf); + SimpleLogger().Write() << "Distance " << min_route_bf.shortest_path_length; + SimpleLogger().Write() << "Time " << TIMER_MSEC(tsp_bf) + TIMER_MSEC(tsp_pre); + + osrm::json::Array json_loc_permutation_bf; + json_loc_permutation_bf.values.insert(json_loc_permutation_bf.values.end(), min_loc_permutation_bf.begin(), min_loc_permutation_bf.end()); + json_result.values["bf_loc_permutation"] = json_loc_permutation_bf; + json_result.values["bf_distance"] = min_route_bf.shortest_path_length; + json_result.values["bf_runtime"] = TIMER_MSEC(tsp_bf) + TIMER_MSEC(tsp_pre); + } else { + json_result.values["bf_distance"] = -1; + json_result.values["bf_runtime"] = -1; + } + + + //######################## FARTHEST INSERTION ###############################// TIMER_START(tsp_fi); - FarthestInsertion(route_parameters, phantom_node_vector, *result_table, min_route_fi, min_loc_permutation_fi); + osrm::tsp::FarthestInsertion(route_parameters, phantom_node_vector, *result_table, min_route_fi, min_loc_permutation_fi); + search_engine_ptr->shortest_path(min_route_fi.segment_end_coordinates, route_parameters.uturns, min_route_fi); TIMER_STOP(tsp_fi); SimpleLogger().Write() << "Distance " << min_route_fi.shortest_path_length; SimpleLogger().Write() << "Time " << TIMER_MSEC(tsp_fi) + TIMER_MSEC(tsp_pre); - // return result to json + osrm::json::Array json_loc_permutation_fi; + json_loc_permutation_fi.values.insert(json_loc_permutation_fi.values.end(), min_loc_permutation_fi.begin(), min_loc_permutation_fi.end()); + json_result.values["fi_loc_permutation"] = json_loc_permutation_fi; + json_result.values["fi_distance"] = min_route_fi.shortest_path_length; + json_result.values["fi_runtime"] = TIMER_MSEC(tsp_fi) + TIMER_MSEC(tsp_pre); + + // return geometry result to json std::unique_ptr> descriptor; descriptor = osrm::make_unique>(facade); descriptor->SetConfig(route_parameters); descriptor->Run(min_route_fi, json_result); - osrm::json::Array json_loc_permutation_fi; - json_loc_permutation_fi.values.insert(json_loc_permutation_fi.values.end(), min_loc_permutation_fi.begin(), min_loc_permutation_fi.end()); - json_result.values["fi_loc_permutation"] = json_loc_permutation_fi; - json_result.values["fi_distance"] = min_route_fi.shortest_path_length; - json_result.values["fi_runtime"] = TIMER_MSEC(tsp_fi) + TIMER_MSEC(tsp_pre); - // for (int i = 0; i < min_loc_permutation_fi.size(); ++i) { - // SimpleLogger().Write() << min_loc_permutation_nn[i] << " " << min_loc_permutation_fi[i]; - // } return 200; } diff --git a/routing_algorithms/tsp_farthest_insertion.hpp b/routing_algorithms/tsp_farthest_insertion.hpp new file mode 100644 index 00000000000..0f5c309b305 --- /dev/null +++ b/routing_algorithms/tsp_farthest_insertion.hpp @@ -0,0 +1,159 @@ +/* + +Copyright (c) 2015, Project OSRM contributors +All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + +Redistributions of source code must retain the above copyright notice, this list +of conditions and the following disclaimer. +Redistributions in binary form must reproduce the above copyright notice, this +list of conditions and the following disclaimer in the documentation and/or +other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +*/ + +#ifndef TSP_FARTHEST_INSERTION_HPP +#define TSP_FARTHEST_INSERTION_HPP + + +#include "../data_structures/search_engine.hpp" +#include "../util/string_util.hpp" + +#include + +#include + +#include +#include +#include +#include + + +namespace osrm +{ +namespace tsp +{ + +void FarthestInsertion(const RouteParameters & route_parameters, + const PhantomNodeArray & phantom_node_vector, + const std::vector & dist_table, + InternalRouteResult & min_route, + std::vector & min_loc_permutation) { + ////////////////////////////////////////////////////////////////////////////////////////////////// + // START FARTHEST INSERTION HERE + // 1. start at a random round trip of 2 locations + // 2. find the location that is the farthest away from the visited locations and whose insertion will make the round trip the longest + // 3. add the found location to the current round trip such that round trip is the shortest + // 4. repeat 2-3 until all locations are visited + // 5. DONE! + ////////////////////////////////////////////////////////////////////////////////////////////////// + + const auto number_of_locations = phantom_node_vector.size(); + // list of the trip that will be found incrementally + std::list current_trip; + // tracks which nodes have been already visited + std::vector visited(number_of_locations, false); + + + // find the pair of location with the biggest distance and make the pair the initial start trip + const auto index = std::distance(dist_table.begin(), std::max_element(dist_table.begin(), dist_table.end())); + const int max_from = index / number_of_locations; + const int max_to = index % number_of_locations; + + visited[max_from] = true; + visited[max_to] = true; + current_trip.push_back(max_from); + current_trip.push_back(max_to); + + // add all other nodes missing (two nodes are already in the initial start trip) + for (int j = 2; j < number_of_locations; ++j) { + auto shortest_max_tour = -1; + int next_node = -1; + std::list::iterator min_max_insert; + + // find unvisited loc i that is the farthest away from all other visited locs + for (int i = 0; i < number_of_locations; ++i) { + if (!visited[i]) { + // longest_min_tour is the distance of the longest of all insertions with the minimal distance + auto longest_min_tour = std::numeric_limits::max(); + // following_loc is the location that comes after the location that is to be inserted + std::list::iterator following_loc; + + // for all nodes in the current trip find the best insertion resulting in the shortest path + for (auto from_node = current_trip.begin(); from_node != std::prev(current_trip.end()); ++from_node) { + auto to_node = std::next(from_node); + + auto dist_from = *(dist_table.begin() + (*from_node * number_of_locations) + i); + auto dist_to = *(dist_table.begin() + (i * number_of_locations) + *to_node); + auto trip_dist = dist_from + dist_to - *(dist_table.begin() + (*from_node * number_of_locations) + *to_node); + + // from all possible insertions to the current trip, choose the longest of all minimal insertions + if (trip_dist < longest_min_tour) { + longest_min_tour = trip_dist; + following_loc = to_node; + } + } + { // check insertion between last and first location too + auto from_node = std::prev(current_trip.end()); + auto to_node = current_trip.begin(); + + auto dist_from = *(dist_table.begin() + (*from_node * number_of_locations) + i); + auto dist_to = *(dist_table.begin() + (i * number_of_locations) + *to_node); + auto trip_dist = dist_from + dist_to - *(dist_table.begin() + (*from_node * number_of_locations) + *to_node); + if (trip_dist < longest_min_tour) { + longest_min_tour = trip_dist; + following_loc = to_node; + } + } + + // add the location to the current trip such that it results in the shortest total tour + if (longest_min_tour > shortest_max_tour) { + shortest_max_tour = longest_min_tour; + next_node = i; + min_max_insert = following_loc; + } + } + } + // mark as visited and insert node + visited[next_node] = true; + current_trip.insert(min_max_insert, next_node); + } + + // given he final trip, compute total distance and return the route and location permutation + PhantomNodes viapoint; + int perm = 0; + for (auto it = current_trip.begin(); it != std::prev(current_trip.end()); ++it) { + auto from_node = *it; + auto to_node = *std::next(it); + + viapoint = PhantomNodes{phantom_node_vector[from_node][0], phantom_node_vector[to_node][0]}; + min_route.segment_end_coordinates.emplace_back(viapoint); + + min_loc_permutation[from_node] = perm; + ++perm; + } + { // check dist between last and first location too + viapoint = PhantomNodes{phantom_node_vector[*std::prev(current_trip.end())][0], phantom_node_vector[current_trip.front()][0]}; + min_route.segment_end_coordinates.emplace_back(viapoint); + min_loc_permutation[*std::prev(current_trip.end())] = perm; + } +} + + +} +} + +#endif // TSP_FARTHEST_INSERTION_HPP \ No newline at end of file diff --git a/routing_algorithms/tsp_nearest_neighbour.hpp b/routing_algorithms/tsp_nearest_neighbour.hpp new file mode 100644 index 00000000000..11ff7b905ea --- /dev/null +++ b/routing_algorithms/tsp_nearest_neighbour.hpp @@ -0,0 +1,168 @@ +/* + +Copyright (c) 2015, Project OSRM contributors +All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + +Redistributions of source code must retain the above copyright notice, this list +of conditions and the following disclaimer. +Redistributions in binary form must reproduce the above copyright notice, this +list of conditions and the following disclaimer in the documentation and/or +other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +*/ + +#ifndef TSP_NEAREST_NEIGHBOUR_HPP +#define TSP_NEAREST_NEIGHBOUR_HPP + + +#include "../data_structures/search_engine.hpp" +#include "../util/string_util.hpp" +#include "../util/simple_logger.hpp" + +#include + +#include +#include +#include +#include +#include + + + +namespace osrm +{ +namespace tsp +{ + +void NearestNeighbour(const RouteParameters & route_parameters, + const PhantomNodeArray & phantom_node_vector, + const std::vector & dist_table, + InternalRouteResult & min_route, + std::vector & min_loc_permutation) { + ////////////////////////////////////////////////////////////////////////////////////////////////// + // START GREEDY NEAREST NEIGHBOUR HERE + // 1. grab a random location and mark as starting point + // 2. find the nearest unvisited neighbour, set it as the current location and mark as visited + // 3. repeat 2 until there is no unvisited location + // 4. return route back to starting point + // 5. compute route + // 6. repeat 1-5 with different starting points and choose iteration with shortest trip + // 7. DONE! + ////////////////////////////////////////////////////////////////////////////////////////////////// + + const auto number_of_locations = phantom_node_vector.size(); + min_route.shortest_path_length = std::numeric_limits::max(); + + // is_lonely_island[i] indicates whether node i is a node that cannot be reached from other nodes + // 1 means that node i is a lonely island + // 0 means that it is not known for node i + // -1 means that node i is not a lonely island but a reachable, connected node + std::vector is_lonely_island(number_of_locations, 0); + int count_unreachables; + + // ALWAYS START AT ANOTHER STARTING POINT + for(int start_node = 0; start_node < number_of_locations; ++start_node) + { + + if (is_lonely_island[start_node] >= 0) + { + // if node is a lonely island it is an unsuitable node to start from and shall be skipped + if (is_lonely_island[start_node]) + continue; + count_unreachables = 0; + auto start_dist_begin = dist_table.begin() + (start_node * number_of_locations); + auto start_dist_end = dist_table.begin() + ((start_node + 1) * number_of_locations); + for (auto it2 = start_dist_begin; it2 != start_dist_end; ++it2) { + if (*it2 == 0 || *it2 == std::numeric_limits::max()) { + ++count_unreachables; + } + } + if (count_unreachables >= number_of_locations) { + is_lonely_island[start_node] = 1; + continue; + } + } + + int curr_node = start_node; + is_lonely_island[curr_node] = -1; + InternalRouteResult raw_route; + //TODO: Should we always use the same vector or does it not matter at all because of loop scope? + std::vector loc_permutation(number_of_locations, -1); + loc_permutation[start_node] = 0; + // visited[i] indicates whether node i was already visited by the salesman + std::vector visited(number_of_locations, false); + visited[start_node] = true; + + PhantomNodes viapoint; + // 3. REPEAT FOR EVERY UNVISITED NODE + int trip_dist = 0; + for(int via_point = 1; via_point < number_of_locations; ++via_point) + { + int min_dist = std::numeric_limits::max(); + int min_id = -1; + + // 2. FIND NEAREST NEIGHBOUR + auto row_begin_iterator = dist_table.begin() + (curr_node * number_of_locations); + auto row_end_iterator = dist_table.begin() + ((curr_node + 1) * number_of_locations); + for (auto it = row_begin_iterator; it != row_end_iterator; ++it) { + auto index = std::distance(row_begin_iterator, it); + if (is_lonely_island[index] < 1 && !visited[index] && *it < min_dist) + { + min_dist = *it; + min_id = index; + } + } + // in case there was no unvisited and reachable node found, it means that all remaining (unvisited) nodes must be lonely islands + if (min_id == -1) + { + for(int loc = 0; loc < visited.size(); ++loc) { + if (!visited[loc]) { + is_lonely_island[loc] = 1; + } + } + break; + } + // set the nearest unvisited location as the next via_point + else + { + is_lonely_island[min_id] = -1; + loc_permutation[min_id] = via_point; + visited[min_id] = true; + viapoint = PhantomNodes{phantom_node_vector[curr_node][0], phantom_node_vector[min_id][0]}; + raw_route.segment_end_coordinates.emplace_back(viapoint); + trip_dist += min_dist; + curr_node = min_id; + } + } + + // 4. ROUTE BACK TO STARTING POINT + viapoint = PhantomNodes{raw_route.segment_end_coordinates.back().target_phantom, phantom_node_vector[start_node][0]}; + raw_route.segment_end_coordinates.emplace_back(viapoint); + + // check round trip with this starting point is shorter than the shortest round trip found till now + if (trip_dist < min_route.shortest_path_length) { + min_route = raw_route; + min_route.shortest_path_length = trip_dist; + //TODO: this gets copied right? fix this + min_loc_permutation = loc_permutation; + } + } +} + +} +} +#endif // TSP_NEAREST_NEIGHBOUR_HPP \ No newline at end of file From a40b3a98dc72b348142f4320a1ad5ba0eec8ab18 Mon Sep 17 00:00:00 2001 From: Chau Nguyen Date: Mon, 22 Jun 2015 20:09:00 +0200 Subject: [PATCH 068/122] split algorithms in different plugins for better evaluation split tsp brute force algorithm for better testing refactor and clean up --- library/osrm_impl.cpp | 6 + library/osrm_impl.hpp | 4 + plugins/round_trip.hpp | 119 +++++++------ plugins/round_trip_BF.hpp | 157 ++++++++++++++++++ plugins/round_trip_FI.hpp | 152 +++++++++++++++++ plugins/round_trip_NN.hpp | 150 +++++++++++++++++ routing_algorithms/tsp_brute_force.hpp | 53 ++++-- routing_algorithms/tsp_farthest_insertion.hpp | 11 +- routing_algorithms/tsp_nearest_neighbour.hpp | 9 +- tools/tsp_logs.hpp | 103 ++++++++++++ unit_tests/routing_algorithms/tsp.cpp | 45 +++++ 11 files changed, 729 insertions(+), 80 deletions(-) create mode 100644 plugins/round_trip_BF.hpp create mode 100644 plugins/round_trip_FI.hpp create mode 100644 plugins/round_trip_NN.hpp create mode 100644 tools/tsp_logs.hpp create mode 100644 unit_tests/routing_algorithms/tsp.cpp diff --git a/library/osrm_impl.cpp b/library/osrm_impl.cpp index 8057069eb88..26ba83e53a8 100644 --- a/library/osrm_impl.cpp +++ b/library/osrm_impl.cpp @@ -42,6 +42,9 @@ class named_mutex; #include "../plugins/nearest.hpp" #include "../plugins/timestamp.hpp" #include "../plugins/round_trip.hpp" +#include "../plugins/round_trip_NN.hpp" +#include "../plugins/round_trip_BF.hpp" +#include "../plugins/round_trip_FI.hpp" #include "../plugins/viaroute.hpp" #include "../plugins/match.hpp" #include "../server/data_structures/datafacade_base.hpp" @@ -88,6 +91,9 @@ OSRM_impl::OSRM_impl(libosrm_config &lib_config) RegisterPlugin(new TimestampPlugin>(query_data_facade)); RegisterPlugin(new ViaRoutePlugin>(query_data_facade)); RegisterPlugin(new RoundTripPlugin>(query_data_facade)); + RegisterPlugin(new RoundTripPluginNN>(query_data_facade)); + RegisterPlugin(new RoundTripPluginBF>(query_data_facade)); + RegisterPlugin(new RoundTripPluginFI>(query_data_facade)); } OSRM_impl::~OSRM_impl() diff --git a/library/osrm_impl.hpp b/library/osrm_impl.hpp index a736c042f6e..6984c5ff27b 100644 --- a/library/osrm_impl.hpp +++ b/library/osrm_impl.hpp @@ -28,6 +28,10 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #ifndef OSRM_IMPL_HPP #define OSRM_IMPL_HPP +// #if __cplusplus > 199711L +// #define register // Deprecated in C++11. +// #endif // #if __cplusplus > 199711L + class BasePlugin; struct RouteParameters; diff --git a/plugins/round_trip.hpp b/plugins/round_trip.hpp index 652e82f3b84..891e422eb04 100644 --- a/plugins/round_trip.hpp +++ b/plugins/round_trip.hpp @@ -116,70 +116,83 @@ template class RoundTripPlugin final : public BasePlugin } // compute TSP round trip - InternalRouteResult min_route_nn; - InternalRouteResult min_route_fi; - InternalRouteResult min_route_bf; - std::vector min_loc_permutation_nn(phantom_node_vector.size(), -1); - std::vector min_loc_permutation_fi(phantom_node_vector.size(), -1); - std::vector min_loc_permutation_bf(phantom_node_vector.size(), -1); + InternalRouteResult min_route; + std::vector min_loc_permutation(phantom_node_vector.size(), -1); TIMER_STOP(tsp_pre); - //######################### NEAREST NEIGHBOUR ###############################// - TIMER_START(tsp_nn); - osrm::tsp::NearestNeighbour(route_parameters, phantom_node_vector, *result_table, min_route_nn, min_loc_permutation_nn); - search_engine_ptr->shortest_path(min_route_nn.segment_end_coordinates, route_parameters.uturns, min_route_nn); - TIMER_STOP(tsp_nn); - SimpleLogger().Write() << "Distance " << min_route_nn.shortest_path_length; - SimpleLogger().Write() << "Time " << TIMER_MSEC(tsp_nn) + TIMER_MSEC(tsp_pre); - - osrm::json::Array json_loc_permutation_nn; - json_loc_permutation_nn.values.insert(json_loc_permutation_nn.values.end(), min_loc_permutation_nn.begin(), min_loc_permutation_nn.end()); - json_result.values["nn_loc_permutation"] = json_loc_permutation_nn; - json_result.values["nn_distance"] = min_route_nn.shortest_path_length; - json_result.values["nn_runtime"] = TIMER_MSEC(tsp_nn) + TIMER_MSEC(tsp_pre); - - - //########################### BRUTE FORCE ####################################// - if (route_parameters.coordinates.size() < 12) { - TIMER_START(tsp_bf); - osrm::tsp::BruteForce(route_parameters, phantom_node_vector, *result_table, min_route_bf, min_loc_permutation_bf); - search_engine_ptr->shortest_path(min_route_bf.segment_end_coordinates, route_parameters.uturns, min_route_bf); - TIMER_STOP(tsp_bf); - SimpleLogger().Write() << "Distance " << min_route_bf.shortest_path_length; - SimpleLogger().Write() << "Time " << TIMER_MSEC(tsp_bf) + TIMER_MSEC(tsp_pre); - - osrm::json::Array json_loc_permutation_bf; - json_loc_permutation_bf.values.insert(json_loc_permutation_bf.values.end(), min_loc_permutation_bf.begin(), min_loc_permutation_bf.end()); - json_result.values["bf_loc_permutation"] = json_loc_permutation_bf; - json_result.values["bf_distance"] = min_route_bf.shortest_path_length; - json_result.values["bf_runtime"] = TIMER_MSEC(tsp_bf) + TIMER_MSEC(tsp_pre); - } else { - json_result.values["bf_distance"] = -1; - json_result.values["bf_runtime"] = -1; - } - //######################## FARTHEST INSERTION ###############################// - TIMER_START(tsp_fi); - osrm::tsp::FarthestInsertion(route_parameters, phantom_node_vector, *result_table, min_route_fi, min_loc_permutation_fi); - search_engine_ptr->shortest_path(min_route_fi.segment_end_coordinates, route_parameters.uturns, min_route_fi); - TIMER_STOP(tsp_fi); - SimpleLogger().Write() << "Distance " << min_route_fi.shortest_path_length; - SimpleLogger().Write() << "Time " << TIMER_MSEC(tsp_fi) + TIMER_MSEC(tsp_pre); - - osrm::json::Array json_loc_permutation_fi; - json_loc_permutation_fi.values.insert(json_loc_permutation_fi.values.end(), min_loc_permutation_fi.begin(), min_loc_permutation_fi.end()); - json_result.values["fi_loc_permutation"] = json_loc_permutation_fi; - json_result.values["fi_distance"] = min_route_fi.shortest_path_length; - json_result.values["fi_runtime"] = TIMER_MSEC(tsp_fi) + TIMER_MSEC(tsp_pre); + //######################### NEAREST NEIGHBOUR ###############################// + TIMER_START(tsp); + osrm::tsp::NearestNeighbour(route_parameters, phantom_node_vector, *result_table, min_route, min_loc_permutation); + search_engine_ptr->shortest_path(min_route.segment_end_coordinates, route_parameters.uturns, min_route); + TIMER_STOP(tsp); + SimpleLogger().Write() << "Distance " << min_route.shortest_path_length; + SimpleLogger().Write() << "Time " << TIMER_MSEC(tsp) + TIMER_MSEC(tsp_pre); + + osrm::json::Array json_loc_permutation; + json_loc_permutation.values.insert(json_loc_permutation.values.end(), min_loc_permutation.begin(), min_loc_permutation.end()); + json_result.values["loc_permutation"] = json_loc_permutation; + json_result.values["distance"] = min_route.shortest_path_length; + json_result.values["runtime"] = TIMER_MSEC(tsp); + + + + // if (route_parameters.tsp_algo.compare("NN")) + // //######################### NEAREST NEIGHBOUR ###############################// + // TIMER_START(tsp); + // osrm::tsp::NearestNeighbour(route_parameters, phantom_node_vector, *result_table, min_route, min_loc_permutation); + // search_engine_ptr->shortest_path(min_route.segment_end_coordinates, route_parameters.uturns, min_route); + // TIMER_STOP(tsp); + // SimpleLogger().Write() << "Distance " << min_route.shortest_path_length; + // SimpleLogger().Write() << "Time " << TIMER_MSEC(tsp) + TIMER_MSEC(tsp_pre); + + // osrm::json::Array json_loc_permutation; + // json_loc_permutation.values.insert(json_loc_permutation.values.end(), min_loc_permutation.begin(), min_loc_permutation.end()); + // json_result.values["loc_permutation"] = json_loc_permutation; + // json_result.values["distance"] = min_route.shortest_path_length; + // json_result.values["runtime"] = TIMER_MSEC(tsp); + // else if (route_parameters.tsp_algo.compare("BF") + // //########################### BRUTE FORCE ####################################// + // if (route_parameters.coordinates.size() < 12) { + // TIMER_START(tsp); + // osrm::tsp::BruteForce(route_parameters, phantom_node_vector, *result_table, min_route, min_loc_permutation); + // search_engine_ptr->shortest_path(min_route.segment_end_coordinates, route_parameters.uturns, min_route); + // TIMER_STOP(tsp); + // SimpleLogger().Write() << "Distance " << min_route.shortest_path_length; + // SimpleLogger().Write() << "Time " << TIMER_MSEC(tsp); + + // osrm::json::Array json_loc_permutation; + // json_loc_permutation.values.insert(json_loc_permutation.values.end(), min_loc_permutation.begin(), min_loc_permutation.end()); + // json_result.values["loc_permutation"] = json_loc_permutation; + // json_result.values["distance"] = min_route.shortest_path_length; + // json_result.values["runtime"] = TIMER_MSEC(tsp); + // } else { + // json_result.values["distance"] = -1; + // json_result.values["runtime"] = -1; + // } + // else + // //######################## FARTHEST INSERTION ###############################// + // TIMER_START(tsp); + // osrm::tsp::FarthestInsertion(route_parameters, phantom_node_vector, *result_table, min_route, min_loc_permutation); + // search_engine_ptr->shortest_path(min_route.segment_end_coordinates, route_parameters.uturns, min_route); + // TIMER_STOP(tsp); + // SimpleLogger().Write() << "Distance " << min_route.shortest_path_length; + // SimpleLogger().Write() << "Time " << TIMER_MSEC(tsp); + + // osrm::json::Array json_loc_permutation; + // json_loc_permutation.values.insert(json_loc_permutation.values.end(), min_loc_permutation.begin(), min_loc_permutation.end()); + // json_result.values["loc_permutation"] = json_loc_permutation; + // json_result.values["distance"] = min_route.shortest_path_length; + // json_result.values["runtime"] = TIMER_MSEC(tsp); // return geometry result to json std::unique_ptr> descriptor; descriptor = osrm::make_unique>(facade); descriptor->SetConfig(route_parameters); - descriptor->Run(min_route_fi, json_result); + descriptor->Run(min_route, json_result); diff --git a/plugins/round_trip_BF.hpp b/plugins/round_trip_BF.hpp new file mode 100644 index 00000000000..611c9d54207 --- /dev/null +++ b/plugins/round_trip_BF.hpp @@ -0,0 +1,157 @@ +/* + +Copyright (c) 2015, Project OSRM contributors +All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + +Redistributions of source code must retain the above copyright notice, this list +of conditions and the following disclaimer. +Redistributions in binary form must reproduce the above copyright notice, this +list of conditions and the following disclaimer in the documentation and/or +other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +*/ + +#ifndef ROUND_TRIP_BF_HPP +#define ROUND_TRIP_BF_HPP + +#include "plugin_base.hpp" + +#include "../algorithms/object_encoder.hpp" +#include "../routing_algorithms/tsp_nearest_neighbour.hpp" +#include "../routing_algorithms/tsp_farthest_insertion.hpp" +#include "../routing_algorithms/tsp_brute_force.hpp" +#include "../data_structures/query_edge.hpp" +#include "../data_structures/search_engine.hpp" +#include "../descriptors/descriptor_base.hpp" +#include "../descriptors/json_descriptor.hpp" +#include "../util/json_renderer.hpp" +#include "../util/make_unique.hpp" +#include "../util/string_util.hpp" +#include "../util/timing_util.hpp" +#include "../util/simple_logger.hpp" + +#include + +#include +#include +#include +#include +#include +#include +#include + +template class RoundTripPluginBF final : public BasePlugin +{ + private: + std::string descriptor_string; + DataFacadeT *facade; + std::unique_ptr> search_engine_ptr; + + public: + explicit RoundTripPluginBF(DataFacadeT *facade) + : descriptor_string("tripBF"), facade(facade) + { + search_engine_ptr = osrm::make_unique>(facade); + } + + const std::string GetDescriptor() const override final { return descriptor_string; } + + int HandleRequest(const RouteParameters &route_parameters, + osrm::json::Object &json_result) override final + { + TIMER_START(tsp_pre); + // check if all inputs are coordinates + if (!check_all_coordinates(route_parameters.coordinates)) + { + return 400; + } + const bool checksum_OK = (route_parameters.check_sum == facade->GetCheckSum()); + + // find phantom nodes for all input coords + PhantomNodeArray phantom_node_vector(route_parameters.coordinates.size()); + for (const auto i : osrm::irange(0, route_parameters.coordinates.size())) + { + // if client hints are helpful, encode hints + if (checksum_OK && i < route_parameters.hints.size() && + !route_parameters.hints[i].empty()) + { + PhantomNode current_phantom_node; + ObjectEncoder::DecodeFromBase64(route_parameters.hints[i], current_phantom_node); + if (current_phantom_node.is_valid(facade->GetNumberOfNodes())) + { + phantom_node_vector[i].emplace_back(std::move(current_phantom_node)); + continue; + } + } + facade->IncrementalFindPhantomNodeForCoordinate(route_parameters.coordinates[i], + phantom_node_vector[i], 1); + if (phantom_node_vector[i].size() > 1) + { + phantom_node_vector[i].erase(phantom_node_vector[i].begin()); + } + BOOST_ASSERT(phantom_node_vector[i].front().is_valid(facade->GetNumberOfNodes())); + } + + // compute the distance table of all phantom nodes + const std::shared_ptr> result_table = + search_engine_ptr->distance_table(phantom_node_vector); + + if (!result_table) + { + return 400; + } + + // compute TSP round trip + InternalRouteResult min_route; + std::vector min_loc_permutation(phantom_node_vector.size(), -1); + TIMER_STOP(tsp_pre); + + //########################### BRUTE FORCE ####################################// + if (route_parameters.coordinates.size() < 11) { + TIMER_START(tsp); + osrm::tsp::BruteForceTSP(phantom_node_vector, *result_table, min_route, min_loc_permutation); + search_engine_ptr->shortest_path(min_route.segment_end_coordinates, route_parameters.uturns, min_route); + TIMER_STOP(tsp); + + BOOST_ASSERT(min_route.segment_end_coordinates.size() == route_parameters.coordinates.size()); + + osrm::json::Array json_loc_permutation; + json_loc_permutation.values.insert(json_loc_permutation.values.end(), min_loc_permutation.begin(), min_loc_permutation.end()); + json_result.values["loc_permutation"] = json_loc_permutation; + json_result.values["distance"] = min_route.shortest_path_length; + SimpleLogger().Write() << "BF GEOM DISTANCE " << min_route.shortest_path_length; + json_result.values["runtime"] = TIMER_MSEC(tsp); + } else { + json_result.values["distance"] = -1; + json_result.values["runtime"] = -1; + } + // return geometry result to json + + std::unique_ptr> descriptor; + descriptor = osrm::make_unique>(facade); + + descriptor->SetConfig(route_parameters); + descriptor->Run(min_route, json_result); + + + + return 200; + } + +}; + +#endif // ROUND_TRIP_BF_HPP diff --git a/plugins/round_trip_FI.hpp b/plugins/round_trip_FI.hpp new file mode 100644 index 00000000000..f470052fa00 --- /dev/null +++ b/plugins/round_trip_FI.hpp @@ -0,0 +1,152 @@ +/* + +Copyright (c) 2015, Project OSRM contributors +All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + +Redistributions of source code must retain the above copyright notice, this list +of conditions and the following disclaimer. +Redistributions in binary form must reproduce the above copyright notice, this +list of conditions and the following disclaimer in the documentation and/or +other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +*/ + +#ifndef ROUND_TRIP_FI_HPP +#define ROUND_TRIP_FI_HPP + +#include "plugin_base.hpp" + +#include "../algorithms/object_encoder.hpp" +#include "../routing_algorithms/tsp_nearest_neighbour.hpp" +#include "../routing_algorithms/tsp_farthest_insertion.hpp" +#include "../routing_algorithms/tsp_brute_force.hpp" +#include "../data_structures/query_edge.hpp" +#include "../data_structures/search_engine.hpp" +#include "../descriptors/descriptor_base.hpp" +#include "../descriptors/json_descriptor.hpp" +#include "../util/json_renderer.hpp" +#include "../util/make_unique.hpp" +#include "../util/string_util.hpp" +#include "../util/timing_util.hpp" +#include "../util/simple_logger.hpp" + +#include + +#include +#include +#include +#include +#include +#include +#include + +template class RoundTripPluginFI final : public BasePlugin +{ + private: + std::string descriptor_string; + DataFacadeT *facade; + std::unique_ptr> search_engine_ptr; + + public: + explicit RoundTripPluginFI(DataFacadeT *facade) + : descriptor_string("tripFI"), facade(facade) + { + search_engine_ptr = osrm::make_unique>(facade); + } + + const std::string GetDescriptor() const override final { return descriptor_string; } + + int HandleRequest(const RouteParameters &route_parameters, + osrm::json::Object &json_result) override final + { + TIMER_START(tsp_pre); + // check if all inputs are coordinates + if (!check_all_coordinates(route_parameters.coordinates)) + { + return 400; + } + const bool checksum_OK = (route_parameters.check_sum == facade->GetCheckSum()); + + // find phantom nodes for all input coords + PhantomNodeArray phantom_node_vector(route_parameters.coordinates.size()); + for (const auto i : osrm::irange(0, route_parameters.coordinates.size())) + { + // if client hints are helpful, encode hints + if (checksum_OK && i < route_parameters.hints.size() && + !route_parameters.hints[i].empty()) + { + PhantomNode current_phantom_node; + ObjectEncoder::DecodeFromBase64(route_parameters.hints[i], current_phantom_node); + if (current_phantom_node.is_valid(facade->GetNumberOfNodes())) + { + phantom_node_vector[i].emplace_back(std::move(current_phantom_node)); + continue; + } + } + facade->IncrementalFindPhantomNodeForCoordinate(route_parameters.coordinates[i], + phantom_node_vector[i], 1); + if (phantom_node_vector[i].size() > 1) + { + phantom_node_vector[i].erase(phantom_node_vector[i].begin()); + } + BOOST_ASSERT(phantom_node_vector[i].front().is_valid(facade->GetNumberOfNodes())); + } + + // compute the distance table of all phantom nodes + const std::shared_ptr> result_table = + search_engine_ptr->distance_table(phantom_node_vector); + + if (!result_table) + { + return 400; + } + + // compute TSP round trip + InternalRouteResult min_route; + std::vector min_loc_permutation(phantom_node_vector.size(), -1); + TIMER_STOP(tsp_pre); + + //######################## FARTHEST INSERTION ###############################// + TIMER_START(tsp); + osrm::tsp::FarthestInsertionTSP(phantom_node_vector, *result_table, min_route, min_loc_permutation); + search_engine_ptr->shortest_path(min_route.segment_end_coordinates, route_parameters.uturns, min_route); + TIMER_STOP(tsp); + + BOOST_ASSERT(min_route.segment_end_coordinates.size() == route_parameters.coordinates.size()); + + osrm::json::Array json_loc_permutation; + json_loc_permutation.values.insert(json_loc_permutation.values.end(), min_loc_permutation.begin(), min_loc_permutation.end()); + json_result.values["loc_permutation"] = json_loc_permutation; + json_result.values["distance"] = min_route.shortest_path_length; + SimpleLogger().Write() << "FI GEOM DISTANCE " << min_route.shortest_path_length; + json_result.values["runtime"] = TIMER_MSEC(tsp); + + // return geometry result to json + std::unique_ptr> descriptor; + descriptor = osrm::make_unique>(facade); + + descriptor->SetConfig(route_parameters); + descriptor->Run(min_route, json_result); + + + + return 200; + } + +}; + +#endif // ROUND_TRIP_FI_HPP diff --git a/plugins/round_trip_NN.hpp b/plugins/round_trip_NN.hpp new file mode 100644 index 00000000000..4d6d674afdf --- /dev/null +++ b/plugins/round_trip_NN.hpp @@ -0,0 +1,150 @@ +/* + +Copyright (c) 2015, Project OSRM contributors +All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + +Redistributions of source code must retain the above copyright notice, this list +of conditions and the following disclaimer. +Redistributions in binary form must reproduce the above copyright notice, this +list of conditions and the following disclaimer in the documentation and/or +other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +*/ + +#ifndef ROUND_TRIP_NN_HPP +#define ROUND_TRIP_NN_HPP + +#include "plugin_base.hpp" + +#include "../algorithms/object_encoder.hpp" +#include "../routing_algorithms/tsp_nearest_neighbour.hpp" +#include "../routing_algorithms/tsp_farthest_insertion.hpp" +#include "../routing_algorithms/tsp_brute_force.hpp" +#include "../data_structures/query_edge.hpp" +#include "../data_structures/search_engine.hpp" +#include "../descriptors/descriptor_base.hpp" +#include "../descriptors/json_descriptor.hpp" +#include "../util/json_renderer.hpp" +#include "../util/make_unique.hpp" +#include "../util/string_util.hpp" +#include "../util/timing_util.hpp" +#include "../util/simple_logger.hpp" + +#include + +#include +#include +#include +#include +#include +#include +#include + +template class RoundTripPluginNN final : public BasePlugin +{ + private: + std::string descriptor_string; + DataFacadeT *facade; + std::unique_ptr> search_engine_ptr; + + public: + explicit RoundTripPluginNN(DataFacadeT *facade) + : descriptor_string("tripNN"), facade(facade) + { + search_engine_ptr = osrm::make_unique>(facade); + } + + const std::string GetDescriptor() const override final { return descriptor_string; } + + int HandleRequest(const RouteParameters &route_parameters, + osrm::json::Object &json_result) override final + { + TIMER_START(tsp_pre); + // check if all inputs are coordinates + if (!check_all_coordinates(route_parameters.coordinates)) + { + return 400; + } + const bool checksum_OK = (route_parameters.check_sum == facade->GetCheckSum()); + + // find phantom nodes for all input coords + PhantomNodeArray phantom_node_vector(route_parameters.coordinates.size()); + for (const auto i : osrm::irange(0, route_parameters.coordinates.size())) + { + // if client hints are helpful, encode hints + if (checksum_OK && i < route_parameters.hints.size() && + !route_parameters.hints[i].empty()) + { + PhantomNode current_phantom_node; + ObjectEncoder::DecodeFromBase64(route_parameters.hints[i], current_phantom_node); + if (current_phantom_node.is_valid(facade->GetNumberOfNodes())) + { + phantom_node_vector[i].emplace_back(std::move(current_phantom_node)); + continue; + } + } + facade->IncrementalFindPhantomNodeForCoordinate(route_parameters.coordinates[i], + phantom_node_vector[i], 1); + if (phantom_node_vector[i].size() > 1) + { + phantom_node_vector[i].erase(phantom_node_vector[i].begin()); + } + BOOST_ASSERT(phantom_node_vector[i].front().is_valid(facade->GetNumberOfNodes())); + } + + // compute the distance table of all phantom nodes + const std::shared_ptr> result_table = + search_engine_ptr->distance_table(phantom_node_vector); + + if (!result_table) + { + return 400; + } + + // compute TSP round trip + InternalRouteResult min_route; + std::vector min_loc_permutation(phantom_node_vector.size(), -1); + TIMER_STOP(tsp_pre); + + //######################### NEAREST NEIGHBOUR ###############################// + TIMER_START(tsp); + osrm::tsp::NearestNeighbourTSP(phantom_node_vector, *result_table, min_route, min_loc_permutation); + search_engine_ptr->shortest_path(min_route.segment_end_coordinates, route_parameters.uturns, min_route); + TIMER_STOP(tsp); + BOOST_ASSERT(min_route.segment_end_coordinates.size() == route_parameters.coordinates.size()); + + osrm::json::Array json_loc_permutation; + json_loc_permutation.values.insert(json_loc_permutation.values.end(), min_loc_permutation.begin(), min_loc_permutation.end()); + json_result.values["loc_permutation"] = json_loc_permutation; + json_result.values["distance"] = min_route.shortest_path_length; + json_result.values["runtime"] = TIMER_MSEC(tsp); + + // return geometry result to json + std::unique_ptr> descriptor; + descriptor = osrm::make_unique>(facade); + + descriptor->SetConfig(route_parameters); + descriptor->Run(min_route, json_result); + + + + return 200; + } + +}; + +#endif // ROUND_TRIP_NN_HPP diff --git a/routing_algorithms/tsp_brute_force.hpp b/routing_algorithms/tsp_brute_force.hpp index 099ed4e4a8c..a414659c41c 100644 --- a/routing_algorithms/tsp_brute_force.hpp +++ b/routing_algorithms/tsp_brute_force.hpp @@ -40,47 +40,66 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include #include #include +#include #include "../util/simple_logger.hpp" -// HAHAHA. NICE TRY, CHAU. namespace osrm { namespace tsp { -void BruteForce(const RouteParameters & route_parameters, - const PhantomNodeArray & phantom_node_vector, - const std::vector & dist_table, - InternalRouteResult & min_route, - std::vector & min_loc_permutation) { + + +int ReturnDistance(const std::vector & dist_table, const std::vector location_order, const int min_route_dist, const int number_of_locations) { + int i = 0; + int route_dist = 0; + + // compute length and stop if length is longer than route already found + while (i < number_of_locations - 1 && route_dist < min_route_dist) { + //get distance from location i to location i+1 + route_dist += *(dist_table.begin() + (location_order[i] * number_of_locations) + location_order[i+1]); + ++i; + } + //get distance from last location to first location + route_dist += *(dist_table.begin() + (location_order[number_of_locations-1] * number_of_locations) + location_order[0]); + + if (route_dist < min_route_dist) { + return route_dist; + } + else { + return -1; + } +} + +void BruteForceTSP(const PhantomNodeArray & phantom_node_vector, + const std::vector & dist_table, + InternalRouteResult & min_route, + std::vector & min_loc_permutation) { const auto number_of_locations = phantom_node_vector.size(); + // fill a vector with node ids std::vector location_ids(number_of_locations); std::iota(location_ids.begin(), location_ids.end(), 0); int min_route_dist = std::numeric_limits::max(); - do { - int route_dist = 0; - for (int i = 0; i < number_of_locations - 1; ++i) { - route_dist += *(dist_table.begin() + (location_ids[i] * number_of_locations) + location_ids[i+1]); - } - route_dist += *(dist_table.begin() + (location_ids[number_of_locations-1] * number_of_locations) + location_ids[0]); - if (route_dist < min_route_dist) { - min_route_dist = route_dist; + // check length of all possible permutation of the location ids + do { + int new_distance = ReturnDistance(dist_table, location_ids, min_route_dist, number_of_locations); + if (new_distance != -1) { + min_route_dist = new_distance; //TODO: this gets copied right? fix this min_loc_permutation = location_ids; } } while(std::next_permutation(location_ids.begin(), location_ids.end())); - PhantomNodes viapoint; for (int i = 0; i < number_of_locations - 1; ++i) { - viapoint = PhantomNodes{phantom_node_vector[i][0], phantom_node_vector[i + 1][0]}; + viapoint = PhantomNodes{phantom_node_vector[min_loc_permutation[i]][0], phantom_node_vector[min_loc_permutation[i + 1]][0]}; min_route.segment_end_coordinates.emplace_back(viapoint); } - viapoint = PhantomNodes{phantom_node_vector[number_of_locations - 1][0], phantom_node_vector[0][0]}; + viapoint = PhantomNodes{phantom_node_vector[min_loc_permutation[number_of_locations - 1]][0], phantom_node_vector[min_loc_permutation[0]][0]}; min_route.segment_end_coordinates.emplace_back(viapoint); } diff --git a/routing_algorithms/tsp_farthest_insertion.hpp b/routing_algorithms/tsp_farthest_insertion.hpp index 0f5c309b305..05ef5e612cd 100644 --- a/routing_algorithms/tsp_farthest_insertion.hpp +++ b/routing_algorithms/tsp_farthest_insertion.hpp @@ -47,11 +47,10 @@ namespace osrm namespace tsp { -void FarthestInsertion(const RouteParameters & route_parameters, - const PhantomNodeArray & phantom_node_vector, - const std::vector & dist_table, - InternalRouteResult & min_route, - std::vector & min_loc_permutation) { +void FarthestInsertionTSP(const PhantomNodeArray & phantom_node_vector, + const std::vector & dist_table, + InternalRouteResult & min_route, + std::vector & min_loc_permutation) { ////////////////////////////////////////////////////////////////////////////////////////////////// // START FARTHEST INSERTION HERE // 1. start at a random round trip of 2 locations @@ -67,6 +66,8 @@ void FarthestInsertion(const RouteParameters & route_parameters, // tracks which nodes have been already visited std::vector visited(number_of_locations, false); + // PrintDistTable(dist_table, number_of_locations); + // find the pair of location with the biggest distance and make the pair the initial start trip const auto index = std::distance(dist_table.begin(), std::max_element(dist_table.begin(), dist_table.end())); diff --git a/routing_algorithms/tsp_nearest_neighbour.hpp b/routing_algorithms/tsp_nearest_neighbour.hpp index 11ff7b905ea..113e8d36fdb 100644 --- a/routing_algorithms/tsp_nearest_neighbour.hpp +++ b/routing_algorithms/tsp_nearest_neighbour.hpp @@ -48,11 +48,10 @@ namespace osrm namespace tsp { -void NearestNeighbour(const RouteParameters & route_parameters, - const PhantomNodeArray & phantom_node_vector, - const std::vector & dist_table, - InternalRouteResult & min_route, - std::vector & min_loc_permutation) { +void NearestNeighbourTSP(const PhantomNodeArray & phantom_node_vector, + const std::vector & dist_table, + InternalRouteResult & min_route, + std::vector & min_loc_permutation) { ////////////////////////////////////////////////////////////////////////////////////////////////// // START GREEDY NEAREST NEIGHBOUR HERE // 1. grab a random location and mark as starting point diff --git a/tools/tsp_logs.hpp b/tools/tsp_logs.hpp new file mode 100644 index 00000000000..fcc9af09666 --- /dev/null +++ b/tools/tsp_logs.hpp @@ -0,0 +1,103 @@ +/* + +Copyright (c) 2015, Project OSRM contributors +All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + +Redistributions of source code must retain the above copyright notice, this list +of conditions and the following disclaimer. +Redistributions in binary form must reproduce the above copyright notice, this +list of conditions and the following disclaimer in the documentation and/or +other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +*/ + +#ifndef TSP_LOGS_HPP +#define TSP_LOGS_HPP + + +#include "../data_structures/search_engine.hpp" +#include "../util/string_util.hpp" + +#include + +#include + +#include +#include +#include +#include +#include +#include "../util/simple_logger.hpp" + + +namespace osrm +{ +namespace tsp +{ + + +inline void PrintDistTable(const std::vector & dist_table, const int number_of_locations) { + int j = 0; + for (auto i = dist_table.begin(); i != dist_table.end(); ++i){ + if (j % number_of_locations == 0) { + std::cout << std::endl; + } + std::cout << std::setw(6) << *i << " "; + ++j; + } + std::cout << std::endl; +} + +bool CheckSymmetricTable(const std::vector & dist_table, const int number_of_locations) { + bool is_quadratic = true; + for (int i = 0; i < number_of_locations; ++i) { + for(int j = 0; j < number_of_locations; ++j) { + int a = *(dist_table.begin() + (i * number_of_locations) + j); + int b = *(dist_table.begin() + (j * number_of_locations) + i); + if (a !=b) { + is_quadratic = false; + } + } + } + return is_quadratic; +} + +void LogRoute(std::vector location_ids){ + SimpleLogger().Write() << "LOC ORDER"; + for (auto x : location_ids) { + SimpleLogger().Write() << x; + } +} + +int ReturnDistanceFI(const std::vector & dist_table, std::list current_trip, const int number_of_locations) { + int route_dist = 0; + + // compute length and stop if length is longer than route already found + for (auto i = current_trip.begin(); i != std::prev(current_trip.end()); ++i) { + //get distance from location i to location i+1 + route_dist += *(dist_table.begin() + (*i * number_of_locations) + *std::next(i)); + } + //get distance from last location to first location + route_dist += *(dist_table.begin() + (*std::prev(current_trip.end()) * number_of_locations) + current_trip.front()); + + return route_dist; +} + + +} +} +#endif // TSP_LOGS_HPP \ No newline at end of file diff --git a/unit_tests/routing_algorithms/tsp.cpp b/unit_tests/routing_algorithms/tsp.cpp new file mode 100644 index 00000000000..0c6a52462f7 --- /dev/null +++ b/unit_tests/routing_algorithms/tsp.cpp @@ -0,0 +1,45 @@ +/* + +Copyright (c) 2014, Project OSRM contributors +All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + +Redistributions of source code must retain the above copyright notice, this list +of conditions and the following disclaimer. +Redistributions in binary form must reproduce the above copyright notice, this +list of conditions and the following disclaimer in the documentation and/or +other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +*/ + +#include "../../routing_algorithms/tsp_brute_force.hpp" + +#include +#include +#include + +#include + +#include + +BOOST_AUTO_TEST_SUITE(tsp) + +// BOOST_AUTO_TEST_CASE(check_distance_computation) +// { +// BOOST_CHECK_EQUAL(true, true); +// } + +BOOST_AUTO_TEST_SUITE_END() From 6eeadddd4dc8e34ecce8b662ecbfe10a008aec9d Mon Sep 17 00:00:00 2001 From: Chau Nguyen Date: Wed, 1 Jul 2015 11:32:23 +0200 Subject: [PATCH 069/122] remove attention on unaccessible locations as we filter them beforehand --- plugins/round_trip.hpp | 31 +++++++++++ plugins/round_trip_BF.hpp | 31 +++++++++++ plugins/round_trip_FI.hpp | 31 +++++++++++ plugins/round_trip_NN.hpp | 34 +++++++++++- routing_algorithms/tsp_nearest_neighbour.hpp | 57 +++----------------- 5 files changed, 133 insertions(+), 51 deletions(-) diff --git a/plugins/round_trip.hpp b/plugins/round_trip.hpp index 891e422eb04..92922671bc8 100644 --- a/plugins/round_trip.hpp +++ b/plugins/round_trip.hpp @@ -115,6 +115,37 @@ template class RoundTripPlugin final : public BasePlugin return 400; } + auto number_of_locations = phantom_node_vector.size(); + const auto maxint = std::numeric_limits::max(); + + //////////////////////////////////// DELETE UNACCESSIBLE LOCATIONS ///////////////////////////////////////// + + if (*std::max_element(result_table->begin(), result_table->end()) == maxint) { + const int half = number_of_locations / 2; + std::vector to_delete; + + for (int i = number_of_locations - 1; i >= 0; --i) { + // if the location is unaccessible by most of the other locations, remember the location + if (std::count(result_table->begin() + i * number_of_locations, result_table->begin() + (i+1) * number_of_locations, maxint) > half) { + to_delete.push_back(i); + } + } + //delete all unaccessible locations + for (int k = 0; k < to_delete.size(); ++k) { + // delete its row + result_table->erase(result_table->begin() + to_delete[k] * number_of_locations, result_table->begin() + (to_delete[k]+1) * number_of_locations); + --number_of_locations; + // delete its column + for (int j = 0; j < number_of_locations; ++j) { + result_table->erase(result_table->begin() + j * number_of_locations + to_delete[k]); + } + } + } + //todo: delete + if (*std::max_element(result_table->begin(), result_table->end()) == maxint) { + SimpleLogger().Write() << "SOMETHING WENT WRONG"; + } + // compute TSP round trip InternalRouteResult min_route; std::vector min_loc_permutation(phantom_node_vector.size(), -1); diff --git a/plugins/round_trip_BF.hpp b/plugins/round_trip_BF.hpp index 611c9d54207..60c12c2b5d8 100644 --- a/plugins/round_trip_BF.hpp +++ b/plugins/round_trip_BF.hpp @@ -115,6 +115,37 @@ template class RoundTripPluginBF final : public BasePlugin return 400; } + auto number_of_locations = phantom_node_vector.size(); + const auto maxint = std::numeric_limits::max(); + + //////////////////////////////////// DELETE UNACCESSIBLE LOCATIONS ///////////////////////////////////////// + + if (*std::max_element(result_table->begin(), result_table->end()) == maxint) { + const int half = number_of_locations / 2; + std::vector to_delete; + + for (int i = number_of_locations - 1; i >= 0; --i) { + // if the location is unaccessible by most of the other locations, remember the location + if (std::count(result_table->begin() + i * number_of_locations, result_table->begin() + (i+1) * number_of_locations, maxint) > half) { + to_delete.push_back(i); + } + } + //delete all unaccessible locations + for (int k = 0; k < to_delete.size(); ++k) { + // delete its row + result_table->erase(result_table->begin() + to_delete[k] * number_of_locations, result_table->begin() + (to_delete[k]+1) * number_of_locations); + --number_of_locations; + // delete its column + for (int j = 0; j < number_of_locations; ++j) { + result_table->erase(result_table->begin() + j * number_of_locations + to_delete[k]); + } + } + } + //todo: delete + if (*std::max_element(result_table->begin(), result_table->end()) == maxint) { + SimpleLogger().Write() << "SOMETHING WENT WRONG"; + } + // compute TSP round trip InternalRouteResult min_route; std::vector min_loc_permutation(phantom_node_vector.size(), -1); diff --git a/plugins/round_trip_FI.hpp b/plugins/round_trip_FI.hpp index f470052fa00..96197a48ad6 100644 --- a/plugins/round_trip_FI.hpp +++ b/plugins/round_trip_FI.hpp @@ -115,6 +115,37 @@ template class RoundTripPluginFI final : public BasePlugin return 400; } + auto number_of_locations = phantom_node_vector.size(); + const auto maxint = std::numeric_limits::max(); + + //////////////////////////////////// DELETE UNACCESSIBLE LOCATIONS ///////////////////////////////////////// + + if (*std::max_element(result_table->begin(), result_table->end()) == maxint) { + const int half = number_of_locations / 2; + std::vector to_delete; + + for (int i = number_of_locations - 1; i >= 0; --i) { + // if the location is unaccessible by most of the other locations, remember the location + if (std::count(result_table->begin() + i * number_of_locations, result_table->begin() + (i+1) * number_of_locations, maxint) > half) { + to_delete.push_back(i); + } + } + //delete all unaccessible locations + for (int k = 0; k < to_delete.size(); ++k) { + // delete its row + result_table->erase(result_table->begin() + to_delete[k] * number_of_locations, result_table->begin() + (to_delete[k]+1) * number_of_locations); + --number_of_locations; + // delete its column + for (int j = 0; j < number_of_locations; ++j) { + result_table->erase(result_table->begin() + j * number_of_locations + to_delete[k]); + } + } + } + //todo: delete + if (*std::max_element(result_table->begin(), result_table->end()) == maxint) { + SimpleLogger().Write() << "SOMETHING WENT WRONG"; + } + // compute TSP round trip InternalRouteResult min_route; std::vector min_loc_permutation(phantom_node_vector.size(), -1); diff --git a/plugins/round_trip_NN.hpp b/plugins/round_trip_NN.hpp index 4d6d674afdf..bc6c4cdc672 100644 --- a/plugins/round_trip_NN.hpp +++ b/plugins/round_trip_NN.hpp @@ -43,6 +43,7 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "../util/string_util.hpp" #include "../util/timing_util.hpp" #include "../util/simple_logger.hpp" +#include "../tools/tsp_logs.hpp" #include @@ -115,9 +116,40 @@ template class RoundTripPluginNN final : public BasePlugin return 400; } + auto number_of_locations = phantom_node_vector.size(); + const auto maxint = std::numeric_limits::max(); + + //////////////////////////////////// DELETE UNACCESSIBLE LOCATIONS ///////////////////////////////////////// + + if (*std::max_element(result_table->begin(), result_table->end()) == maxint) { + const int half = number_of_locations / 2; + std::vector to_delete; + + for (int i = number_of_locations - 1; i >= 0; --i) { + // if the location is unaccessible by most of the other locations, remember the location + if (std::count(result_table->begin() + i * number_of_locations, result_table->begin() + (i+1) * number_of_locations, maxint) > half) { + to_delete.push_back(i); + } + } + //delete all unaccessible locations + for (int k = 0; k < to_delete.size(); ++k) { + // delete its row + result_table->erase(result_table->begin() + to_delete[k] * number_of_locations, result_table->begin() + (to_delete[k]+1) * number_of_locations); + --number_of_locations; + // delete its column + for (int j = 0; j < number_of_locations; ++j) { + result_table->erase(result_table->begin() + j * number_of_locations + to_delete[k]); + } + } + } + //todo: delete + if (*std::max_element(result_table->begin(), result_table->end()) == maxint) { + SimpleLogger().Write() << "SOMETHING WENT WRONG"; + } + // compute TSP round trip InternalRouteResult min_route; - std::vector min_loc_permutation(phantom_node_vector.size(), -1); + std::vector min_loc_permutation(number_of_locations, -1); TIMER_STOP(tsp_pre); //######################### NEAREST NEIGHBOUR ###############################// diff --git a/routing_algorithms/tsp_nearest_neighbour.hpp b/routing_algorithms/tsp_nearest_neighbour.hpp index 113e8d36fdb..bf949229301 100644 --- a/routing_algorithms/tsp_nearest_neighbour.hpp +++ b/routing_algorithms/tsp_nearest_neighbour.hpp @@ -66,38 +66,10 @@ void NearestNeighbourTSP(const PhantomNodeArray & phantom_node_vector, const auto number_of_locations = phantom_node_vector.size(); min_route.shortest_path_length = std::numeric_limits::max(); - // is_lonely_island[i] indicates whether node i is a node that cannot be reached from other nodes - // 1 means that node i is a lonely island - // 0 means that it is not known for node i - // -1 means that node i is not a lonely island but a reachable, connected node - std::vector is_lonely_island(number_of_locations, 0); - int count_unreachables; - // ALWAYS START AT ANOTHER STARTING POINT for(int start_node = 0; start_node < number_of_locations; ++start_node) { - - if (is_lonely_island[start_node] >= 0) - { - // if node is a lonely island it is an unsuitable node to start from and shall be skipped - if (is_lonely_island[start_node]) - continue; - count_unreachables = 0; - auto start_dist_begin = dist_table.begin() + (start_node * number_of_locations); - auto start_dist_end = dist_table.begin() + ((start_node + 1) * number_of_locations); - for (auto it2 = start_dist_begin; it2 != start_dist_end; ++it2) { - if (*it2 == 0 || *it2 == std::numeric_limits::max()) { - ++count_unreachables; - } - } - if (count_unreachables >= number_of_locations) { - is_lonely_island[start_node] = 1; - continue; - } - } - int curr_node = start_node; - is_lonely_island[curr_node] = -1; InternalRouteResult raw_route; //TODO: Should we always use the same vector or does it not matter at all because of loop scope? std::vector loc_permutation(number_of_locations, -1); @@ -119,33 +91,18 @@ void NearestNeighbourTSP(const PhantomNodeArray & phantom_node_vector, auto row_end_iterator = dist_table.begin() + ((curr_node + 1) * number_of_locations); for (auto it = row_begin_iterator; it != row_end_iterator; ++it) { auto index = std::distance(row_begin_iterator, it); - if (is_lonely_island[index] < 1 && !visited[index] && *it < min_dist) + if (!visited[index] && *it < min_dist) { min_dist = *it; min_id = index; } } - // in case there was no unvisited and reachable node found, it means that all remaining (unvisited) nodes must be lonely islands - if (min_id == -1) - { - for(int loc = 0; loc < visited.size(); ++loc) { - if (!visited[loc]) { - is_lonely_island[loc] = 1; - } - } - break; - } - // set the nearest unvisited location as the next via_point - else - { - is_lonely_island[min_id] = -1; - loc_permutation[min_id] = via_point; - visited[min_id] = true; - viapoint = PhantomNodes{phantom_node_vector[curr_node][0], phantom_node_vector[min_id][0]}; - raw_route.segment_end_coordinates.emplace_back(viapoint); - trip_dist += min_dist; - curr_node = min_id; - } + loc_permutation[min_id] = via_point; + visited[min_id] = true; + viapoint = PhantomNodes{phantom_node_vector[curr_node][0], phantom_node_vector[min_id][0]}; + raw_route.segment_end_coordinates.emplace_back(viapoint); + trip_dist += min_dist; + curr_node = min_id; } // 4. ROUTE BACK TO STARTING POINT From 84c12793e881cfdcd08efcc4842cb6395b60f434 Mon Sep 17 00:00:00 2001 From: Chau Nguyen Date: Wed, 1 Jul 2015 14:07:25 +0200 Subject: [PATCH 070/122] clean up some code --- plugins/round_trip.hpp | 104 +++++++++++++++++++------------------- plugins/round_trip_BF.hpp | 88 +++++++++++++++++++------------- plugins/round_trip_FI.hpp | 99 ++++++++++++++++++------------------ plugins/round_trip_NN.hpp | 97 ++++++++++++++++++----------------- 4 files changed, 205 insertions(+), 183 deletions(-) diff --git a/plugins/round_trip.hpp b/plugins/round_trip.hpp index 92922671bc8..525c6ba9822 100644 --- a/plugins/round_trip.hpp +++ b/plugins/round_trip.hpp @@ -70,25 +70,14 @@ template class RoundTripPlugin final : public BasePlugin const std::string GetDescriptor() const override final { return descriptor_string; } - int HandleRequest(const RouteParameters &route_parameters, - osrm::json::Object &json_result) override final - { - TIMER_START(tsp_pre); - // check if all inputs are coordinates - if (!check_all_coordinates(route_parameters.coordinates)) - { - return 400; - } + void GetPhantomNodes(const RouteParameters &route_parameters, PhantomNodeArray & phantom_node_vector) { const bool checksum_OK = (route_parameters.check_sum == facade->GetCheckSum()); // find phantom nodes for all input coords - PhantomNodeArray phantom_node_vector(route_parameters.coordinates.size()); - for (const auto i : osrm::irange(0, route_parameters.coordinates.size())) - { + for (const auto i : osrm::irange(0, route_parameters.coordinates.size())) { // if client hints are helpful, encode hints if (checksum_OK && i < route_parameters.hints.size() && - !route_parameters.hints[i].empty()) - { + !route_parameters.hints[i].empty()) { PhantomNode current_phantom_node; ObjectEncoder::DecodeFromBase64(route_parameters.hints[i], current_phantom_node); if (current_phantom_node.is_valid(facade->GetNumberOfNodes())) @@ -99,74 +88,53 @@ template class RoundTripPlugin final : public BasePlugin } facade->IncrementalFindPhantomNodeForCoordinate(route_parameters.coordinates[i], phantom_node_vector[i], 1); - if (phantom_node_vector[i].size() > 1) - { + if (phantom_node_vector[i].size() > 1) { phantom_node_vector[i].erase(phantom_node_vector[i].begin()); } BOOST_ASSERT(phantom_node_vector[i].front().is_valid(facade->GetNumberOfNodes())); } + } - // compute the distance table of all phantom nodes - const std::shared_ptr> result_table = - search_engine_ptr->distance_table(phantom_node_vector); - - if (!result_table) - { - return 400; - } - + void SplitUnaccessibleLocations(PhantomNodeArray & phantom_node_vector, std::vector & result_table) { auto number_of_locations = phantom_node_vector.size(); const auto maxint = std::numeric_limits::max(); //////////////////////////////////// DELETE UNACCESSIBLE LOCATIONS ///////////////////////////////////////// - - if (*std::max_element(result_table->begin(), result_table->end()) == maxint) { + if (*std::max_element(result_table.begin(), result_table.end()) == maxint) { const int half = number_of_locations / 2; std::vector to_delete; for (int i = number_of_locations - 1; i >= 0; --i) { // if the location is unaccessible by most of the other locations, remember the location - if (std::count(result_table->begin() + i * number_of_locations, result_table->begin() + (i+1) * number_of_locations, maxint) > half) { + if (std::count(result_table.begin() + i * number_of_locations, result_table.begin() + (i+1) * number_of_locations, maxint) > half) { to_delete.push_back(i); } } //delete all unaccessible locations for (int k = 0; k < to_delete.size(); ++k) { // delete its row - result_table->erase(result_table->begin() + to_delete[k] * number_of_locations, result_table->begin() + (to_delete[k]+1) * number_of_locations); + result_table.erase(result_table.begin() + to_delete[k] * number_of_locations, result_table.begin() + (to_delete[k]+1) * number_of_locations); --number_of_locations; // delete its column for (int j = 0; j < number_of_locations; ++j) { - result_table->erase(result_table->begin() + j * number_of_locations + to_delete[k]); + result_table.erase(result_table.begin() + j * number_of_locations + to_delete[k]); } + // delete its PhantomNode + phantom_node_vector.erase(phantom_node_vector.begin() + to_delete[k]); } } - //todo: delete - if (*std::max_element(result_table->begin(), result_table->end()) == maxint) { - SimpleLogger().Write() << "SOMETHING WENT WRONG"; - } - - // compute TSP round trip - InternalRouteResult min_route; - std::vector min_loc_permutation(phantom_node_vector.size(), -1); - TIMER_STOP(tsp_pre); - - - - - //######################### NEAREST NEIGHBOUR ###############################// - TIMER_START(tsp); - osrm::tsp::NearestNeighbour(route_parameters, phantom_node_vector, *result_table, min_route, min_loc_permutation); - search_engine_ptr->shortest_path(min_route.segment_end_coordinates, route_parameters.uturns, min_route); - TIMER_STOP(tsp); - SimpleLogger().Write() << "Distance " << min_route.shortest_path_length; - SimpleLogger().Write() << "Time " << TIMER_MSEC(tsp) + TIMER_MSEC(tsp_pre); + } + void SetJSONOutput (const RouteParameters &route_parameters, + int tsp_time, + InternalRouteResult & min_route, + std::vector & min_loc_permutation, + osrm::json::Object & json_result){ osrm::json::Array json_loc_permutation; json_loc_permutation.values.insert(json_loc_permutation.values.end(), min_loc_permutation.begin(), min_loc_permutation.end()); json_result.values["loc_permutation"] = json_loc_permutation; json_result.values["distance"] = min_route.shortest_path_length; - json_result.values["runtime"] = TIMER_MSEC(tsp); + json_result.values["runtime"] = tsp_time; @@ -224,6 +192,40 @@ template class RoundTripPlugin final : public BasePlugin descriptor->SetConfig(route_parameters); descriptor->Run(min_route, json_result); + } + + int HandleRequest(const RouteParameters &route_parameters, + osrm::json::Object &json_result) override final + { + // check if all inputs are coordinates + if (!check_all_coordinates(route_parameters.coordinates)) { + return 400; + } + + PhantomNodeArray phantom_node_vector(route_parameters.coordinates.size()); + GetPhantomNodes(route_parameters, phantom_node_vector); + + // compute the distance table of all phantom nodes + const std::shared_ptr> result_table = + search_engine_ptr->distance_table(phantom_node_vector); + if (!result_table){ + return 400; + } + + SplitUnaccessibleLocations(phantom_node_vector, *result_table); + + auto number_of_locations = phantom_node_vector.size(); + InternalRouteResult min_route; + std::vector min_loc_permutation(number_of_locations, -1); + //######################## FARTHEST INSERTION ###############################// + TIMER_START(tsp); + osrm::tsp::FarthestInsertionTSP(phantom_node_vector, *result_table, min_route, min_loc_permutation); + search_engine_ptr->shortest_path(min_route.segment_end_coordinates, route_parameters.uturns, min_route); + TIMER_STOP(tsp); + + BOOST_ASSERT(min_route.segment_end_coordinates.size() == route_parameters.coordinates.size()); + + SetJSONOutput(route_parameters, TIMER_MSEC(tsp), min_route, min_loc_permutation, json_result); diff --git a/plugins/round_trip_BF.hpp b/plugins/round_trip_BF.hpp index 60c12c2b5d8..7a667604ad6 100644 --- a/plugins/round_trip_BF.hpp +++ b/plugins/round_trip_BF.hpp @@ -1,3 +1,4 @@ + /* Copyright (c) 2015, Project OSRM contributors @@ -70,25 +71,14 @@ template class RoundTripPluginBF final : public BasePlugin const std::string GetDescriptor() const override final { return descriptor_string; } - int HandleRequest(const RouteParameters &route_parameters, - osrm::json::Object &json_result) override final - { - TIMER_START(tsp_pre); - // check if all inputs are coordinates - if (!check_all_coordinates(route_parameters.coordinates)) - { - return 400; - } + void GetPhantomNodes(const RouteParameters &route_parameters, PhantomNodeArray & phantom_node_vector) { const bool checksum_OK = (route_parameters.check_sum == facade->GetCheckSum()); // find phantom nodes for all input coords - PhantomNodeArray phantom_node_vector(route_parameters.coordinates.size()); - for (const auto i : osrm::irange(0, route_parameters.coordinates.size())) - { + for (const auto i : osrm::irange(0, route_parameters.coordinates.size())) { // if client hints are helpful, encode hints if (checksum_OK && i < route_parameters.hints.size() && - !route_parameters.hints[i].empty()) - { + !route_parameters.hints[i].empty()) { PhantomNode current_phantom_node; ObjectEncoder::DecodeFromBase64(route_parameters.hints[i], current_phantom_node); if (current_phantom_node.is_valid(facade->GetNumberOfNodes())) @@ -99,58 +89,84 @@ template class RoundTripPluginBF final : public BasePlugin } facade->IncrementalFindPhantomNodeForCoordinate(route_parameters.coordinates[i], phantom_node_vector[i], 1); - if (phantom_node_vector[i].size() > 1) - { + if (phantom_node_vector[i].size() > 1) { phantom_node_vector[i].erase(phantom_node_vector[i].begin()); } BOOST_ASSERT(phantom_node_vector[i].front().is_valid(facade->GetNumberOfNodes())); } + } - // compute the distance table of all phantom nodes - const std::shared_ptr> result_table = - search_engine_ptr->distance_table(phantom_node_vector); - - if (!result_table) - { - return 400; - } - + void SplitUnaccessibleLocations(PhantomNodeArray & phantom_node_vector, std::vector & result_table) { auto number_of_locations = phantom_node_vector.size(); const auto maxint = std::numeric_limits::max(); //////////////////////////////////// DELETE UNACCESSIBLE LOCATIONS ///////////////////////////////////////// - - if (*std::max_element(result_table->begin(), result_table->end()) == maxint) { + if (*std::max_element(result_table.begin(), result_table.end()) == maxint) { const int half = number_of_locations / 2; std::vector to_delete; for (int i = number_of_locations - 1; i >= 0; --i) { // if the location is unaccessible by most of the other locations, remember the location - if (std::count(result_table->begin() + i * number_of_locations, result_table->begin() + (i+1) * number_of_locations, maxint) > half) { + if (std::count(result_table.begin() + i * number_of_locations, result_table.begin() + (i+1) * number_of_locations, maxint) > half) { to_delete.push_back(i); } } //delete all unaccessible locations for (int k = 0; k < to_delete.size(); ++k) { // delete its row - result_table->erase(result_table->begin() + to_delete[k] * number_of_locations, result_table->begin() + (to_delete[k]+1) * number_of_locations); + result_table.erase(result_table.begin() + to_delete[k] * number_of_locations, result_table.begin() + (to_delete[k]+1) * number_of_locations); --number_of_locations; // delete its column for (int j = 0; j < number_of_locations; ++j) { - result_table->erase(result_table->begin() + j * number_of_locations + to_delete[k]); + result_table.erase(result_table.begin() + j * number_of_locations + to_delete[k]); } + // delete its PhantomNode + phantom_node_vector.erase(phantom_node_vector.begin() + to_delete[k]); } } - //todo: delete - if (*std::max_element(result_table->begin(), result_table->end()) == maxint) { - SimpleLogger().Write() << "SOMETHING WENT WRONG"; + } + + void SetJSONOutput (const RouteParameters &route_parameters, + int tsp_time, + InternalRouteResult & min_route, + std::vector & min_loc_permutation, + osrm::json::Object & json_result){ + osrm::json::Array json_loc_permutation; + json_loc_permutation.values.insert(json_loc_permutation.values.end(), min_loc_permutation.begin(), min_loc_permutation.end()); + json_result.values["loc_permutation"] = json_loc_permutation; + json_result.values["distance"] = min_route.shortest_path_length; + json_result.values["runtime"] = tsp_time; + + // return geometry result to json + std::unique_ptr> descriptor; + descriptor = osrm::make_unique>(facade); + + descriptor->SetConfig(route_parameters); + descriptor->Run(min_route, json_result); + } + + int HandleRequest(const RouteParameters &route_parameters, + osrm::json::Object &json_result) override final + { + // check if all inputs are coordinates + if (!check_all_coordinates(route_parameters.coordinates)) { + return 400; } - // compute TSP round trip + PhantomNodeArray phantom_node_vector(route_parameters.coordinates.size()); + GetPhantomNodes(route_parameters, phantom_node_vector); + + // compute the distance table of all phantom nodes + const std::shared_ptr> result_table = + search_engine_ptr->distance_table(phantom_node_vector); + if (!result_table){ + return 400; + } + + SplitUnaccessibleLocations(phantom_node_vector, *result_table); + InternalRouteResult min_route; std::vector min_loc_permutation(phantom_node_vector.size(), -1); - TIMER_STOP(tsp_pre); - //########################### BRUTE FORCE ####################################// if (route_parameters.coordinates.size() < 11) { TIMER_START(tsp); diff --git a/plugins/round_trip_FI.hpp b/plugins/round_trip_FI.hpp index 96197a48ad6..b06220055c0 100644 --- a/plugins/round_trip_FI.hpp +++ b/plugins/round_trip_FI.hpp @@ -70,25 +70,14 @@ template class RoundTripPluginFI final : public BasePlugin const std::string GetDescriptor() const override final { return descriptor_string; } - int HandleRequest(const RouteParameters &route_parameters, - osrm::json::Object &json_result) override final - { - TIMER_START(tsp_pre); - // check if all inputs are coordinates - if (!check_all_coordinates(route_parameters.coordinates)) - { - return 400; - } + void GetPhantomNodes(const RouteParameters &route_parameters, PhantomNodeArray & phantom_node_vector) { const bool checksum_OK = (route_parameters.check_sum == facade->GetCheckSum()); // find phantom nodes for all input coords - PhantomNodeArray phantom_node_vector(route_parameters.coordinates.size()); - for (const auto i : osrm::irange(0, route_parameters.coordinates.size())) - { + for (const auto i : osrm::irange(0, route_parameters.coordinates.size())) { // if client hints are helpful, encode hints if (checksum_OK && i < route_parameters.hints.size() && - !route_parameters.hints[i].empty()) - { + !route_parameters.hints[i].empty()) { PhantomNode current_phantom_node; ObjectEncoder::DecodeFromBase64(route_parameters.hints[i], current_phantom_node); if (current_phantom_node.is_valid(facade->GetNumberOfNodes())) @@ -99,72 +88,53 @@ template class RoundTripPluginFI final : public BasePlugin } facade->IncrementalFindPhantomNodeForCoordinate(route_parameters.coordinates[i], phantom_node_vector[i], 1); - if (phantom_node_vector[i].size() > 1) - { + if (phantom_node_vector[i].size() > 1) { phantom_node_vector[i].erase(phantom_node_vector[i].begin()); } BOOST_ASSERT(phantom_node_vector[i].front().is_valid(facade->GetNumberOfNodes())); } + } - // compute the distance table of all phantom nodes - const std::shared_ptr> result_table = - search_engine_ptr->distance_table(phantom_node_vector); - - if (!result_table) - { - return 400; - } - + void SplitUnaccessibleLocations(PhantomNodeArray & phantom_node_vector, std::vector & result_table) { auto number_of_locations = phantom_node_vector.size(); const auto maxint = std::numeric_limits::max(); //////////////////////////////////// DELETE UNACCESSIBLE LOCATIONS ///////////////////////////////////////// - - if (*std::max_element(result_table->begin(), result_table->end()) == maxint) { + if (*std::max_element(result_table.begin(), result_table.end()) == maxint) { const int half = number_of_locations / 2; std::vector to_delete; for (int i = number_of_locations - 1; i >= 0; --i) { // if the location is unaccessible by most of the other locations, remember the location - if (std::count(result_table->begin() + i * number_of_locations, result_table->begin() + (i+1) * number_of_locations, maxint) > half) { + if (std::count(result_table.begin() + i * number_of_locations, result_table.begin() + (i+1) * number_of_locations, maxint) > half) { to_delete.push_back(i); } } //delete all unaccessible locations for (int k = 0; k < to_delete.size(); ++k) { // delete its row - result_table->erase(result_table->begin() + to_delete[k] * number_of_locations, result_table->begin() + (to_delete[k]+1) * number_of_locations); + result_table.erase(result_table.begin() + to_delete[k] * number_of_locations, result_table.begin() + (to_delete[k]+1) * number_of_locations); --number_of_locations; // delete its column for (int j = 0; j < number_of_locations; ++j) { - result_table->erase(result_table->begin() + j * number_of_locations + to_delete[k]); + result_table.erase(result_table.begin() + j * number_of_locations + to_delete[k]); } + // delete its PhantomNode + phantom_node_vector.erase(phantom_node_vector.begin() + to_delete[k]); } } - //todo: delete - if (*std::max_element(result_table->begin(), result_table->end()) == maxint) { - SimpleLogger().Write() << "SOMETHING WENT WRONG"; - } - - // compute TSP round trip - InternalRouteResult min_route; - std::vector min_loc_permutation(phantom_node_vector.size(), -1); - TIMER_STOP(tsp_pre); - - //######################## FARTHEST INSERTION ###############################// - TIMER_START(tsp); - osrm::tsp::FarthestInsertionTSP(phantom_node_vector, *result_table, min_route, min_loc_permutation); - search_engine_ptr->shortest_path(min_route.segment_end_coordinates, route_parameters.uturns, min_route); - TIMER_STOP(tsp); - - BOOST_ASSERT(min_route.segment_end_coordinates.size() == route_parameters.coordinates.size()); + } + void SetJSONOutput (const RouteParameters &route_parameters, + int tsp_time, + InternalRouteResult & min_route, + std::vector & min_loc_permutation, + osrm::json::Object & json_result){ osrm::json::Array json_loc_permutation; json_loc_permutation.values.insert(json_loc_permutation.values.end(), min_loc_permutation.begin(), min_loc_permutation.end()); json_result.values["loc_permutation"] = json_loc_permutation; json_result.values["distance"] = min_route.shortest_path_length; - SimpleLogger().Write() << "FI GEOM DISTANCE " << min_route.shortest_path_length; - json_result.values["runtime"] = TIMER_MSEC(tsp); + json_result.values["runtime"] = tsp_time; // return geometry result to json std::unique_ptr> descriptor; @@ -172,8 +142,39 @@ template class RoundTripPluginFI final : public BasePlugin descriptor->SetConfig(route_parameters); descriptor->Run(min_route, json_result); + } + + int HandleRequest(const RouteParameters &route_parameters, + osrm::json::Object &json_result) override final + { + // check if all inputs are coordinates + if (!check_all_coordinates(route_parameters.coordinates)) { + return 400; + } + + PhantomNodeArray phantom_node_vector(route_parameters.coordinates.size()); + GetPhantomNodes(route_parameters, phantom_node_vector); + // compute the distance table of all phantom nodes + const std::shared_ptr> result_table = + search_engine_ptr->distance_table(phantom_node_vector); + if (!result_table){ + return 400; + } + + SplitUnaccessibleLocations(phantom_node_vector, *result_table); + + InternalRouteResult min_route; + std::vector min_loc_permutation(phantom_node_vector.size(), -1); + //######################## FARTHEST INSERTION ###############################// + TIMER_START(tsp); + osrm::tsp::FarthestInsertionTSP(phantom_node_vector, *result_table, min_route, min_loc_permutation); + search_engine_ptr->shortest_path(min_route.segment_end_coordinates, route_parameters.uturns, min_route); + TIMER_STOP(tsp); + + BOOST_ASSERT(min_route.segment_end_coordinates.size() == route_parameters.coordinates.size()); + SetJSONOutput(route_parameters, TIMER_MSEC(tsp), min_route, min_loc_permutation, json_result); return 200; } diff --git a/plugins/round_trip_NN.hpp b/plugins/round_trip_NN.hpp index bc6c4cdc672..620d0fabcc8 100644 --- a/plugins/round_trip_NN.hpp +++ b/plugins/round_trip_NN.hpp @@ -71,25 +71,14 @@ template class RoundTripPluginNN final : public BasePlugin const std::string GetDescriptor() const override final { return descriptor_string; } - int HandleRequest(const RouteParameters &route_parameters, - osrm::json::Object &json_result) override final - { - TIMER_START(tsp_pre); - // check if all inputs are coordinates - if (!check_all_coordinates(route_parameters.coordinates)) - { - return 400; - } + void GetPhantomNodes(const RouteParameters &route_parameters, PhantomNodeArray & phantom_node_vector) { const bool checksum_OK = (route_parameters.check_sum == facade->GetCheckSum()); // find phantom nodes for all input coords - PhantomNodeArray phantom_node_vector(route_parameters.coordinates.size()); - for (const auto i : osrm::irange(0, route_parameters.coordinates.size())) - { + for (const auto i : osrm::irange(0, route_parameters.coordinates.size())) { // if client hints are helpful, encode hints if (checksum_OK && i < route_parameters.hints.size() && - !route_parameters.hints[i].empty()) - { + !route_parameters.hints[i].empty()) { PhantomNode current_phantom_node; ObjectEncoder::DecodeFromBase64(route_parameters.hints[i], current_phantom_node); if (current_phantom_node.is_valid(facade->GetNumberOfNodes())) @@ -100,70 +89,53 @@ template class RoundTripPluginNN final : public BasePlugin } facade->IncrementalFindPhantomNodeForCoordinate(route_parameters.coordinates[i], phantom_node_vector[i], 1); - if (phantom_node_vector[i].size() > 1) - { + if (phantom_node_vector[i].size() > 1) { phantom_node_vector[i].erase(phantom_node_vector[i].begin()); } BOOST_ASSERT(phantom_node_vector[i].front().is_valid(facade->GetNumberOfNodes())); } + } - // compute the distance table of all phantom nodes - const std::shared_ptr> result_table = - search_engine_ptr->distance_table(phantom_node_vector); - - if (!result_table) - { - return 400; - } - + void SplitUnaccessibleLocations(PhantomNodeArray & phantom_node_vector, std::vector & result_table) { auto number_of_locations = phantom_node_vector.size(); const auto maxint = std::numeric_limits::max(); //////////////////////////////////// DELETE UNACCESSIBLE LOCATIONS ///////////////////////////////////////// - - if (*std::max_element(result_table->begin(), result_table->end()) == maxint) { + if (*std::max_element(result_table.begin(), result_table.end()) == maxint) { const int half = number_of_locations / 2; std::vector to_delete; for (int i = number_of_locations - 1; i >= 0; --i) { // if the location is unaccessible by most of the other locations, remember the location - if (std::count(result_table->begin() + i * number_of_locations, result_table->begin() + (i+1) * number_of_locations, maxint) > half) { + if (std::count(result_table.begin() + i * number_of_locations, result_table.begin() + (i+1) * number_of_locations, maxint) > half) { to_delete.push_back(i); } } //delete all unaccessible locations for (int k = 0; k < to_delete.size(); ++k) { // delete its row - result_table->erase(result_table->begin() + to_delete[k] * number_of_locations, result_table->begin() + (to_delete[k]+1) * number_of_locations); + result_table.erase(result_table.begin() + to_delete[k] * number_of_locations, result_table.begin() + (to_delete[k]+1) * number_of_locations); --number_of_locations; // delete its column for (int j = 0; j < number_of_locations; ++j) { - result_table->erase(result_table->begin() + j * number_of_locations + to_delete[k]); + result_table.erase(result_table.begin() + j * number_of_locations + to_delete[k]); } + // delete its PhantomNode + phantom_node_vector.erase(phantom_node_vector.begin() + to_delete[k]); } } - //todo: delete - if (*std::max_element(result_table->begin(), result_table->end()) == maxint) { - SimpleLogger().Write() << "SOMETHING WENT WRONG"; - } - - // compute TSP round trip - InternalRouteResult min_route; - std::vector min_loc_permutation(number_of_locations, -1); - TIMER_STOP(tsp_pre); - - //######################### NEAREST NEIGHBOUR ###############################// - TIMER_START(tsp); - osrm::tsp::NearestNeighbourTSP(phantom_node_vector, *result_table, min_route, min_loc_permutation); - search_engine_ptr->shortest_path(min_route.segment_end_coordinates, route_parameters.uturns, min_route); - TIMER_STOP(tsp); - BOOST_ASSERT(min_route.segment_end_coordinates.size() == route_parameters.coordinates.size()); + } + void SetJSONOutput (const RouteParameters &route_parameters, + int tsp_time, + InternalRouteResult & min_route, + std::vector & min_loc_permutation, + osrm::json::Object & json_result){ osrm::json::Array json_loc_permutation; json_loc_permutation.values.insert(json_loc_permutation.values.end(), min_loc_permutation.begin(), min_loc_permutation.end()); json_result.values["loc_permutation"] = json_loc_permutation; json_result.values["distance"] = min_route.shortest_path_length; - json_result.values["runtime"] = TIMER_MSEC(tsp); + json_result.values["runtime"] = tsp_time; // return geometry result to json std::unique_ptr> descriptor; @@ -171,8 +143,39 @@ template class RoundTripPluginNN final : public BasePlugin descriptor->SetConfig(route_parameters); descriptor->Run(min_route, json_result); + } + + int HandleRequest(const RouteParameters &route_parameters, + osrm::json::Object &json_result) override final + { + // check if all inputs are coordinates + if (!check_all_coordinates(route_parameters.coordinates)) { + return 400; + } + + PhantomNodeArray phantom_node_vector(route_parameters.coordinates.size()); + GetPhantomNodes(route_parameters, phantom_node_vector); + + // compute the distance table of all phantom nodes + const std::shared_ptr> result_table = + search_engine_ptr->distance_table(phantom_node_vector); + if (!result_table){ + return 400; + } + SplitUnaccessibleLocations(phantom_node_vector, *result_table); + + InternalRouteResult min_route; + std::vector min_loc_permutation(phantom_node_vector.size(), -1); + //######################### NEAREST NEIGHBOUR ###############################// + TIMER_START(tsp); + osrm::tsp::NearestNeighbourTSP(phantom_node_vector, *result_table, min_route, min_loc_permutation); + search_engine_ptr->shortest_path(min_route.segment_end_coordinates, route_parameters.uturns, min_route); + TIMER_STOP(tsp); + + BOOST_ASSERT(min_route.segment_end_coordinates.size() == route_parameters.coordinates.size()); + SetJSONOutput(route_parameters, TIMER_MSEC(tsp), min_route, min_loc_permutation, json_result); return 200; } From b15f8f68e4c4287a6d677c7efe5a1760410b7a39 Mon Sep 17 00:00:00 2001 From: Chau Nguyen Date: Sun, 5 Jul 2015 00:15:55 +0200 Subject: [PATCH 071/122] refactor and improve the round trip computation of multiple SCCs Problem: - old solution was slow - depending on the result of TarjanSCC, new distance tables and new phantom node vectors were created to run tsp on it Solution: - dont create new distance tables and phantom node vectors - pass an additional vector with the information which locations are in the same component and ignore all others fix bug for scc split computation --- CMakeLists.txt | 2 +- algorithms/tarjan_scc.hpp | 5 + data_structures/matrix_graph_wrapper.hpp | 66 +++++++ plugins/round_trip.hpp | 171 ++++++++--------- plugins/round_trip_BF.hpp | 133 +++++++------ plugins/round_trip_FI.hpp | 115 +++++++----- plugins/round_trip_NN.hpp | 116 +++++++----- routing_algorithms/tsp_brute_force.hpp | 43 ++++- routing_algorithms/tsp_farthest_insertion.hpp | 174 +++++++++++++----- routing_algorithms/tsp_nearest_neighbour.hpp | 122 ++++++++++++ 10 files changed, 647 insertions(+), 300 deletions(-) create mode 100644 data_structures/matrix_graph_wrapper.hpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 89c2c58f887..990ce884c19 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -91,7 +91,7 @@ set( add_library(COORDINATE OBJECT ${CoordinateGlob}) add_library(GITDESCRIPTION OBJECT util/git_sha.cpp) -add_library(OSRM ${OSRMSources} $ $ $ $ $ $ $ $ $ $) +add_library(OSRM ${OSRMSources} $ $ $ $ $ $ $ $ $ $ $) add_library(FINGERPRINT OBJECT util/fingerprint.cpp) add_dependencies(FINGERPRINT FingerPrintConfigure) diff --git a/algorithms/tarjan_scc.hpp b/algorithms/tarjan_scc.hpp index 0d8743543bb..05bbb0422a0 100644 --- a/algorithms/tarjan_scc.hpp +++ b/algorithms/tarjan_scc.hpp @@ -201,6 +201,11 @@ template class TarjanSCC return component_size_vector[component_id]; } + unsigned get_component_size_by_id(const unsigned component_id) const + { + return component_size_vector[component_id]; + } + unsigned get_component_id(const NodeID node) const { return components_index[node]; } }; diff --git a/data_structures/matrix_graph_wrapper.hpp b/data_structures/matrix_graph_wrapper.hpp new file mode 100644 index 00000000000..3bf9259ec17 --- /dev/null +++ b/data_structures/matrix_graph_wrapper.hpp @@ -0,0 +1,66 @@ +/* + +Copyright (c) 2015, Project OSRM contributors +All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + +Redistributions of source code must retain the above copyright notice, this list +of conditions and the following disclaimer. +Redistributions in binary form must reproduce the above copyright notice, this +list of conditions and the following disclaimer in the documentation and/or +other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +*/ + +#ifndef MATRIX_GRAPH_WRAPPER_H +#define MATRIX_GRAPH_WRAPPER_H + +#include + +//This Wrapper provides all methods that are needed for TarjanSCC, when the graph is given in a +//matrix representation (e.g. as output from a distance table call) + +template class MatrixGraphWrapper { +public: + + MatrixGraphWrapper(std::vector table, const unsigned number_of_nodes) : table_(table), number_of_nodes_(number_of_nodes) {}; + + unsigned GetNumberOfNodes() { + return number_of_nodes_; + } + + std::vector GetAdjacentEdgeRange(const unsigned node) const { + std::vector edges; + const auto maxint = std::numeric_limits::max(); + for (auto i = 0; i < number_of_nodes_; ++i) { + if (*(table_.begin() + node * number_of_nodes_ + i) != maxint) { + edges.push_back(i); + } + } + return edges; + } + + unsigned GetTarget(const unsigned edge) { + return edge; + } + +private: + std::vector table_; + const unsigned number_of_nodes_; +}; + + +#endif // MATRIX_GRAPH_WRAPPER_H diff --git a/plugins/round_trip.hpp b/plugins/round_trip.hpp index 525c6ba9822..bdb42a6ef88 100644 --- a/plugins/round_trip.hpp +++ b/plugins/round_trip.hpp @@ -31,11 +31,13 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "plugin_base.hpp" #include "../algorithms/object_encoder.hpp" +#include "../algorithms/tiny_components.hpp" #include "../routing_algorithms/tsp_nearest_neighbour.hpp" #include "../routing_algorithms/tsp_farthest_insertion.hpp" #include "../routing_algorithms/tsp_brute_force.hpp" #include "../data_structures/query_edge.hpp" #include "../data_structures/search_engine.hpp" +#include "../data_structures/matrix_graph_wrapper.hpp" #include "../descriptors/descriptor_base.hpp" #include "../descriptors/json_descriptor.hpp" #include "../util/json_renderer.hpp" @@ -95,97 +97,41 @@ template class RoundTripPlugin final : public BasePlugin } } - void SplitUnaccessibleLocations(PhantomNodeArray & phantom_node_vector, std::vector & result_table) { - auto number_of_locations = phantom_node_vector.size(); - const auto maxint = std::numeric_limits::max(); + void SplitUnaccessibleLocations(PhantomNodeArray & phantom_node_vector, + std::vector & result_table, + std::vector> & components) { - //////////////////////////////////// DELETE UNACCESSIBLE LOCATIONS ///////////////////////////////////////// - if (*std::max_element(result_table.begin(), result_table.end()) == maxint) { - const int half = number_of_locations / 2; - std::vector to_delete; + // Run TarjanSCC + auto number_of_locations = phantom_node_vector.size(); + auto wrapper = std::make_shared>(result_table, number_of_locations); + auto empty_restriction = RestrictionMap(std::vector()); + auto empty_vector = std::vector(); + auto scc = TarjanSCC>(wrapper, empty_restriction, empty_vector); + scc.run(); + + for (int j = 0; j < scc.get_number_of_components(); ++j){ + components.push_back(std::vector()); + } - for (int i = number_of_locations - 1; i >= 0; --i) { - // if the location is unaccessible by most of the other locations, remember the location - if (std::count(result_table.begin() + i * number_of_locations, result_table.begin() + (i+1) * number_of_locations, maxint) > half) { - to_delete.push_back(i); - } - } - //delete all unaccessible locations - for (int k = 0; k < to_delete.size(); ++k) { - // delete its row - result_table.erase(result_table.begin() + to_delete[k] * number_of_locations, result_table.begin() + (to_delete[k]+1) * number_of_locations); - --number_of_locations; - // delete its column - for (int j = 0; j < number_of_locations; ++j) { - result_table.erase(result_table.begin() + j * number_of_locations + to_delete[k]); - } - // delete its PhantomNode - phantom_node_vector.erase(phantom_node_vector.begin() + to_delete[k]); - } + for (int i = 0; i < number_of_locations; ++i) { + components[scc.get_component_id(i)].push_back(i); } } - void SetJSONOutput (const RouteParameters &route_parameters, - int tsp_time, - InternalRouteResult & min_route, - std::vector & min_loc_permutation, - osrm::json::Object & json_result){ + void SetLocPermutationOutput(const std::vector & loc_permutation, osrm::json::Object & json_result){ osrm::json::Array json_loc_permutation; - json_loc_permutation.values.insert(json_loc_permutation.values.end(), min_loc_permutation.begin(), min_loc_permutation.end()); + json_loc_permutation.values.insert(json_loc_permutation.values.end(), loc_permutation.begin(), loc_permutation.end()); json_result.values["loc_permutation"] = json_loc_permutation; - json_result.values["distance"] = min_route.shortest_path_length; - json_result.values["runtime"] = tsp_time; - - - - // if (route_parameters.tsp_algo.compare("NN")) - // //######################### NEAREST NEIGHBOUR ###############################// - // TIMER_START(tsp); - // osrm::tsp::NearestNeighbour(route_parameters, phantom_node_vector, *result_table, min_route, min_loc_permutation); - // search_engine_ptr->shortest_path(min_route.segment_end_coordinates, route_parameters.uturns, min_route); - // TIMER_STOP(tsp); - // SimpleLogger().Write() << "Distance " << min_route.shortest_path_length; - // SimpleLogger().Write() << "Time " << TIMER_MSEC(tsp) + TIMER_MSEC(tsp_pre); - - // osrm::json::Array json_loc_permutation; - // json_loc_permutation.values.insert(json_loc_permutation.values.end(), min_loc_permutation.begin(), min_loc_permutation.end()); - // json_result.values["loc_permutation"] = json_loc_permutation; - // json_result.values["distance"] = min_route.shortest_path_length; - // json_result.values["runtime"] = TIMER_MSEC(tsp); - // else if (route_parameters.tsp_algo.compare("BF") - // //########################### BRUTE FORCE ####################################// - // if (route_parameters.coordinates.size() < 12) { - // TIMER_START(tsp); - // osrm::tsp::BruteForce(route_parameters, phantom_node_vector, *result_table, min_route, min_loc_permutation); - // search_engine_ptr->shortest_path(min_route.segment_end_coordinates, route_parameters.uturns, min_route); - // TIMER_STOP(tsp); - // SimpleLogger().Write() << "Distance " << min_route.shortest_path_length; - // SimpleLogger().Write() << "Time " << TIMER_MSEC(tsp); - - // osrm::json::Array json_loc_permutation; - // json_loc_permutation.values.insert(json_loc_permutation.values.end(), min_loc_permutation.begin(), min_loc_permutation.end()); - // json_result.values["loc_permutation"] = json_loc_permutation; - // json_result.values["distance"] = min_route.shortest_path_length; - // json_result.values["runtime"] = TIMER_MSEC(tsp); - // } else { - // json_result.values["distance"] = -1; - // json_result.values["runtime"] = -1; - // } - // else - // //######################## FARTHEST INSERTION ###############################// - // TIMER_START(tsp); - // osrm::tsp::FarthestInsertion(route_parameters, phantom_node_vector, *result_table, min_route, min_loc_permutation); - // search_engine_ptr->shortest_path(min_route.segment_end_coordinates, route_parameters.uturns, min_route); - // TIMER_STOP(tsp); - // SimpleLogger().Write() << "Distance " << min_route.shortest_path_length; - // SimpleLogger().Write() << "Time " << TIMER_MSEC(tsp); - - // osrm::json::Array json_loc_permutation; - // json_loc_permutation.values.insert(json_loc_permutation.values.end(), min_loc_permutation.begin(), min_loc_permutation.end()); - // json_result.values["loc_permutation"] = json_loc_permutation; - // json_result.values["distance"] = min_route.shortest_path_length; - // json_result.values["runtime"] = TIMER_MSEC(tsp); + } + void SetDistanceOutput(const int distance, osrm::json::Object & json_result) { + json_result.values["distance"] = distance; + } + void SetRuntimeOutput(const float runtime, osrm::json::Object & json_result) { + json_result.values["runtime"] = runtime; + } + + void SetGeometry(const RouteParameters &route_parameters, const InternalRouteResult & min_route, osrm::json::Object & json_result) { // return geometry result to json std::unique_ptr> descriptor; descriptor = osrm::make_unique>(facade); @@ -212,20 +158,53 @@ template class RoundTripPlugin final : public BasePlugin return 400; } - SplitUnaccessibleLocations(phantom_node_vector, *result_table); - - auto number_of_locations = phantom_node_vector.size(); - InternalRouteResult min_route; - std::vector min_loc_permutation(number_of_locations, -1); - //######################## FARTHEST INSERTION ###############################// - TIMER_START(tsp); - osrm::tsp::FarthestInsertionTSP(phantom_node_vector, *result_table, min_route, min_loc_permutation); - search_engine_ptr->shortest_path(min_route.segment_end_coordinates, route_parameters.uturns, min_route); - TIMER_STOP(tsp); - - BOOST_ASSERT(min_route.segment_end_coordinates.size() == route_parameters.coordinates.size()); - SetJSONOutput(route_parameters, TIMER_MSEC(tsp), min_route, min_loc_permutation, json_result); + const auto maxint = std::numeric_limits::max(); + if (*std::max_element(result_table->begin(), result_table->end()) == maxint) { + std::unique_ptr> descriptor; + descriptor = osrm::make_unique>(facade); + descriptor->SetConfig(route_parameters); + + std::vector> components; + TIMER_START(tsp); + SplitUnaccessibleLocations(phantom_node_vector, *result_table, components); + auto number_of_locations = phantom_node_vector.size(); + std::vector min_loc_permutation(number_of_locations, -1); + auto min_dist = 0; + for(auto k = 0; k < components.size(); ++k) { + if (components[k].size() > 1) { + InternalRouteResult min_route; + osrm::tsp::FarthestInsertionTSP(components[k], phantom_node_vector, *result_table, min_route, min_loc_permutation); + search_engine_ptr->shortest_path(min_route.segment_end_coordinates, route_parameters.uturns, min_route); + min_dist += min_route.shortest_path_length; + descriptor->Run(min_route, json_result); + } + } + TIMER_STOP(tsp); + + SetRuntimeOutput(TIMER_MSEC(tsp), json_result); + SetDistanceOutput(min_dist, json_result); + SetLocPermutationOutput(min_loc_permutation, json_result); + } else { + auto number_of_locations = phantom_node_vector.size(); + InternalRouteResult min_route; + std::vector min_loc_permutation(number_of_locations, -1); + //######################## FARTHEST INSERTION ###############################// + TIMER_START(tsp); + osrm::tsp::FarthestInsertionTSP(phantom_node_vector, *result_table, min_route, min_loc_permutation); + search_engine_ptr->shortest_path(min_route.segment_end_coordinates, route_parameters.uturns, min_route); + TIMER_STOP(tsp); + // //######################### NEAREST NEIGHBOUR ###############################// + // TIMER_START(tsp); + // osrm::tsp::NearestNeighbourTSP(phantom_node_vector, *result_table, min_route, min_loc_permutation); + // search_engine_ptr->shortest_path(min_route.segment_end_coordinates, route_parameters.uturns, min_route); + // TIMER_STOP(tsp); + BOOST_ASSERT(min_route.segment_end_coordinates.size() == route_parameters.coordinates.size()); + SetLocPermutationOutput(min_loc_permutation, json_result); + SetDistanceOutput(min_route.shortest_path_length, json_result); + SetRuntimeOutput(TIMER_MSEC(tsp), json_result); + SetGeometry(route_parameters, min_route, json_result); + } diff --git a/plugins/round_trip_BF.hpp b/plugins/round_trip_BF.hpp index 7a667604ad6..10510aa0323 100644 --- a/plugins/round_trip_BF.hpp +++ b/plugins/round_trip_BF.hpp @@ -96,47 +96,41 @@ template class RoundTripPluginBF final : public BasePlugin } } - void SplitUnaccessibleLocations(PhantomNodeArray & phantom_node_vector, std::vector & result_table) { + void SplitUnaccessibleLocations(PhantomNodeArray & phantom_node_vector, + std::vector & result_table, + std::vector> & components) { + // Run TarjanSCC auto number_of_locations = phantom_node_vector.size(); - const auto maxint = std::numeric_limits::max(); - - //////////////////////////////////// DELETE UNACCESSIBLE LOCATIONS ///////////////////////////////////////// - if (*std::max_element(result_table.begin(), result_table.end()) == maxint) { - const int half = number_of_locations / 2; - std::vector to_delete; + auto wrapper = std::make_shared>(result_table, number_of_locations); + auto empty_restriction = RestrictionMap(std::vector()); + auto empty_vector = std::vector(); + auto scc = TarjanSCC>(wrapper, empty_restriction, empty_vector); + scc.run(); + + for (int j = 0; j < scc.get_number_of_components(); ++j){ + components.push_back(std::vector()); + } - for (int i = number_of_locations - 1; i >= 0; --i) { - // if the location is unaccessible by most of the other locations, remember the location - if (std::count(result_table.begin() + i * number_of_locations, result_table.begin() + (i+1) * number_of_locations, maxint) > half) { - to_delete.push_back(i); - } - } - //delete all unaccessible locations - for (int k = 0; k < to_delete.size(); ++k) { - // delete its row - result_table.erase(result_table.begin() + to_delete[k] * number_of_locations, result_table.begin() + (to_delete[k]+1) * number_of_locations); - --number_of_locations; - // delete its column - for (int j = 0; j < number_of_locations; ++j) { - result_table.erase(result_table.begin() + j * number_of_locations + to_delete[k]); - } - // delete its PhantomNode - phantom_node_vector.erase(phantom_node_vector.begin() + to_delete[k]); - } + for (int i = 0; i < number_of_locations; ++i) { + components[scc.get_component_id(i)].push_back(i); } } - void SetJSONOutput (const RouteParameters &route_parameters, - int tsp_time, - InternalRouteResult & min_route, - std::vector & min_loc_permutation, - osrm::json::Object & json_result){ + void SetLocPermutationOutput(const std::vector & loc_permutation, osrm::json::Object & json_result){ osrm::json::Array json_loc_permutation; - json_loc_permutation.values.insert(json_loc_permutation.values.end(), min_loc_permutation.begin(), min_loc_permutation.end()); + json_loc_permutation.values.insert(json_loc_permutation.values.end(), loc_permutation.begin(), loc_permutation.end()); json_result.values["loc_permutation"] = json_loc_permutation; - json_result.values["distance"] = min_route.shortest_path_length; - json_result.values["runtime"] = tsp_time; + } + void SetDistanceOutput(const int distance, osrm::json::Object & json_result) { + json_result.values["distance"] = distance; + } + + void SetRuntimeOutput(const float runtime, osrm::json::Object & json_result) { + json_result.values["runtime"] = runtime; + } + + void SetGeometry(const RouteParameters &route_parameters, const InternalRouteResult & min_route, osrm::json::Object & json_result) { // return geometry result to json std::unique_ptr> descriptor; descriptor = osrm::make_unique>(facade); @@ -145,6 +139,7 @@ template class RoundTripPluginBF final : public BasePlugin descriptor->Run(min_route, json_result); } + int HandleRequest(const RouteParameters &route_parameters, osrm::json::Object &json_result) override final { @@ -163,39 +158,57 @@ template class RoundTripPluginBF final : public BasePlugin return 400; } - SplitUnaccessibleLocations(phantom_node_vector, *result_table); + if (route_parameters.coordinates.size() < 14) { + const auto maxint = std::numeric_limits::max(); + if (*std::max_element(result_table->begin(), result_table->end()) == maxint) { + std::unique_ptr> descriptor; + descriptor = osrm::make_unique>(facade); + descriptor->SetConfig(route_parameters); - InternalRouteResult min_route; - std::vector min_loc_permutation(phantom_node_vector.size(), -1); - //########################### BRUTE FORCE ####################################// - if (route_parameters.coordinates.size() < 11) { + std::vector> components; TIMER_START(tsp); - osrm::tsp::BruteForceTSP(phantom_node_vector, *result_table, min_route, min_loc_permutation); - search_engine_ptr->shortest_path(min_route.segment_end_coordinates, route_parameters.uturns, min_route); + SplitUnaccessibleLocations(phantom_node_vector, *result_table, components); + + std::vector min_loc_permutation(phantom_node_vector.size(), -1); + auto min_dist = 0; + for(auto k = 0; k < components.size(); ++k) { + if (components[k].size() > 1) { + InternalRouteResult min_route; + //run nearest neighbour + osrm::tsp::BruteForceTSP(components[k], phantom_node_vector, *result_table, min_route, min_loc_permutation); + //compute route + search_engine_ptr->shortest_path(min_route.segment_end_coordinates, route_parameters.uturns, min_route); + //return geometry + min_dist += min_route.shortest_path_length; + descriptor->Run(min_route, json_result); + } + } TIMER_STOP(tsp); - BOOST_ASSERT(min_route.segment_end_coordinates.size() == route_parameters.coordinates.size()); - - osrm::json::Array json_loc_permutation; - json_loc_permutation.values.insert(json_loc_permutation.values.end(), min_loc_permutation.begin(), min_loc_permutation.end()); - json_result.values["loc_permutation"] = json_loc_permutation; - json_result.values["distance"] = min_route.shortest_path_length; - SimpleLogger().Write() << "BF GEOM DISTANCE " << min_route.shortest_path_length; - json_result.values["runtime"] = TIMER_MSEC(tsp); + SetRuntimeOutput(TIMER_MSEC(tsp), json_result); + SetDistanceOutput(min_dist, json_result); + SetLocPermutationOutput(min_loc_permutation, json_result); + } else { + auto number_of_locations = phantom_node_vector.size(); + InternalRouteResult min_route; + std::vector min_loc_permutation(number_of_locations, -1); + //########################### BRUTE FORCE ####################################// + TIMER_START(tsp); + osrm::tsp::BruteForceTSP(phantom_node_vector, *result_table, min_route, min_loc_permutation); + search_engine_ptr->shortest_path(min_route.segment_end_coordinates, route_parameters.uturns, min_route); + TIMER_STOP(tsp); + + BOOST_ASSERT(min_route.segment_end_coordinates.size() == route_parameters.coordinates.size()); + + SetLocPermutationOutput(min_loc_permutation, json_result); + SetDistanceOutput(min_route.shortest_path_length, json_result); + SetRuntimeOutput(TIMER_MSEC(tsp), json_result); + SetGeometry(route_parameters, min_route, json_result); + } } else { - json_result.values["distance"] = -1; - json_result.values["runtime"] = -1; + SetRuntimeOutput(-1, json_result); + SetDistanceOutput(-1, json_result); } - // return geometry result to json - - std::unique_ptr> descriptor; - descriptor = osrm::make_unique>(facade); - - descriptor->SetConfig(route_parameters); - descriptor->Run(min_route, json_result); - - - return 200; } diff --git a/plugins/round_trip_FI.hpp b/plugins/round_trip_FI.hpp index b06220055c0..c52075bd366 100644 --- a/plugins/round_trip_FI.hpp +++ b/plugins/round_trip_FI.hpp @@ -95,47 +95,41 @@ template class RoundTripPluginFI final : public BasePlugin } } - void SplitUnaccessibleLocations(PhantomNodeArray & phantom_node_vector, std::vector & result_table) { + void SplitUnaccessibleLocations(PhantomNodeArray & phantom_node_vector, + std::vector & result_table, + std::vector> & components) { + // Run TarjanSCC auto number_of_locations = phantom_node_vector.size(); - const auto maxint = std::numeric_limits::max(); - - //////////////////////////////////// DELETE UNACCESSIBLE LOCATIONS ///////////////////////////////////////// - if (*std::max_element(result_table.begin(), result_table.end()) == maxint) { - const int half = number_of_locations / 2; - std::vector to_delete; + auto wrapper = std::make_shared>(result_table, number_of_locations); + auto empty_restriction = RestrictionMap(std::vector()); + auto empty_vector = std::vector(); + auto scc = TarjanSCC>(wrapper, empty_restriction, empty_vector); + scc.run(); + + for (int j = 0; j < scc.get_number_of_components(); ++j){ + components.push_back(std::vector()); + } - for (int i = number_of_locations - 1; i >= 0; --i) { - // if the location is unaccessible by most of the other locations, remember the location - if (std::count(result_table.begin() + i * number_of_locations, result_table.begin() + (i+1) * number_of_locations, maxint) > half) { - to_delete.push_back(i); - } - } - //delete all unaccessible locations - for (int k = 0; k < to_delete.size(); ++k) { - // delete its row - result_table.erase(result_table.begin() + to_delete[k] * number_of_locations, result_table.begin() + (to_delete[k]+1) * number_of_locations); - --number_of_locations; - // delete its column - for (int j = 0; j < number_of_locations; ++j) { - result_table.erase(result_table.begin() + j * number_of_locations + to_delete[k]); - } - // delete its PhantomNode - phantom_node_vector.erase(phantom_node_vector.begin() + to_delete[k]); - } + for (int i = 0; i < number_of_locations; ++i) { + components[scc.get_component_id(i)].push_back(i); } } - void SetJSONOutput (const RouteParameters &route_parameters, - int tsp_time, - InternalRouteResult & min_route, - std::vector & min_loc_permutation, - osrm::json::Object & json_result){ + void SetLocPermutationOutput(const std::vector & loc_permutation, osrm::json::Object & json_result){ osrm::json::Array json_loc_permutation; - json_loc_permutation.values.insert(json_loc_permutation.values.end(), min_loc_permutation.begin(), min_loc_permutation.end()); + json_loc_permutation.values.insert(json_loc_permutation.values.end(), loc_permutation.begin(), loc_permutation.end()); json_result.values["loc_permutation"] = json_loc_permutation; - json_result.values["distance"] = min_route.shortest_path_length; - json_result.values["runtime"] = tsp_time; + } + + void SetDistanceOutput(const int distance, osrm::json::Object & json_result) { + json_result.values["distance"] = distance; + } + void SetRuntimeOutput(const float runtime, osrm::json::Object & json_result) { + json_result.values["runtime"] = runtime; + } + + void SetGeometry(const RouteParameters &route_parameters, const InternalRouteResult & min_route, osrm::json::Object & json_result) { // return geometry result to json std::unique_ptr> descriptor; descriptor = osrm::make_unique>(facade); @@ -162,19 +156,48 @@ template class RoundTripPluginFI final : public BasePlugin return 400; } - SplitUnaccessibleLocations(phantom_node_vector, *result_table); - - InternalRouteResult min_route; - std::vector min_loc_permutation(phantom_node_vector.size(), -1); - //######################## FARTHEST INSERTION ###############################// - TIMER_START(tsp); - osrm::tsp::FarthestInsertionTSP(phantom_node_vector, *result_table, min_route, min_loc_permutation); - search_engine_ptr->shortest_path(min_route.segment_end_coordinates, route_parameters.uturns, min_route); - TIMER_STOP(tsp); - - BOOST_ASSERT(min_route.segment_end_coordinates.size() == route_parameters.coordinates.size()); - - SetJSONOutput(route_parameters, TIMER_MSEC(tsp), min_route, min_loc_permutation, json_result); + const auto maxint = std::numeric_limits::max(); + if (*std::max_element(result_table->begin(), result_table->end()) == maxint) { + std::unique_ptr> descriptor; + descriptor = osrm::make_unique>(facade); + descriptor->SetConfig(route_parameters); + + std::vector> components; + TIMER_START(tsp); + SplitUnaccessibleLocations(phantom_node_vector, *result_table, components); + auto number_of_locations = phantom_node_vector.size(); + std::vector min_loc_permutation(number_of_locations, -1); + auto min_dist = 0; + for(auto k = 0; k < components.size(); ++k) { + if (components[k].size() > 1) { + InternalRouteResult min_route; + osrm::tsp::FarthestInsertionTSP(components[k], phantom_node_vector, *result_table, min_route, min_loc_permutation); + search_engine_ptr->shortest_path(min_route.segment_end_coordinates, route_parameters.uturns, min_route); + min_dist += min_route.shortest_path_length; + descriptor->Run(min_route, json_result); + } + } + TIMER_STOP(tsp); + + SetRuntimeOutput(TIMER_MSEC(tsp), json_result); + SetDistanceOutput(min_dist, json_result); + SetLocPermutationOutput(min_loc_permutation, json_result); + } else { + auto number_of_locations = phantom_node_vector.size(); + InternalRouteResult min_route; + std::vector min_loc_permutation(number_of_locations, -1); + //######################## FARTHEST INSERTION ###############################// + TIMER_START(tsp); + osrm::tsp::FarthestInsertionTSP(phantom_node_vector, *result_table, min_route, min_loc_permutation); + search_engine_ptr->shortest_path(min_route.segment_end_coordinates, route_parameters.uturns, min_route); + TIMER_STOP(tsp); + + BOOST_ASSERT(min_route.segment_end_coordinates.size() == route_parameters.coordinates.size()); + SetLocPermutationOutput(min_loc_permutation, json_result); + SetDistanceOutput(min_route.shortest_path_length, json_result); + SetRuntimeOutput(TIMER_MSEC(tsp), json_result); + SetGeometry(route_parameters, min_route, json_result); + } return 200; } diff --git a/plugins/round_trip_NN.hpp b/plugins/round_trip_NN.hpp index 620d0fabcc8..cc88d899bf9 100644 --- a/plugins/round_trip_NN.hpp +++ b/plugins/round_trip_NN.hpp @@ -96,47 +96,41 @@ template class RoundTripPluginNN final : public BasePlugin } } - void SplitUnaccessibleLocations(PhantomNodeArray & phantom_node_vector, std::vector & result_table) { + void SplitUnaccessibleLocations(PhantomNodeArray & phantom_node_vector, + std::vector & result_table, + std::vector> & components) { + // Run TarjanSCC auto number_of_locations = phantom_node_vector.size(); - const auto maxint = std::numeric_limits::max(); - - //////////////////////////////////// DELETE UNACCESSIBLE LOCATIONS ///////////////////////////////////////// - if (*std::max_element(result_table.begin(), result_table.end()) == maxint) { - const int half = number_of_locations / 2; - std::vector to_delete; + auto wrapper = std::make_shared>(result_table, number_of_locations); + auto empty_restriction = RestrictionMap(std::vector()); + auto empty_vector = std::vector(); + auto scc = TarjanSCC>(wrapper, empty_restriction, empty_vector); + scc.run(); + + for (int j = 0; j < scc.get_number_of_components(); ++j){ + components.push_back(std::vector()); + } - for (int i = number_of_locations - 1; i >= 0; --i) { - // if the location is unaccessible by most of the other locations, remember the location - if (std::count(result_table.begin() + i * number_of_locations, result_table.begin() + (i+1) * number_of_locations, maxint) > half) { - to_delete.push_back(i); - } - } - //delete all unaccessible locations - for (int k = 0; k < to_delete.size(); ++k) { - // delete its row - result_table.erase(result_table.begin() + to_delete[k] * number_of_locations, result_table.begin() + (to_delete[k]+1) * number_of_locations); - --number_of_locations; - // delete its column - for (int j = 0; j < number_of_locations; ++j) { - result_table.erase(result_table.begin() + j * number_of_locations + to_delete[k]); - } - // delete its PhantomNode - phantom_node_vector.erase(phantom_node_vector.begin() + to_delete[k]); - } + for (int i = 0; i < number_of_locations; ++i) { + components[scc.get_component_id(i)].push_back(i); } } - void SetJSONOutput (const RouteParameters &route_parameters, - int tsp_time, - InternalRouteResult & min_route, - std::vector & min_loc_permutation, - osrm::json::Object & json_result){ + void SetLocPermutationOutput(const std::vector & loc_permutation, osrm::json::Object & json_result){ osrm::json::Array json_loc_permutation; - json_loc_permutation.values.insert(json_loc_permutation.values.end(), min_loc_permutation.begin(), min_loc_permutation.end()); + json_loc_permutation.values.insert(json_loc_permutation.values.end(), loc_permutation.begin(), loc_permutation.end()); json_result.values["loc_permutation"] = json_loc_permutation; - json_result.values["distance"] = min_route.shortest_path_length; - json_result.values["runtime"] = tsp_time; + } + + void SetDistanceOutput(const int distance, osrm::json::Object & json_result) { + json_result.values["distance"] = distance; + } + + void SetRuntimeOutput(const float runtime, osrm::json::Object & json_result) { + json_result.values["runtime"] = runtime; + } + void SetGeometry(const RouteParameters &route_parameters, const InternalRouteResult & min_route, osrm::json::Object & json_result) { // return geometry result to json std::unique_ptr> descriptor; descriptor = osrm::make_unique>(facade); @@ -163,19 +157,51 @@ template class RoundTripPluginNN final : public BasePlugin return 400; } - SplitUnaccessibleLocations(phantom_node_vector, *result_table); - - InternalRouteResult min_route; - std::vector min_loc_permutation(phantom_node_vector.size(), -1); - //######################### NEAREST NEIGHBOUR ###############################// - TIMER_START(tsp); - osrm::tsp::NearestNeighbourTSP(phantom_node_vector, *result_table, min_route, min_loc_permutation); - search_engine_ptr->shortest_path(min_route.segment_end_coordinates, route_parameters.uturns, min_route); - TIMER_STOP(tsp); - - BOOST_ASSERT(min_route.segment_end_coordinates.size() == route_parameters.coordinates.size()); - SetJSONOutput(route_parameters, TIMER_MSEC(tsp), min_route, min_loc_permutation, json_result); + const auto maxint = std::numeric_limits::max(); + if (*std::max_element(result_table->begin(), result_table->end()) == maxint) { + std::unique_ptr> descriptor; + descriptor = osrm::make_unique>(facade); + descriptor->SetConfig(route_parameters); + + std::vector> components; + TIMER_START(tsp); + SplitUnaccessibleLocations(phantom_node_vector, *result_table, components); + + std::vector min_loc_permutation(phantom_node_vector.size(), -1); + auto min_dist = 0; + for(auto k = 0; k < components.size(); ++k) { + if (components[k].size() > 1) { + InternalRouteResult min_route; + //run nearest neighbour + osrm::tsp::NearestNeighbourTSP(components[k], phantom_node_vector, *result_table, min_route, min_loc_permutation); + //compute route + search_engine_ptr->shortest_path(min_route.segment_end_coordinates, route_parameters.uturns, min_route); + //return geometry + min_dist += min_route.shortest_path_length; + descriptor->Run(min_route, json_result); + } + } + TIMER_STOP(tsp); + + SetRuntimeOutput(TIMER_MSEC(tsp), json_result); + SetDistanceOutput(min_dist, json_result); + SetLocPermutationOutput(min_loc_permutation, json_result); + } else { + auto number_of_locations = phantom_node_vector.size(); + InternalRouteResult min_route; + std::vector min_loc_permutation(number_of_locations, -1); + //######################### NEAREST NEIGHBOUR ###############################// + TIMER_START(tsp); + osrm::tsp::NearestNeighbourTSP(phantom_node_vector, *result_table, min_route, min_loc_permutation); + search_engine_ptr->shortest_path(min_route.segment_end_coordinates, route_parameters.uturns, min_route); + TIMER_STOP(tsp); + BOOST_ASSERT(min_route.segment_end_coordinates.size() == route_parameters.coordinates.size()); + SetLocPermutationOutput(min_loc_permutation, json_result); + SetDistanceOutput(min_route.shortest_path_length, json_result); + SetRuntimeOutput(TIMER_MSEC(tsp), json_result); + SetGeometry(route_parameters, min_route, json_result); + } return 200; } diff --git a/routing_algorithms/tsp_brute_force.hpp b/routing_algorithms/tsp_brute_force.hpp index a414659c41c..9e0c266f2fc 100644 --- a/routing_algorithms/tsp_brute_force.hpp +++ b/routing_algorithms/tsp_brute_force.hpp @@ -51,19 +51,19 @@ namespace osrm namespace tsp { - -int ReturnDistance(const std::vector & dist_table, const std::vector location_order, const int min_route_dist, const int number_of_locations) { +template +int ReturnDistance(const std::vector & dist_table, const std::vector & location_order, const int min_route_dist, const int number_of_locations, const int component_size) { int i = 0; int route_dist = 0; // compute length and stop if length is longer than route already found - while (i < number_of_locations - 1 && route_dist < min_route_dist) { + while (i < component_size - 1 && route_dist < min_route_dist) { //get distance from location i to location i+1 route_dist += *(dist_table.begin() + (location_order[i] * number_of_locations) + location_order[i+1]); ++i; } //get distance from last location to first location - route_dist += *(dist_table.begin() + (location_order[number_of_locations-1] * number_of_locations) + location_order[0]); + route_dist += *(dist_table.begin() + (location_order[component_size-1] * number_of_locations) + location_order[0]); if (route_dist < min_route_dist) { return route_dist; @@ -73,6 +73,39 @@ int ReturnDistance(const std::vector & dist_table, const std::vector } } +void BruteForceTSP(std::vector & location, + const PhantomNodeArray & phantom_node_vector, + const std::vector & dist_table, + InternalRouteResult & min_route, + std::vector & min_loc_permutation) { + + const auto number_of_location = phantom_node_vector.size(); + const int component_size = location.size(); + int min_route_dist = std::numeric_limits::max(); + + std::vector min_location; + + // check length of all possible permutation of the location ids + do { + // int new_distance = ReturnDistance(dist_table, location, min_route_dist, number_of_location, component_size); + int new_distance = 4; + if (new_distance != -1) { + min_route_dist = new_distance; + min_location = location; + } + } while(std::next_permutation(location.begin(), location.end())); + + PhantomNodes viapoint; + for (int i = 0; i < component_size - 1; ++i) { + viapoint = PhantomNodes{phantom_node_vector[min_location[i]][0], phantom_node_vector[min_location[i + 1]][0]}; + min_route.segment_end_coordinates.emplace_back(viapoint); + min_loc_permutation[min_location[i]] = i; + } + min_loc_permutation[min_location[component_size - 1]] = component_size - 1; + viapoint = PhantomNodes{phantom_node_vector[min_location[component_size - 1]][0], phantom_node_vector[min_location[0]][0]}; + min_route.segment_end_coordinates.emplace_back(viapoint); +} + void BruteForceTSP(const PhantomNodeArray & phantom_node_vector, const std::vector & dist_table, InternalRouteResult & min_route, @@ -87,7 +120,7 @@ void BruteForceTSP(const PhantomNodeArray & phantom_node_vector, // check length of all possible permutation of the location ids do { - int new_distance = ReturnDistance(dist_table, location_ids, min_route_dist, number_of_locations); + int new_distance = ReturnDistance(dist_table, location_ids, min_route_dist, number_of_locations, number_of_locations); if (new_distance != -1) { min_route_dist = new_distance; //TODO: this gets copied right? fix this diff --git a/routing_algorithms/tsp_farthest_insertion.hpp b/routing_algorithms/tsp_farthest_insertion.hpp index 05ef5e612cd..cb3353d9e4b 100644 --- a/routing_algorithms/tsp_farthest_insertion.hpp +++ b/routing_algorithms/tsp_farthest_insertion.hpp @@ -31,6 +31,7 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "../data_structures/search_engine.hpp" #include "../util/string_util.hpp" +#include "../tools/tsp_logs.hpp" #include @@ -41,12 +42,136 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include #include +#include namespace osrm { namespace tsp { +void GetLongestRoundTrip(const int current_loc, + std::list & current_trip, + const std::vector & dist_table, + const int number_of_locations, + int & longest_min_tour, + std::list::iterator & following_loc){ + // for all nodes in the current trip find the best insertion resulting in the shortest path + for (auto from_node = current_trip.begin(); from_node != std::prev(current_trip.end()); ++from_node) { + auto to_node = std::next(from_node); + + auto dist_from = *(dist_table.begin() + (*from_node * number_of_locations) + current_loc); + auto dist_to = *(dist_table.begin() + (current_loc * number_of_locations) + *to_node); + auto trip_dist = dist_from + dist_to - *(dist_table.begin() + (*from_node * number_of_locations) + *to_node); + + // from all possible insertions to the current trip, choose the longest of all minimal insertions + if (trip_dist < longest_min_tour) { + longest_min_tour = trip_dist; + following_loc = to_node; + } + } + { // check insertion between last and first location too + auto from_node = std::prev(current_trip.end()); + auto to_node = current_trip.begin(); + + auto dist_from = *(dist_table.begin() + (*from_node * number_of_locations) + current_loc); + auto dist_to = *(dist_table.begin() + (current_loc * number_of_locations) + *to_node); + auto trip_dist = dist_from + dist_to - *(dist_table.begin() + (*from_node * number_of_locations) + *to_node); + if (trip_dist < longest_min_tour) { + longest_min_tour = trip_dist; + following_loc = to_node; + } + } +} + +void ComputeRouteAndPermutation(const PhantomNodeArray & phantom_node_vector, + std::list & current_trip, + InternalRouteResult & min_route, + std::vector & min_loc_permutation) { + // given he final trip, compute total distance and return the route and location permutation + PhantomNodes viapoint; + int perm = 0; + for (auto it = current_trip.begin(); it != std::prev(current_trip.end()); ++it) { + auto from_node = *it; + auto to_node = *std::next(it); + viapoint = PhantomNodes{phantom_node_vector[from_node][0], phantom_node_vector[to_node][0]}; + min_route.segment_end_coordinates.emplace_back(viapoint); + min_loc_permutation[from_node] = perm; + ++perm; + } + // check dist between last and first location too + viapoint = PhantomNodes{phantom_node_vector[*std::prev(current_trip.end())][0], phantom_node_vector[current_trip.front()][0]}; + min_route.segment_end_coordinates.emplace_back(viapoint); + min_loc_permutation[*std::prev(current_trip.end())] = perm; +} + +void FarthestInsertionTSP(const std::vector & locations, + const PhantomNodeArray & phantom_node_vector, + const std::vector & dist_table, + InternalRouteResult & min_route, + std::vector & min_loc_permutation) { + ////////////////////////////////////////////////////////////////////////////////////////////////// + // START FARTHEST INSERTION HERE + // 1. start at a random round trip of 2 locations + // 2. find the location that is the farthest away from the visited locations and whose insertion will make the round trip the longest + // 3. add the found location to the current round trip such that round trip is the shortest + // 4. repeat 2-3 until all locations are visited + // 5. DONE! + ////////////////////////////////////////////////////////////////////////////////////////////////// + const int number_of_locations = phantom_node_vector.size(); + const int size_of_component = locations.size(); + // list of the trip that will be found incrementally + std::list current_trip; + // tracks which nodes have been already visited + std::vector visited(number_of_locations, false); + + auto max_dist = 0; + auto index = -1; + for (auto x : locations) { + for (auto y : locations) { + if (*(dist_table.begin() + x * number_of_locations + y) > max_dist) { + max_dist = *(dist_table.begin() + x * number_of_locations + y); + index = x * number_of_locations + y; + } + } + } + const int max_from = index / number_of_locations; + const int max_to = index % number_of_locations; + + visited[max_from] = true; + visited[max_to] = true; + current_trip.push_back(max_from); + current_trip.push_back(max_to); + + // add all other nodes missing (two nodes are already in the initial start trip) + for (int j = 2; j < size_of_component; ++j) { + auto shortest_max_tour = -1; + int next_node = -1; + std::list::iterator min_max_insert; + + // find unvisited loc i that is the farthest away from all other visited locs + for (auto i : locations) { + if (!visited[i]) { + // longest_min_tour is the distance of the longest of all insertions with the minimal distance + auto longest_min_tour = std::numeric_limits::max(); + // following_loc is the location that comes after the location that is to be inserted + std::list::iterator following_loc; + GetLongestRoundTrip(i, current_trip, dist_table, number_of_locations, longest_min_tour, following_loc); + + // add the location to the current trip such that it results in the shortest total tour + if (longest_min_tour > shortest_max_tour) { + shortest_max_tour = longest_min_tour; + next_node = i; + min_max_insert = following_loc; + } + } + } + // mark as visited and insert node + visited[next_node] = true; + current_trip.insert(min_max_insert, next_node); + } + ComputeRouteAndPermutation(phantom_node_vector, current_trip, min_route, min_loc_permutation); +} + void FarthestInsertionTSP(const PhantomNodeArray & phantom_node_vector, const std::vector & dist_table, InternalRouteResult & min_route, @@ -66,14 +191,11 @@ void FarthestInsertionTSP(const PhantomNodeArray & phantom_node_vector, // tracks which nodes have been already visited std::vector visited(number_of_locations, false); - // PrintDistTable(dist_table, number_of_locations); - // find the pair of location with the biggest distance and make the pair the initial start trip const auto index = std::distance(dist_table.begin(), std::max_element(dist_table.begin(), dist_table.end())); const int max_from = index / number_of_locations; const int max_to = index % number_of_locations; - visited[max_from] = true; visited[max_to] = true; current_trip.push_back(max_from); @@ -93,32 +215,7 @@ void FarthestInsertionTSP(const PhantomNodeArray & phantom_node_vector, // following_loc is the location that comes after the location that is to be inserted std::list::iterator following_loc; - // for all nodes in the current trip find the best insertion resulting in the shortest path - for (auto from_node = current_trip.begin(); from_node != std::prev(current_trip.end()); ++from_node) { - auto to_node = std::next(from_node); - - auto dist_from = *(dist_table.begin() + (*from_node * number_of_locations) + i); - auto dist_to = *(dist_table.begin() + (i * number_of_locations) + *to_node); - auto trip_dist = dist_from + dist_to - *(dist_table.begin() + (*from_node * number_of_locations) + *to_node); - - // from all possible insertions to the current trip, choose the longest of all minimal insertions - if (trip_dist < longest_min_tour) { - longest_min_tour = trip_dist; - following_loc = to_node; - } - } - { // check insertion between last and first location too - auto from_node = std::prev(current_trip.end()); - auto to_node = current_trip.begin(); - - auto dist_from = *(dist_table.begin() + (*from_node * number_of_locations) + i); - auto dist_to = *(dist_table.begin() + (i * number_of_locations) + *to_node); - auto trip_dist = dist_from + dist_to - *(dist_table.begin() + (*from_node * number_of_locations) + *to_node); - if (trip_dist < longest_min_tour) { - longest_min_tour = trip_dist; - following_loc = to_node; - } - } + GetLongestRoundTrip(i, current_trip, dist_table, number_of_locations, longest_min_tour, following_loc); // add the location to the current trip such that it results in the shortest total tour if (longest_min_tour > shortest_max_tour) { @@ -133,24 +230,7 @@ void FarthestInsertionTSP(const PhantomNodeArray & phantom_node_vector, current_trip.insert(min_max_insert, next_node); } - // given he final trip, compute total distance and return the route and location permutation - PhantomNodes viapoint; - int perm = 0; - for (auto it = current_trip.begin(); it != std::prev(current_trip.end()); ++it) { - auto from_node = *it; - auto to_node = *std::next(it); - - viapoint = PhantomNodes{phantom_node_vector[from_node][0], phantom_node_vector[to_node][0]}; - min_route.segment_end_coordinates.emplace_back(viapoint); - - min_loc_permutation[from_node] = perm; - ++perm; - } - { // check dist between last and first location too - viapoint = PhantomNodes{phantom_node_vector[*std::prev(current_trip.end())][0], phantom_node_vector[current_trip.front()][0]}; - min_route.segment_end_coordinates.emplace_back(viapoint); - min_loc_permutation[*std::prev(current_trip.end())] = perm; - } + ComputeRouteAndPermutation(phantom_node_vector, current_trip, min_route, min_loc_permutation); } diff --git a/routing_algorithms/tsp_nearest_neighbour.hpp b/routing_algorithms/tsp_nearest_neighbour.hpp index bf949229301..6a5d059948c 100644 --- a/routing_algorithms/tsp_nearest_neighbour.hpp +++ b/routing_algorithms/tsp_nearest_neighbour.hpp @@ -48,6 +48,128 @@ namespace osrm namespace tsp { +void NearestNeighbourTSP(const std::vector & locations, + const PhantomNodeArray & phantom_node_vector, + const std::vector & dist_table, + InternalRouteResult & min_route, + std::vector & min_loc_permutation) { + ////////////////////////////////////////////////////////////////////////////////////////////////// + // START GREEDY NEAREST NEIGHBOUR HERE + // 1. grab a random location and mark as starting point + // 2. find the nearest unvisited neighbour, set it as the current location and mark as visited + // 3. repeat 2 until there is no unvisited location + // 4. return route back to starting point + // 5. compute route + // 6. repeat 1-5 with different starting points and choose iteration with shortest trip + // 7. DONE! + ////////////////////////////////////////////////////////////////////////////////////////////////// + + const auto number_of_locations = phantom_node_vector.size(); + const int size_of_component = locations.size(); + min_route.shortest_path_length = std::numeric_limits::max(); + + // ALWAYS START AT ANOTHER STARTING POINT + for(auto start_node : locations) + { + int curr_node = start_node; + InternalRouteResult raw_route; + //TODO: Should we always use the same vector or does it not matter at all because of loop scope? + std::vector loc_permutation(number_of_locations, -1); + loc_permutation[start_node] = 0; + // visited[i] indicates whether node i was already visited by the salesman + std::vector visited(number_of_locations, false); + visited[start_node] = true; + + PhantomNodes viapoint; + // 3. REPEAT FOR EVERY UNVISITED NODE + int trip_dist = 0; + for(int via_point = 1; via_point < size_of_component; ++via_point) + { + int min_dist = std::numeric_limits::max(); + int min_id = -1; + + // 2. FIND NEAREST NEIGHBOUR + for (auto next : locations) { + if(!visited[next] && + *(dist_table.begin() + curr_node * number_of_locations + next) < min_dist) { + min_dist = *(dist_table.begin() + curr_node * number_of_locations + next); + min_id = next; + } + } + loc_permutation[min_id] = via_point; + visited[min_id] = true; + viapoint = PhantomNodes{phantom_node_vector[curr_node][0], phantom_node_vector[min_id][0]}; + raw_route.segment_end_coordinates.emplace_back(viapoint); + trip_dist += min_dist; + curr_node = min_id; + } + + // 4. ROUTE BACK TO STARTING POINT + viapoint = PhantomNodes{raw_route.segment_end_coordinates.back().target_phantom, phantom_node_vector[start_node][0]}; + raw_route.segment_end_coordinates.emplace_back(viapoint); + + // check round trip with this starting point is shorter than the shortest round trip found till now + if (trip_dist < min_route.shortest_path_length) { + min_route = raw_route; + min_route.shortest_path_length = trip_dist; + //TODO: this gets copied right? fix this + min_loc_permutation = loc_permutation; + } + } + + // // ALWAYS START AT ANOTHER STARTING POINT + // for(auto start_node : locations) { + // SimpleLogger().Write() << "STARTING AT " << start_node; + // int curr_node = start_node; + // InternalRouteResult raw_route; + // //TODO: Should we always use the same vector or does it not matter at all because of loop scope? + // std::vector loc_permutation(number_of_locations, -1); + + // // visited[i] indicates whether node i was already visited by the salesman + // std::vector visited(number_of_locations, false); + // visited[start_node] = true; + // loc_permutation[start_node] = 0; + + // PhantomNodes viapoint; + // // 3. REPEAT FOR EVERY UNVISITED NODE + // int trip_dist = 0; + // for(int via_point = 1; via_point < size_of_component; ++via_point) + // { + // int min_dist = std::numeric_limits::max(); + // int min_id = -1; + + // // 2. FIND NEAREST NEIGHBOUR + // for (auto next : locations) { + // if(!visited[next] && + // *(dist_table.begin() + curr_node * number_of_locations + next) < min_dist) { + // min_dist = *(dist_table.begin() + curr_node * number_of_locations + next); + // min_id = next; + // } + // } + + // loc_permutation[min_id] = via_point; + // visited[min_id] = true; + // SimpleLogger().Write() << "MOVING TO " << min_id; + // viapoint = PhantomNodes{phantom_node_vector[curr_node][0], phantom_node_vector[min_id][0]}; + // raw_route.segment_end_coordinates.emplace_back(viapoint); + // trip_dist += min_dist; + // curr_node = min_id; + // } + + // // 4. ROUTE BACK TO STARTING POINT + // viapoint = PhantomNodes{raw_route.segment_end_coordinates.back().target_phantom, phantom_node_vector[start_node][0]}; + // raw_route.segment_end_coordinates.emplace_back(viapoint); + + // // check round trip with this starting point is shorter than the shortest round trip found till now + // if (trip_dist < min_route.shortest_path_length) { + // min_route = raw_route; + // min_route.shortest_path_length = trip_dist; + // //TODO: this gets copied right? fix this + // min_loc_permutation = loc_permutation; + // } + // } +} + void NearestNeighbourTSP(const PhantomNodeArray & phantom_node_vector, const std::vector & dist_table, InternalRouteResult & min_route, From 6191b6bee230ef345a216353318d77bf64c0ab3b Mon Sep 17 00:00:00 2001 From: Chau Nguyen Date: Fri, 10 Jul 2015 12:25:35 -0400 Subject: [PATCH 072/122] add parameter to choose algorithm for tsp calculation and remove redundant code --- data_structures/route_parameters.cpp | 5 + include/osrm/route_parameters.hpp | 3 + library/osrm_impl.cpp | 6 - plugins/round_trip.hpp | 37 +++-- plugins/round_trip_BF.hpp | 217 ------------------------- plugins/round_trip_FI.hpp | 207 ----------------------- plugins/round_trip_NN.hpp | 211 ------------------------ routing_algorithms/tsp_brute_force.hpp | 1 - server/api_grammar.hpp | 7 +- 9 files changed, 39 insertions(+), 655 deletions(-) delete mode 100644 plugins/round_trip_BF.hpp delete mode 100644 plugins/round_trip_FI.hpp delete mode 100644 plugins/round_trip_NN.hpp diff --git a/data_structures/route_parameters.cpp b/data_structures/route_parameters.cpp index 86b9553580c..31e91c1fa73 100644 --- a/data_structures/route_parameters.cpp +++ b/data_structures/route_parameters.cpp @@ -122,6 +122,11 @@ void RouteParameters::setLanguage(const std::string &language_string) language = language_string; } +void RouteParameters::setTSPAlgo(const std::string &tsp_algo_string) +{ + tsp_algo = tsp_algo_string; +} + void RouteParameters::setGeometryFlag(const bool flag) { geometry = flag; } void RouteParameters::setCompressionFlag(const bool flag) { compression = flag; } diff --git a/include/osrm/route_parameters.hpp b/include/osrm/route_parameters.hpp index bc413d73bd0..94ed3a33600 100644 --- a/include/osrm/route_parameters.hpp +++ b/include/osrm/route_parameters.hpp @@ -81,6 +81,8 @@ struct RouteParameters void getCoordinatesFromGeometry(const std::string geometry_string); + void setTSPAlgo(const std::string &tsp_algo); + short zoom_level; bool print_instructions; bool alternate_route; @@ -97,6 +99,7 @@ struct RouteParameters std::string output_format; std::string jsonp_parameter; std::string language; + std::string tsp_algo; std::vector hints; std::vector timestamps; std::vector uturns; diff --git a/library/osrm_impl.cpp b/library/osrm_impl.cpp index 26ba83e53a8..8057069eb88 100644 --- a/library/osrm_impl.cpp +++ b/library/osrm_impl.cpp @@ -42,9 +42,6 @@ class named_mutex; #include "../plugins/nearest.hpp" #include "../plugins/timestamp.hpp" #include "../plugins/round_trip.hpp" -#include "../plugins/round_trip_NN.hpp" -#include "../plugins/round_trip_BF.hpp" -#include "../plugins/round_trip_FI.hpp" #include "../plugins/viaroute.hpp" #include "../plugins/match.hpp" #include "../server/data_structures/datafacade_base.hpp" @@ -91,9 +88,6 @@ OSRM_impl::OSRM_impl(libosrm_config &lib_config) RegisterPlugin(new TimestampPlugin>(query_data_facade)); RegisterPlugin(new ViaRoutePlugin>(query_data_facade)); RegisterPlugin(new RoundTripPlugin>(query_data_facade)); - RegisterPlugin(new RoundTripPluginNN>(query_data_facade)); - RegisterPlugin(new RoundTripPluginBF>(query_data_facade)); - RegisterPlugin(new RoundTripPluginFI>(query_data_facade)); } OSRM_impl::~OSRM_impl() diff --git a/plugins/round_trip.hpp b/plugins/round_trip.hpp index bdb42a6ef88..03840824aa4 100644 --- a/plugins/round_trip.hpp +++ b/plugins/round_trip.hpp @@ -158,9 +158,10 @@ template class RoundTripPlugin final : public BasePlugin return 400; } - + //check if locations are in different strongly connected components (SCC) const auto maxint = std::numeric_limits::max(); if (*std::max_element(result_table->begin(), result_table->end()) == maxint) { + //run TSP computation for every SCC std::unique_ptr> descriptor; descriptor = osrm::make_unique>(facade); descriptor->SetConfig(route_parameters); @@ -170,35 +171,49 @@ template class RoundTripPlugin final : public BasePlugin SplitUnaccessibleLocations(phantom_node_vector, *result_table, components); auto number_of_locations = phantom_node_vector.size(); std::vector min_loc_permutation(number_of_locations, -1); + auto min_dist = 0; for(auto k = 0; k < components.size(); ++k) { if (components[k].size() > 1) { + // Compute the TSP with the given algorithm InternalRouteResult min_route; - osrm::tsp::FarthestInsertionTSP(components[k], phantom_node_vector, *result_table, min_route, min_loc_permutation); + if (route_parameters.tsp_algo == "BF" && route_parameters.coordinates.size() < 14) { + osrm::tsp::BruteForceTSP(components[k], phantom_node_vector, *result_table, min_route, min_loc_permutation); + } else if (route_parameters.tsp_algo == "NN") { + osrm::tsp::NearestNeighbourTSP(components[k], phantom_node_vector, *result_table, min_route, min_loc_permutation); + } else if (route_parameters.tsp_algo == "FI") { + osrm::tsp::FarthestInsertionTSP(components[k], phantom_node_vector, *result_table, min_route, min_loc_permutation); + } else{ + osrm::tsp::FarthestInsertionTSP(components[k], phantom_node_vector, *result_table, min_route, min_loc_permutation); + } search_engine_ptr->shortest_path(min_route.segment_end_coordinates, route_parameters.uturns, min_route); min_dist += min_route.shortest_path_length; descriptor->Run(min_route, json_result); } } - TIMER_STOP(tsp); + TIMER_STOP(tsp); SetRuntimeOutput(TIMER_MSEC(tsp), json_result); SetDistanceOutput(min_dist, json_result); SetLocPermutationOutput(min_loc_permutation, json_result); - } else { + } else { //run TSP computation for all locations auto number_of_locations = phantom_node_vector.size(); InternalRouteResult min_route; std::vector min_loc_permutation(number_of_locations, -1); - //######################## FARTHEST INSERTION ###############################// + + // Compute the TSP with the given algorithm TIMER_START(tsp); - osrm::tsp::FarthestInsertionTSP(phantom_node_vector, *result_table, min_route, min_loc_permutation); + if (route_parameters.tsp_algo == "BF" && route_parameters.coordinates.size() < 14) { + osrm::tsp::BruteForceTSP(phantom_node_vector, *result_table, min_route, min_loc_permutation); + } else if (route_parameters.tsp_algo == "NN") { + osrm::tsp::NearestNeighbourTSP(phantom_node_vector, *result_table, min_route, min_loc_permutation); + } else if (route_parameters.tsp_algo == "FI") { + osrm::tsp::FarthestInsertionTSP(phantom_node_vector, *result_table, min_route, min_loc_permutation); + } else { + osrm::tsp::FarthestInsertionTSP(phantom_node_vector, *result_table, min_route, min_loc_permutation); + } search_engine_ptr->shortest_path(min_route.segment_end_coordinates, route_parameters.uturns, min_route); TIMER_STOP(tsp); - // //######################### NEAREST NEIGHBOUR ###############################// - // TIMER_START(tsp); - // osrm::tsp::NearestNeighbourTSP(phantom_node_vector, *result_table, min_route, min_loc_permutation); - // search_engine_ptr->shortest_path(min_route.segment_end_coordinates, route_parameters.uturns, min_route); - // TIMER_STOP(tsp); BOOST_ASSERT(min_route.segment_end_coordinates.size() == route_parameters.coordinates.size()); SetLocPermutationOutput(min_loc_permutation, json_result); SetDistanceOutput(min_route.shortest_path_length, json_result); diff --git a/plugins/round_trip_BF.hpp b/plugins/round_trip_BF.hpp deleted file mode 100644 index 10510aa0323..00000000000 --- a/plugins/round_trip_BF.hpp +++ /dev/null @@ -1,217 +0,0 @@ - -/* - -Copyright (c) 2015, Project OSRM contributors -All rights reserved. - -Redistribution and use in source and binary forms, with or without modification, -are permitted provided that the following conditions are met: - -Redistributions of source code must retain the above copyright notice, this list -of conditions and the following disclaimer. -Redistributions in binary form must reproduce the above copyright notice, this -list of conditions and the following disclaimer in the documentation and/or -other materials provided with the distribution. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR -ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES -(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON -ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -*/ - -#ifndef ROUND_TRIP_BF_HPP -#define ROUND_TRIP_BF_HPP - -#include "plugin_base.hpp" - -#include "../algorithms/object_encoder.hpp" -#include "../routing_algorithms/tsp_nearest_neighbour.hpp" -#include "../routing_algorithms/tsp_farthest_insertion.hpp" -#include "../routing_algorithms/tsp_brute_force.hpp" -#include "../data_structures/query_edge.hpp" -#include "../data_structures/search_engine.hpp" -#include "../descriptors/descriptor_base.hpp" -#include "../descriptors/json_descriptor.hpp" -#include "../util/json_renderer.hpp" -#include "../util/make_unique.hpp" -#include "../util/string_util.hpp" -#include "../util/timing_util.hpp" -#include "../util/simple_logger.hpp" - -#include - -#include -#include -#include -#include -#include -#include -#include - -template class RoundTripPluginBF final : public BasePlugin -{ - private: - std::string descriptor_string; - DataFacadeT *facade; - std::unique_ptr> search_engine_ptr; - - public: - explicit RoundTripPluginBF(DataFacadeT *facade) - : descriptor_string("tripBF"), facade(facade) - { - search_engine_ptr = osrm::make_unique>(facade); - } - - const std::string GetDescriptor() const override final { return descriptor_string; } - - void GetPhantomNodes(const RouteParameters &route_parameters, PhantomNodeArray & phantom_node_vector) { - const bool checksum_OK = (route_parameters.check_sum == facade->GetCheckSum()); - - // find phantom nodes for all input coords - for (const auto i : osrm::irange(0, route_parameters.coordinates.size())) { - // if client hints are helpful, encode hints - if (checksum_OK && i < route_parameters.hints.size() && - !route_parameters.hints[i].empty()) { - PhantomNode current_phantom_node; - ObjectEncoder::DecodeFromBase64(route_parameters.hints[i], current_phantom_node); - if (current_phantom_node.is_valid(facade->GetNumberOfNodes())) - { - phantom_node_vector[i].emplace_back(std::move(current_phantom_node)); - continue; - } - } - facade->IncrementalFindPhantomNodeForCoordinate(route_parameters.coordinates[i], - phantom_node_vector[i], 1); - if (phantom_node_vector[i].size() > 1) { - phantom_node_vector[i].erase(phantom_node_vector[i].begin()); - } - BOOST_ASSERT(phantom_node_vector[i].front().is_valid(facade->GetNumberOfNodes())); - } - } - - void SplitUnaccessibleLocations(PhantomNodeArray & phantom_node_vector, - std::vector & result_table, - std::vector> & components) { - // Run TarjanSCC - auto number_of_locations = phantom_node_vector.size(); - auto wrapper = std::make_shared>(result_table, number_of_locations); - auto empty_restriction = RestrictionMap(std::vector()); - auto empty_vector = std::vector(); - auto scc = TarjanSCC>(wrapper, empty_restriction, empty_vector); - scc.run(); - - for (int j = 0; j < scc.get_number_of_components(); ++j){ - components.push_back(std::vector()); - } - - for (int i = 0; i < number_of_locations; ++i) { - components[scc.get_component_id(i)].push_back(i); - } - } - - void SetLocPermutationOutput(const std::vector & loc_permutation, osrm::json::Object & json_result){ - osrm::json::Array json_loc_permutation; - json_loc_permutation.values.insert(json_loc_permutation.values.end(), loc_permutation.begin(), loc_permutation.end()); - json_result.values["loc_permutation"] = json_loc_permutation; - } - - void SetDistanceOutput(const int distance, osrm::json::Object & json_result) { - json_result.values["distance"] = distance; - } - - void SetRuntimeOutput(const float runtime, osrm::json::Object & json_result) { - json_result.values["runtime"] = runtime; - } - - void SetGeometry(const RouteParameters &route_parameters, const InternalRouteResult & min_route, osrm::json::Object & json_result) { - // return geometry result to json - std::unique_ptr> descriptor; - descriptor = osrm::make_unique>(facade); - - descriptor->SetConfig(route_parameters); - descriptor->Run(min_route, json_result); - } - - - int HandleRequest(const RouteParameters &route_parameters, - osrm::json::Object &json_result) override final - { - // check if all inputs are coordinates - if (!check_all_coordinates(route_parameters.coordinates)) { - return 400; - } - - PhantomNodeArray phantom_node_vector(route_parameters.coordinates.size()); - GetPhantomNodes(route_parameters, phantom_node_vector); - - // compute the distance table of all phantom nodes - const std::shared_ptr> result_table = - search_engine_ptr->distance_table(phantom_node_vector); - if (!result_table){ - return 400; - } - - if (route_parameters.coordinates.size() < 14) { - const auto maxint = std::numeric_limits::max(); - if (*std::max_element(result_table->begin(), result_table->end()) == maxint) { - std::unique_ptr> descriptor; - descriptor = osrm::make_unique>(facade); - descriptor->SetConfig(route_parameters); - - std::vector> components; - TIMER_START(tsp); - SplitUnaccessibleLocations(phantom_node_vector, *result_table, components); - - std::vector min_loc_permutation(phantom_node_vector.size(), -1); - auto min_dist = 0; - for(auto k = 0; k < components.size(); ++k) { - if (components[k].size() > 1) { - InternalRouteResult min_route; - //run nearest neighbour - osrm::tsp::BruteForceTSP(components[k], phantom_node_vector, *result_table, min_route, min_loc_permutation); - //compute route - search_engine_ptr->shortest_path(min_route.segment_end_coordinates, route_parameters.uturns, min_route); - //return geometry - min_dist += min_route.shortest_path_length; - descriptor->Run(min_route, json_result); - } - } - TIMER_STOP(tsp); - - SetRuntimeOutput(TIMER_MSEC(tsp), json_result); - SetDistanceOutput(min_dist, json_result); - SetLocPermutationOutput(min_loc_permutation, json_result); - } else { - auto number_of_locations = phantom_node_vector.size(); - InternalRouteResult min_route; - std::vector min_loc_permutation(number_of_locations, -1); - //########################### BRUTE FORCE ####################################// - TIMER_START(tsp); - osrm::tsp::BruteForceTSP(phantom_node_vector, *result_table, min_route, min_loc_permutation); - search_engine_ptr->shortest_path(min_route.segment_end_coordinates, route_parameters.uturns, min_route); - TIMER_STOP(tsp); - - BOOST_ASSERT(min_route.segment_end_coordinates.size() == route_parameters.coordinates.size()); - - SetLocPermutationOutput(min_loc_permutation, json_result); - SetDistanceOutput(min_route.shortest_path_length, json_result); - SetRuntimeOutput(TIMER_MSEC(tsp), json_result); - SetGeometry(route_parameters, min_route, json_result); - } - } else { - SetRuntimeOutput(-1, json_result); - SetDistanceOutput(-1, json_result); - } - return 200; - } - -}; - -#endif // ROUND_TRIP_BF_HPP diff --git a/plugins/round_trip_FI.hpp b/plugins/round_trip_FI.hpp deleted file mode 100644 index c52075bd366..00000000000 --- a/plugins/round_trip_FI.hpp +++ /dev/null @@ -1,207 +0,0 @@ -/* - -Copyright (c) 2015, Project OSRM contributors -All rights reserved. - -Redistribution and use in source and binary forms, with or without modification, -are permitted provided that the following conditions are met: - -Redistributions of source code must retain the above copyright notice, this list -of conditions and the following disclaimer. -Redistributions in binary form must reproduce the above copyright notice, this -list of conditions and the following disclaimer in the documentation and/or -other materials provided with the distribution. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR -ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES -(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON -ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -*/ - -#ifndef ROUND_TRIP_FI_HPP -#define ROUND_TRIP_FI_HPP - -#include "plugin_base.hpp" - -#include "../algorithms/object_encoder.hpp" -#include "../routing_algorithms/tsp_nearest_neighbour.hpp" -#include "../routing_algorithms/tsp_farthest_insertion.hpp" -#include "../routing_algorithms/tsp_brute_force.hpp" -#include "../data_structures/query_edge.hpp" -#include "../data_structures/search_engine.hpp" -#include "../descriptors/descriptor_base.hpp" -#include "../descriptors/json_descriptor.hpp" -#include "../util/json_renderer.hpp" -#include "../util/make_unique.hpp" -#include "../util/string_util.hpp" -#include "../util/timing_util.hpp" -#include "../util/simple_logger.hpp" - -#include - -#include -#include -#include -#include -#include -#include -#include - -template class RoundTripPluginFI final : public BasePlugin -{ - private: - std::string descriptor_string; - DataFacadeT *facade; - std::unique_ptr> search_engine_ptr; - - public: - explicit RoundTripPluginFI(DataFacadeT *facade) - : descriptor_string("tripFI"), facade(facade) - { - search_engine_ptr = osrm::make_unique>(facade); - } - - const std::string GetDescriptor() const override final { return descriptor_string; } - - void GetPhantomNodes(const RouteParameters &route_parameters, PhantomNodeArray & phantom_node_vector) { - const bool checksum_OK = (route_parameters.check_sum == facade->GetCheckSum()); - - // find phantom nodes for all input coords - for (const auto i : osrm::irange(0, route_parameters.coordinates.size())) { - // if client hints are helpful, encode hints - if (checksum_OK && i < route_parameters.hints.size() && - !route_parameters.hints[i].empty()) { - PhantomNode current_phantom_node; - ObjectEncoder::DecodeFromBase64(route_parameters.hints[i], current_phantom_node); - if (current_phantom_node.is_valid(facade->GetNumberOfNodes())) - { - phantom_node_vector[i].emplace_back(std::move(current_phantom_node)); - continue; - } - } - facade->IncrementalFindPhantomNodeForCoordinate(route_parameters.coordinates[i], - phantom_node_vector[i], 1); - if (phantom_node_vector[i].size() > 1) { - phantom_node_vector[i].erase(phantom_node_vector[i].begin()); - } - BOOST_ASSERT(phantom_node_vector[i].front().is_valid(facade->GetNumberOfNodes())); - } - } - - void SplitUnaccessibleLocations(PhantomNodeArray & phantom_node_vector, - std::vector & result_table, - std::vector> & components) { - // Run TarjanSCC - auto number_of_locations = phantom_node_vector.size(); - auto wrapper = std::make_shared>(result_table, number_of_locations); - auto empty_restriction = RestrictionMap(std::vector()); - auto empty_vector = std::vector(); - auto scc = TarjanSCC>(wrapper, empty_restriction, empty_vector); - scc.run(); - - for (int j = 0; j < scc.get_number_of_components(); ++j){ - components.push_back(std::vector()); - } - - for (int i = 0; i < number_of_locations; ++i) { - components[scc.get_component_id(i)].push_back(i); - } - } - - void SetLocPermutationOutput(const std::vector & loc_permutation, osrm::json::Object & json_result){ - osrm::json::Array json_loc_permutation; - json_loc_permutation.values.insert(json_loc_permutation.values.end(), loc_permutation.begin(), loc_permutation.end()); - json_result.values["loc_permutation"] = json_loc_permutation; - } - - void SetDistanceOutput(const int distance, osrm::json::Object & json_result) { - json_result.values["distance"] = distance; - } - - void SetRuntimeOutput(const float runtime, osrm::json::Object & json_result) { - json_result.values["runtime"] = runtime; - } - - void SetGeometry(const RouteParameters &route_parameters, const InternalRouteResult & min_route, osrm::json::Object & json_result) { - // return geometry result to json - std::unique_ptr> descriptor; - descriptor = osrm::make_unique>(facade); - - descriptor->SetConfig(route_parameters); - descriptor->Run(min_route, json_result); - } - - int HandleRequest(const RouteParameters &route_parameters, - osrm::json::Object &json_result) override final - { - // check if all inputs are coordinates - if (!check_all_coordinates(route_parameters.coordinates)) { - return 400; - } - - PhantomNodeArray phantom_node_vector(route_parameters.coordinates.size()); - GetPhantomNodes(route_parameters, phantom_node_vector); - - // compute the distance table of all phantom nodes - const std::shared_ptr> result_table = - search_engine_ptr->distance_table(phantom_node_vector); - if (!result_table){ - return 400; - } - - const auto maxint = std::numeric_limits::max(); - if (*std::max_element(result_table->begin(), result_table->end()) == maxint) { - std::unique_ptr> descriptor; - descriptor = osrm::make_unique>(facade); - descriptor->SetConfig(route_parameters); - - std::vector> components; - TIMER_START(tsp); - SplitUnaccessibleLocations(phantom_node_vector, *result_table, components); - auto number_of_locations = phantom_node_vector.size(); - std::vector min_loc_permutation(number_of_locations, -1); - auto min_dist = 0; - for(auto k = 0; k < components.size(); ++k) { - if (components[k].size() > 1) { - InternalRouteResult min_route; - osrm::tsp::FarthestInsertionTSP(components[k], phantom_node_vector, *result_table, min_route, min_loc_permutation); - search_engine_ptr->shortest_path(min_route.segment_end_coordinates, route_parameters.uturns, min_route); - min_dist += min_route.shortest_path_length; - descriptor->Run(min_route, json_result); - } - } - TIMER_STOP(tsp); - - SetRuntimeOutput(TIMER_MSEC(tsp), json_result); - SetDistanceOutput(min_dist, json_result); - SetLocPermutationOutput(min_loc_permutation, json_result); - } else { - auto number_of_locations = phantom_node_vector.size(); - InternalRouteResult min_route; - std::vector min_loc_permutation(number_of_locations, -1); - //######################## FARTHEST INSERTION ###############################// - TIMER_START(tsp); - osrm::tsp::FarthestInsertionTSP(phantom_node_vector, *result_table, min_route, min_loc_permutation); - search_engine_ptr->shortest_path(min_route.segment_end_coordinates, route_parameters.uturns, min_route); - TIMER_STOP(tsp); - - BOOST_ASSERT(min_route.segment_end_coordinates.size() == route_parameters.coordinates.size()); - SetLocPermutationOutput(min_loc_permutation, json_result); - SetDistanceOutput(min_route.shortest_path_length, json_result); - SetRuntimeOutput(TIMER_MSEC(tsp), json_result); - SetGeometry(route_parameters, min_route, json_result); - } - - return 200; - } - -}; - -#endif // ROUND_TRIP_FI_HPP diff --git a/plugins/round_trip_NN.hpp b/plugins/round_trip_NN.hpp deleted file mode 100644 index cc88d899bf9..00000000000 --- a/plugins/round_trip_NN.hpp +++ /dev/null @@ -1,211 +0,0 @@ -/* - -Copyright (c) 2015, Project OSRM contributors -All rights reserved. - -Redistribution and use in source and binary forms, with or without modification, -are permitted provided that the following conditions are met: - -Redistributions of source code must retain the above copyright notice, this list -of conditions and the following disclaimer. -Redistributions in binary form must reproduce the above copyright notice, this -list of conditions and the following disclaimer in the documentation and/or -other materials provided with the distribution. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR -ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES -(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON -ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -*/ - -#ifndef ROUND_TRIP_NN_HPP -#define ROUND_TRIP_NN_HPP - -#include "plugin_base.hpp" - -#include "../algorithms/object_encoder.hpp" -#include "../routing_algorithms/tsp_nearest_neighbour.hpp" -#include "../routing_algorithms/tsp_farthest_insertion.hpp" -#include "../routing_algorithms/tsp_brute_force.hpp" -#include "../data_structures/query_edge.hpp" -#include "../data_structures/search_engine.hpp" -#include "../descriptors/descriptor_base.hpp" -#include "../descriptors/json_descriptor.hpp" -#include "../util/json_renderer.hpp" -#include "../util/make_unique.hpp" -#include "../util/string_util.hpp" -#include "../util/timing_util.hpp" -#include "../util/simple_logger.hpp" -#include "../tools/tsp_logs.hpp" - -#include - -#include -#include -#include -#include -#include -#include -#include - -template class RoundTripPluginNN final : public BasePlugin -{ - private: - std::string descriptor_string; - DataFacadeT *facade; - std::unique_ptr> search_engine_ptr; - - public: - explicit RoundTripPluginNN(DataFacadeT *facade) - : descriptor_string("tripNN"), facade(facade) - { - search_engine_ptr = osrm::make_unique>(facade); - } - - const std::string GetDescriptor() const override final { return descriptor_string; } - - void GetPhantomNodes(const RouteParameters &route_parameters, PhantomNodeArray & phantom_node_vector) { - const bool checksum_OK = (route_parameters.check_sum == facade->GetCheckSum()); - - // find phantom nodes for all input coords - for (const auto i : osrm::irange(0, route_parameters.coordinates.size())) { - // if client hints are helpful, encode hints - if (checksum_OK && i < route_parameters.hints.size() && - !route_parameters.hints[i].empty()) { - PhantomNode current_phantom_node; - ObjectEncoder::DecodeFromBase64(route_parameters.hints[i], current_phantom_node); - if (current_phantom_node.is_valid(facade->GetNumberOfNodes())) - { - phantom_node_vector[i].emplace_back(std::move(current_phantom_node)); - continue; - } - } - facade->IncrementalFindPhantomNodeForCoordinate(route_parameters.coordinates[i], - phantom_node_vector[i], 1); - if (phantom_node_vector[i].size() > 1) { - phantom_node_vector[i].erase(phantom_node_vector[i].begin()); - } - BOOST_ASSERT(phantom_node_vector[i].front().is_valid(facade->GetNumberOfNodes())); - } - } - - void SplitUnaccessibleLocations(PhantomNodeArray & phantom_node_vector, - std::vector & result_table, - std::vector> & components) { - // Run TarjanSCC - auto number_of_locations = phantom_node_vector.size(); - auto wrapper = std::make_shared>(result_table, number_of_locations); - auto empty_restriction = RestrictionMap(std::vector()); - auto empty_vector = std::vector(); - auto scc = TarjanSCC>(wrapper, empty_restriction, empty_vector); - scc.run(); - - for (int j = 0; j < scc.get_number_of_components(); ++j){ - components.push_back(std::vector()); - } - - for (int i = 0; i < number_of_locations; ++i) { - components[scc.get_component_id(i)].push_back(i); - } - } - - void SetLocPermutationOutput(const std::vector & loc_permutation, osrm::json::Object & json_result){ - osrm::json::Array json_loc_permutation; - json_loc_permutation.values.insert(json_loc_permutation.values.end(), loc_permutation.begin(), loc_permutation.end()); - json_result.values["loc_permutation"] = json_loc_permutation; - } - - void SetDistanceOutput(const int distance, osrm::json::Object & json_result) { - json_result.values["distance"] = distance; - } - - void SetRuntimeOutput(const float runtime, osrm::json::Object & json_result) { - json_result.values["runtime"] = runtime; - } - - void SetGeometry(const RouteParameters &route_parameters, const InternalRouteResult & min_route, osrm::json::Object & json_result) { - // return geometry result to json - std::unique_ptr> descriptor; - descriptor = osrm::make_unique>(facade); - - descriptor->SetConfig(route_parameters); - descriptor->Run(min_route, json_result); - } - - int HandleRequest(const RouteParameters &route_parameters, - osrm::json::Object &json_result) override final - { - // check if all inputs are coordinates - if (!check_all_coordinates(route_parameters.coordinates)) { - return 400; - } - - PhantomNodeArray phantom_node_vector(route_parameters.coordinates.size()); - GetPhantomNodes(route_parameters, phantom_node_vector); - - // compute the distance table of all phantom nodes - const std::shared_ptr> result_table = - search_engine_ptr->distance_table(phantom_node_vector); - if (!result_table){ - return 400; - } - - - const auto maxint = std::numeric_limits::max(); - if (*std::max_element(result_table->begin(), result_table->end()) == maxint) { - std::unique_ptr> descriptor; - descriptor = osrm::make_unique>(facade); - descriptor->SetConfig(route_parameters); - - std::vector> components; - TIMER_START(tsp); - SplitUnaccessibleLocations(phantom_node_vector, *result_table, components); - - std::vector min_loc_permutation(phantom_node_vector.size(), -1); - auto min_dist = 0; - for(auto k = 0; k < components.size(); ++k) { - if (components[k].size() > 1) { - InternalRouteResult min_route; - //run nearest neighbour - osrm::tsp::NearestNeighbourTSP(components[k], phantom_node_vector, *result_table, min_route, min_loc_permutation); - //compute route - search_engine_ptr->shortest_path(min_route.segment_end_coordinates, route_parameters.uturns, min_route); - //return geometry - min_dist += min_route.shortest_path_length; - descriptor->Run(min_route, json_result); - } - } - TIMER_STOP(tsp); - - SetRuntimeOutput(TIMER_MSEC(tsp), json_result); - SetDistanceOutput(min_dist, json_result); - SetLocPermutationOutput(min_loc_permutation, json_result); - } else { - auto number_of_locations = phantom_node_vector.size(); - InternalRouteResult min_route; - std::vector min_loc_permutation(number_of_locations, -1); - //######################### NEAREST NEIGHBOUR ###############################// - TIMER_START(tsp); - osrm::tsp::NearestNeighbourTSP(phantom_node_vector, *result_table, min_route, min_loc_permutation); - search_engine_ptr->shortest_path(min_route.segment_end_coordinates, route_parameters.uturns, min_route); - TIMER_STOP(tsp); - BOOST_ASSERT(min_route.segment_end_coordinates.size() == route_parameters.coordinates.size()); - SetLocPermutationOutput(min_loc_permutation, json_result); - SetDistanceOutput(min_route.shortest_path_length, json_result); - SetRuntimeOutput(TIMER_MSEC(tsp), json_result); - SetGeometry(route_parameters, min_route, json_result); - } - - return 200; - } - -}; - -#endif // ROUND_TRIP_NN_HPP diff --git a/routing_algorithms/tsp_brute_force.hpp b/routing_algorithms/tsp_brute_force.hpp index 9e0c266f2fc..eac32aa155b 100644 --- a/routing_algorithms/tsp_brute_force.hpp +++ b/routing_algorithms/tsp_brute_force.hpp @@ -79,7 +79,6 @@ void BruteForceTSP(std::vector & location, InternalRouteResult & min_route, std::vector & min_loc_permutation) { - const auto number_of_location = phantom_node_vector.size(); const int component_size = location.size(); int min_route_dist = std::numeric_limits::max(); diff --git a/server/api_grammar.hpp b/server/api_grammar.hpp index 325f4c356c1..c93518a34b1 100644 --- a/server/api_grammar.hpp +++ b/server/api_grammar.hpp @@ -42,7 +42,7 @@ template struct APIGrammar : qi::grammar> -(uturns); query = ('?') >> (+(zoom | output | jsonp | checksum | location | hint | timestamp | u | cmp | language | instruction | geometry | alt_route | old_API | num_results | - matching_beta | gps_precision | classify | locs)); + matching_beta | gps_precision | classify | tsp_algo | locs)); zoom = (-qi::lit('&')) >> qi::lit('z') >> '=' >> qi::short_[boost::bind(&HandlerT::setZoomLevel, handler, ::_1)]; @@ -85,6 +85,8 @@ template struct APIGrammar : qi::grammar> qi::lit("locs") >> '=' >> stringforPolyline[boost::bind(&HandlerT::getCoordinatesFromGeometry, handler, ::_1)]; + tsp_algo = (-qi::lit('&')) >> qi::lit("tsp_algo") >> '=' >> + string[boost::bind(&HandlerT::setTSPAlgo, handler, ::_1)]; string = +(qi::char_("a-zA-Z")); stringwithDot = +(qi::char_("a-zA-Z0-9_.-")); @@ -96,7 +98,8 @@ template struct APIGrammar : qi::grammar api_call, query; qi::rule service, zoom, output, string, jsonp, checksum, location, hint, timestamp, stringwithDot, stringwithPercent, language, instruction, geometry, cmp, alt_route, u, - uturns, old_API, num_results, matching_beta, gps_precision, classify, locs, stringforPolyline; + uturns, old_API, num_results, matching_beta, gps_precision, classify, locs, stringforPolyline, tsp_algo; + uturns, old_API, num_results, matching_beta, gps_precision, classify, tsp_algo; HandlerT *handler; }; From 77e9e95067b3365c99a5c7b9050abd2f207ebf96 Mon Sep 17 00:00:00 2001 From: Huyen Chau Nguyen Date: Tue, 18 Aug 2015 13:48:12 +0200 Subject: [PATCH 073/122] fix bugs and add todos of code review session with daniel-j-h --- plugins/round_trip.hpp | 151 ++++++++++++++---- routing_algorithms/tsp_brute_force.hpp | 78 +++------ routing_algorithms/tsp_farthest_insertion.hpp | 150 ++++++++--------- routing_algorithms/tsp_nearest_neighbour.hpp | 120 +++----------- routing_algorithms/tsp_tabu_search.hpp | 73 +++++++++ 5 files changed, 303 insertions(+), 269 deletions(-) create mode 100644 routing_algorithms/tsp_tabu_search.hpp diff --git a/plugins/round_trip.hpp b/plugins/round_trip.hpp index 03840824aa4..0670af3d41c 100644 --- a/plugins/round_trip.hpp +++ b/plugins/round_trip.hpp @@ -47,6 +47,7 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "../util/simple_logger.hpp" #include +#include #include #include @@ -55,6 +56,9 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include #include #include +#include + +#include template class RoundTripPlugin final : public BasePlugin { @@ -102,10 +106,10 @@ template class RoundTripPlugin final : public BasePlugin std::vector> & components) { // Run TarjanSCC - auto number_of_locations = phantom_node_vector.size(); + const auto number_of_locations = phantom_node_vector.size(); auto wrapper = std::make_shared>(result_table, number_of_locations); auto empty_restriction = RestrictionMap(std::vector()); - auto empty_vector = std::vector(); + std::vector empty_vector; auto scc = TarjanSCC>(wrapper, empty_restriction, empty_vector); scc.run(); @@ -118,7 +122,8 @@ template class RoundTripPlugin final : public BasePlugin } } - void SetLocPermutationOutput(const std::vector & loc_permutation, osrm::json::Object & json_result){ + template + void SetLocPermutationOutput(const std::vector & loc_permutation, osrm::json::Object & json_result){ osrm::json::Array json_loc_permutation; json_loc_permutation.values.insert(json_loc_permutation.values.end(), loc_permutation.begin(), loc_permutation.end()); json_result.values["loc_permutation"] = json_loc_permutation; @@ -140,6 +145,36 @@ template class RoundTripPlugin final : public BasePlugin descriptor->Run(min_route, json_result); } + void ComputeRoute(const PhantomNodeArray & phantom_node_vector, + const RouteParameters & route_parameters, + std::vector & trip, + InternalRouteResult & min_route) { + // given he final trip, compute total distance and return the route and location permutation + PhantomNodes viapoint; + for (auto it = trip.begin(); it != std::prev(trip.end()); ++it) { + auto from_node = *it; + auto to_node = *std::next(it); + viapoint = PhantomNodes{phantom_node_vector[from_node][0], phantom_node_vector[to_node][0]}; + min_route.segment_end_coordinates.emplace_back(viapoint); + } + // check dist between last and first location too + viapoint = PhantomNodes{phantom_node_vector[*std::prev(trip.end())][0], phantom_node_vector[trip.front()][0]}; + min_route.segment_end_coordinates.emplace_back(viapoint); + search_engine_ptr->shortest_path(min_route.segment_end_coordinates, route_parameters.uturns, min_route); + } + + void ComputeRoute(const PhantomNodeArray & phantom_node_vector, + const RouteParameters & route_parameters, + std::vector> & trip, + std::vector & route) { + for (const auto & curr_trip : trip) { + InternalRouteResult curr_route; + ComputeRoute(phantom_node_vector, route_parameters, curr_trip, curr_route); + route.push_back(curr_route); + search_engine_ptr->shortest_path(route.back().segment_end_coordinates, route_parameters.uturns, route.back()); + } + } + int HandleRequest(const RouteParameters &route_parameters, osrm::json::Object &json_result) override final { @@ -158,67 +193,117 @@ template class RoundTripPlugin final : public BasePlugin return 400; } + + BOOST_ASSERT_MSG(result_table->size() > 0, "Distance Table is empty."); //check if locations are in different strongly connected components (SCC) - const auto maxint = std::numeric_limits::max(); - if (*std::max_element(result_table->begin(), result_table->end()) == maxint) { - //run TSP computation for every SCC + const auto maxint = std::numeric_limits::max(); + if (*std::max_element(std::begin(*result_table), std::end(*result_table)) == maxint) { + + //TODO DELETE + // JSON output related objects std::unique_ptr> descriptor; descriptor = osrm::make_unique>(facade); descriptor->SetConfig(route_parameters); - std::vector> components; TIMER_START(tsp); + // Compute all SCC + std::vector> components; SplitUnaccessibleLocations(phantom_node_vector, *result_table, components); - auto number_of_locations = phantom_node_vector.size(); - std::vector min_loc_permutation(number_of_locations, -1); + // std::vector> res_route (components.size()-1); + std::vector> res_route; + const constexpr std::size_t BF_MAX_FEASABLE = 14; + - auto min_dist = 0; + //run TSP computation for every SCC for(auto k = 0; k < components.size(); ++k) { if (components[k].size() > 1) { + std::vector scc_route; + scc_route.reserve(components[k].size()); + // Compute the TSP with the given algorithm - InternalRouteResult min_route; - if (route_parameters.tsp_algo == "BF" && route_parameters.coordinates.size() < 14) { - osrm::tsp::BruteForceTSP(components[k], phantom_node_vector, *result_table, min_route, min_loc_permutation); + if (route_parameters.tsp_algo == "BF" && route_parameters.coordinates.size() < BF_MAX_FEASABLE) { + SimpleLogger().Write() << "Running SCC BF"; + osrm::tsp::BruteForceTSP(components[k], phantom_node_vector, *result_table, scc_route); + res_route.push_back(scc_route); } else if (route_parameters.tsp_algo == "NN") { - osrm::tsp::NearestNeighbourTSP(components[k], phantom_node_vector, *result_table, min_route, min_loc_permutation); + SimpleLogger().Write() << "Running SCC NN"; + osrm::tsp::NearestNeighbourTSP(components[k], phantom_node_vector, *result_table, scc_route); + res_route.push_back(scc_route); } else if (route_parameters.tsp_algo == "FI") { - osrm::tsp::FarthestInsertionTSP(components[k], phantom_node_vector, *result_table, min_route, min_loc_permutation); + SimpleLogger().Write() << "Running SCC FI"; + osrm::tsp::FarthestInsertionTSP(components[k], phantom_node_vector, *result_table, scc_route); + res_route.push_back(scc_route); } else{ - osrm::tsp::FarthestInsertionTSP(components[k], phantom_node_vector, *result_table, min_route, min_loc_permutation); + SimpleLogger().Write() << "Running SCC FI"; + osrm::tsp::FarthestInsertionTSP(components[k], phantom_node_vector, *result_table, scc_route); + res_route.push_back(scc_route); } - search_engine_ptr->shortest_path(min_route.segment_end_coordinates, route_parameters.uturns, min_route); - min_dist += min_route.shortest_path_length; - descriptor->Run(min_route, json_result); } } - + SimpleLogger().Write() << "DONE"; + std::vector route; + ComputeRoute(phantom_node_vector, route_parameters, res_route, route); TIMER_STOP(tsp); SetRuntimeOutput(TIMER_MSEC(tsp), json_result); - SetDistanceOutput(min_dist, json_result); - SetLocPermutationOutput(min_loc_permutation, json_result); + + // SimpleLogger().Write() << "Route is"; + // for (auto x : res_route) { + // for (auto y : x) + // std::cout << y << " "; + // } + // SimpleLogger().Write() << ""; + + auto dist = 0; + for (auto curr_route : route) { + dist += curr_route.shortest_path_length; + SetGeometry(route_parameters, curr_route, json_result); + } + SetDistanceOutput(dist, json_result); } else { //run TSP computation for all locations auto number_of_locations = phantom_node_vector.size(); - InternalRouteResult min_route; - std::vector min_loc_permutation(number_of_locations, -1); + std::vector res_route; + res_route.reserve(number_of_locations); // Compute the TSP with the given algorithm TIMER_START(tsp); - if (route_parameters.tsp_algo == "BF" && route_parameters.coordinates.size() < 14) { - osrm::tsp::BruteForceTSP(phantom_node_vector, *result_table, min_route, min_loc_permutation); + // TODO patrick nach userfreundlichkeit fragen, BF vs bf usw + if (route_parameters.tsp_algo == "BF" && route_parameters.coordinates.size() < BF_MAX_FEASABLE) { + SimpleLogger().Write() << "Running BF"; + res_route = osrm::tsp::BruteForceTSP(phantom_node_vector, *result_table, res_route); } else if (route_parameters.tsp_algo == "NN") { - osrm::tsp::NearestNeighbourTSP(phantom_node_vector, *result_table, min_route, min_loc_permutation); + SimpleLogger().Write() << "Running NN"; + osrm::tsp::NearestNeighbourTSP(phantom_node_vector, *result_table, res_route); } else if (route_parameters.tsp_algo == "FI") { - osrm::tsp::FarthestInsertionTSP(phantom_node_vector, *result_table, min_route, min_loc_permutation); + SimpleLogger().Write() << "Running FI"; + osrm::tsp::FarthestInsertionTSP(phantom_node_vector, *result_table, res_route); } else { - osrm::tsp::FarthestInsertionTSP(phantom_node_vector, *result_table, min_route, min_loc_permutation); + SimpleLogger().Write() << "Running FI"; + osrm::tsp::FarthestInsertionTSP(phantom_node_vector, *result_table, res_route); + // osrm::tsp::NearestNeighbourTSP(phantom_node_vector, *result_table, res_route); } - search_engine_ptr->shortest_path(min_route.segment_end_coordinates, route_parameters.uturns, min_route); + // TODO asserts numer of result blablabla size + // TODO std::is_permutation + // TODO boost range + SimpleLogger().Write() << "DONE"; + + + InternalRouteResult min_route; + ComputeRoute(phantom_node_vector, route_parameters, res_route, min_route); TIMER_STOP(tsp); - BOOST_ASSERT(min_route.segment_end_coordinates.size() == route_parameters.coordinates.size()); - SetLocPermutationOutput(min_loc_permutation, json_result); - SetDistanceOutput(min_route.shortest_path_length, json_result); + + // SimpleLogger().Write() << "Route is"; + // for (auto x : res_route) { + // std::cout << x << " "; + // } + // SimpleLogger().Write() << ""; + + //TODO TIMER im LOGGER SetRuntimeOutput(TIMER_MSEC(tsp), json_result); + SetLocPermutationOutput(res_route, json_result); + //TODO MEHR ASSERTIONS! :O + SetDistanceOutput(min_route.shortest_path_length, json_result); SetGeometry(route_parameters, min_route, json_result); + BOOST_ASSERT(min_route.segment_end_coordinates.size() == route_parameters.coordinates.size()); } diff --git a/routing_algorithms/tsp_brute_force.hpp b/routing_algorithms/tsp_brute_force.hpp index eac32aa155b..bb83aa888dd 100644 --- a/routing_algorithms/tsp_brute_force.hpp +++ b/routing_algorithms/tsp_brute_force.hpp @@ -52,87 +52,57 @@ namespace tsp { template -int ReturnDistance(const std::vector & dist_table, const std::vector & location_order, const int min_route_dist, const int number_of_locations, const int component_size) { - int i = 0; +int ReturnDistance(const std::vector & dist_table, + const std::vector & location_order, + const int min_route_dist, + const int number_of_locations) { int route_dist = 0; - - // compute length and stop if length is longer than route already found - while (i < component_size - 1 && route_dist < min_route_dist) { - //get distance from location i to location i+1 + int i = 0; + while (i < location_order.size() - 1 && route_dist < min_route_dist) { route_dist += *(dist_table.begin() + (location_order[i] * number_of_locations) + location_order[i+1]); ++i; } //get distance from last location to first location - route_dist += *(dist_table.begin() + (location_order[component_size-1] * number_of_locations) + location_order[0]); - - if (route_dist < min_route_dist) { - return route_dist; - } - else { - return -1; - } + route_dist += *(dist_table.begin() + (location_order[location_order.size()-1] * number_of_locations) + location_order[0]); + return route_dist; } -void BruteForceTSP(std::vector & location, +void BruteForceTSP(std::vector & component, const PhantomNodeArray & phantom_node_vector, const std::vector & dist_table, - InternalRouteResult & min_route, - std::vector & min_loc_permutation) { - - const int component_size = location.size(); - int min_route_dist = std::numeric_limits::max(); + std::vector & route) { - std::vector min_location; + const unsigned component_size = component.size(); + unsigned min_route_dist = std::numeric_limits::max(); - // check length of all possible permutation of the location ids + // check length of all possible permutation of the component ids do { - // int new_distance = ReturnDistance(dist_table, location, min_route_dist, number_of_location, component_size); - int new_distance = 4; - if (new_distance != -1) { + const auto new_distance = ReturnDistance(dist_table, component, min_route_dist, component_size); + if (new_distance < min_route_dist) { min_route_dist = new_distance; - min_location = location; + route = component; } - } while(std::next_permutation(location.begin(), location.end())); - - PhantomNodes viapoint; - for (int i = 0; i < component_size - 1; ++i) { - viapoint = PhantomNodes{phantom_node_vector[min_location[i]][0], phantom_node_vector[min_location[i + 1]][0]}; - min_route.segment_end_coordinates.emplace_back(viapoint); - min_loc_permutation[min_location[i]] = i; - } - min_loc_permutation[min_location[component_size - 1]] = component_size - 1; - viapoint = PhantomNodes{phantom_node_vector[min_location[component_size - 1]][0], phantom_node_vector[min_location[0]][0]}; - min_route.segment_end_coordinates.emplace_back(viapoint); + } while(std::next_permutation(component.begin(), component.end())); } void BruteForceTSP(const PhantomNodeArray & phantom_node_vector, const std::vector & dist_table, - InternalRouteResult & min_route, - std::vector & min_loc_permutation) { - + std::vector & route) { const auto number_of_locations = phantom_node_vector.size(); // fill a vector with node ids - std::vector location_ids(number_of_locations); + std::vector location_ids(number_of_locations); std::iota(location_ids.begin(), location_ids.end(), 0); - int min_route_dist = std::numeric_limits::max(); - + unsigned min_route_dist = std::numeric_limits::max(); // check length of all possible permutation of the location ids do { - int new_distance = ReturnDistance(dist_table, location_ids, min_route_dist, number_of_locations, number_of_locations); - if (new_distance != -1) { + const auto new_distance = ReturnDistance(dist_table, location_ids, min_route_dist, number_of_locations); + + if (new_distance < min_route_dist) { min_route_dist = new_distance; - //TODO: this gets copied right? fix this - min_loc_permutation = location_ids; + route = location_ids; } } while(std::next_permutation(location_ids.begin(), location_ids.end())); - PhantomNodes viapoint; - for (int i = 0; i < number_of_locations - 1; ++i) { - viapoint = PhantomNodes{phantom_node_vector[min_loc_permutation[i]][0], phantom_node_vector[min_loc_permutation[i + 1]][0]}; - min_route.segment_end_coordinates.emplace_back(viapoint); - } - viapoint = PhantomNodes{phantom_node_vector[min_loc_permutation[number_of_locations - 1]][0], phantom_node_vector[min_loc_permutation[0]][0]}; - min_route.segment_end_coordinates.emplace_back(viapoint); } } diff --git a/routing_algorithms/tsp_farthest_insertion.hpp b/routing_algorithms/tsp_farthest_insertion.hpp index cb3353d9e4b..94755306fa1 100644 --- a/routing_algorithms/tsp_farthest_insertion.hpp +++ b/routing_algorithms/tsp_farthest_insertion.hpp @@ -49,66 +49,45 @@ namespace osrm namespace tsp { -void GetLongestRoundTrip(const int current_loc, - std::list & current_trip, +void GetShortestRoundTrip(const int current_loc, const std::vector & dist_table, const int number_of_locations, - int & longest_min_tour, - std::list::iterator & following_loc){ + std::vector & current_trip, + int & min_trip_distance, + std::vector::iterator & next_insert_point_candidate){ // for all nodes in the current trip find the best insertion resulting in the shortest path + // assert min 2 nodes in current_trip for (auto from_node = current_trip.begin(); from_node != std::prev(current_trip.end()); ++from_node) { - auto to_node = std::next(from_node); + const auto to_node = std::next(from_node); - auto dist_from = *(dist_table.begin() + (*from_node * number_of_locations) + current_loc); - auto dist_to = *(dist_table.begin() + (current_loc * number_of_locations) + *to_node); - auto trip_dist = dist_from + dist_to - *(dist_table.begin() + (*from_node * number_of_locations) + *to_node); + const auto dist_from = *(dist_table.begin() + (*from_node * number_of_locations) + current_loc); + const auto dist_to = *(dist_table.begin() + (current_loc * number_of_locations) + *to_node); + const auto trip_dist = dist_from + dist_to - *(dist_table.begin() + (*from_node * number_of_locations) + *to_node); - // from all possible insertions to the current trip, choose the longest of all minimal insertions - if (trip_dist < longest_min_tour) { - longest_min_tour = trip_dist; - following_loc = to_node; + // from all possible insertions to the current trip, choose the shortest of all insertions + if (trip_dist < min_trip_distance) { + min_trip_distance = trip_dist; + next_insert_point_candidate = to_node; } } - { // check insertion between last and first location too - auto from_node = std::prev(current_trip.end()); - auto to_node = current_trip.begin(); - - auto dist_from = *(dist_table.begin() + (*from_node * number_of_locations) + current_loc); - auto dist_to = *(dist_table.begin() + (current_loc * number_of_locations) + *to_node); - auto trip_dist = dist_from + dist_to - *(dist_table.begin() + (*from_node * number_of_locations) + *to_node); - if (trip_dist < longest_min_tour) { - longest_min_tour = trip_dist; - following_loc = to_node; - } - } -} - -void ComputeRouteAndPermutation(const PhantomNodeArray & phantom_node_vector, - std::list & current_trip, - InternalRouteResult & min_route, - std::vector & min_loc_permutation) { - // given he final trip, compute total distance and return the route and location permutation - PhantomNodes viapoint; - int perm = 0; - for (auto it = current_trip.begin(); it != std::prev(current_trip.end()); ++it) { - auto from_node = *it; - auto to_node = *std::next(it); - viapoint = PhantomNodes{phantom_node_vector[from_node][0], phantom_node_vector[to_node][0]}; - min_route.segment_end_coordinates.emplace_back(viapoint); - min_loc_permutation[from_node] = perm; - ++perm; + // check insertion between last and first location too + auto from_node = std::prev(current_trip.end()); + auto to_node = current_trip.begin(); + + auto dist_from = *(dist_table.begin() + (*from_node * number_of_locations) + current_loc); + auto dist_to = *(dist_table.begin() + (current_loc * number_of_locations) + *to_node); + auto trip_dist = dist_from + dist_to - *(dist_table.begin() + (*from_node * number_of_locations) + *to_node); + if (trip_dist < min_trip_distance) { + min_trip_distance = trip_dist; + next_insert_point_candidate = to_node; } - // check dist between last and first location too - viapoint = PhantomNodes{phantom_node_vector[*std::prev(current_trip.end())][0], phantom_node_vector[current_trip.front()][0]}; - min_route.segment_end_coordinates.emplace_back(viapoint); - min_loc_permutation[*std::prev(current_trip.end())] = perm; } +// osrm::tsp::FarthestInsertionTSP(components[k], phantom_node_vector, *result_table, scc_route); void FarthestInsertionTSP(const std::vector & locations, const PhantomNodeArray & phantom_node_vector, const std::vector & dist_table, - InternalRouteResult & min_route, - std::vector & min_loc_permutation) { + std::vector & current_trip) { ////////////////////////////////////////////////////////////////////////////////////////////////// // START FARTHEST INSERTION HERE // 1. start at a random round trip of 2 locations @@ -119,63 +98,65 @@ void FarthestInsertionTSP(const std::vector & locations, ////////////////////////////////////////////////////////////////////////////////////////////////// const int number_of_locations = phantom_node_vector.size(); const int size_of_component = locations.size(); - // list of the trip that will be found incrementally - std::list current_trip; // tracks which nodes have been already visited std::vector visited(number_of_locations, false); auto max_dist = 0; - auto index = -1; + auto max_from = -1; + auto max_to = -1; + + //TODO for (auto x : locations) { for (auto y : locations) { - if (*(dist_table.begin() + x * number_of_locations + y) > max_dist) { - max_dist = *(dist_table.begin() + x * number_of_locations + y); - index = x * number_of_locations + y; + auto xy_dist = *(dist_table.begin() + x * number_of_locations + y); + if (xy_dist > max_dist) { + max_dist = xy_dist; + max_from = x; + max_to = y; } } } - const int max_from = index / number_of_locations; - const int max_to = index % number_of_locations; visited[max_from] = true; visited[max_to] = true; current_trip.push_back(max_from); current_trip.push_back(max_to); - + // SimpleLogger().Write() << size_of_component; // add all other nodes missing (two nodes are already in the initial start trip) for (int j = 2; j < size_of_component; ++j) { - auto shortest_max_tour = -1; - int next_node = -1; - std::list::iterator min_max_insert; + // SimpleLogger().Write() << j << "/" << size_of_component; + auto farthest_distance = 0; + auto next_node = -1; + std::vector::iterator next_insert_point; // find unvisited loc i that is the farthest away from all other visited locs for (auto i : locations) { + // find the shortest distance from i to all visited nodes if (!visited[i]) { - // longest_min_tour is the distance of the longest of all insertions with the minimal distance - auto longest_min_tour = std::numeric_limits::max(); - // following_loc is the location that comes after the location that is to be inserted - std::list::iterator following_loc; - GetLongestRoundTrip(i, current_trip, dist_table, number_of_locations, longest_min_tour, following_loc); + auto min_trip_distance = std::numeric_limits::max(); + std::vector::iterator next_insert_point_candidate; + + GetShortestRoundTrip(i, dist_table, number_of_locations, current_trip, min_trip_distance, next_insert_point_candidate); // add the location to the current trip such that it results in the shortest total tour - if (longest_min_tour > shortest_max_tour) { - shortest_max_tour = longest_min_tour; + // SimpleLogger().Write() << "min_trip_distance " << min_trip_distance; + if (min_trip_distance >= farthest_distance) { + farthest_distance = min_trip_distance; next_node = i; - min_max_insert = following_loc; + next_insert_point = next_insert_point_candidate; } } } + // SimpleLogger().Write() << "next node " << next_node; // mark as visited and insert node visited[next_node] = true; - current_trip.insert(min_max_insert, next_node); + current_trip.insert(next_insert_point, next_node); } - ComputeRouteAndPermutation(phantom_node_vector, current_trip, min_route, min_loc_permutation); } void FarthestInsertionTSP(const PhantomNodeArray & phantom_node_vector, const std::vector & dist_table, - InternalRouteResult & min_route, - std::vector & min_loc_permutation) { + std::vector & current_trip) { ////////////////////////////////////////////////////////////////////////////////////////////////// // START FARTHEST INSERTION HERE // 1. start at a random round trip of 2 locations @@ -186,12 +167,9 @@ void FarthestInsertionTSP(const PhantomNodeArray & phantom_node_vector, ////////////////////////////////////////////////////////////////////////////////////////////////// const auto number_of_locations = phantom_node_vector.size(); - // list of the trip that will be found incrementally - std::list current_trip; // tracks which nodes have been already visited std::vector visited(number_of_locations, false); - // find the pair of location with the biggest distance and make the pair the initial start trip const auto index = std::distance(dist_table.begin(), std::max_element(dist_table.begin(), dist_table.end())); const int max_from = index / number_of_locations; @@ -203,34 +181,32 @@ void FarthestInsertionTSP(const PhantomNodeArray & phantom_node_vector, // add all other nodes missing (two nodes are already in the initial start trip) for (int j = 2; j < number_of_locations; ++j) { - auto shortest_max_tour = -1; - int next_node = -1; - std::list::iterator min_max_insert; + auto farthest_distance = 0; + auto next_node = -1; + //todo move out of loop and overwrite + std::vector::iterator next_insert_point; // find unvisited loc i that is the farthest away from all other visited locs for (int i = 0; i < number_of_locations; ++i) { if (!visited[i]) { - // longest_min_tour is the distance of the longest of all insertions with the minimal distance - auto longest_min_tour = std::numeric_limits::max(); - // following_loc is the location that comes after the location that is to be inserted - std::list::iterator following_loc; + auto min_trip_distance = std::numeric_limits::max(); + std::vector::iterator next_insert_point_candidate; - GetLongestRoundTrip(i, current_trip, dist_table, number_of_locations, longest_min_tour, following_loc); + GetShortestRoundTrip(i, dist_table, number_of_locations, current_trip, min_trip_distance, next_insert_point_candidate); // add the location to the current trip such that it results in the shortest total tour - if (longest_min_tour > shortest_max_tour) { - shortest_max_tour = longest_min_tour; + if (min_trip_distance >= farthest_distance) { + farthest_distance = min_trip_distance; next_node = i; - min_max_insert = following_loc; + next_insert_point = next_insert_point_candidate; } } } + // mark as visited and insert node visited[next_node] = true; - current_trip.insert(min_max_insert, next_node); + current_trip.insert(next_insert_point, next_node); } - - ComputeRouteAndPermutation(phantom_node_vector, current_trip, min_route, min_loc_permutation); } diff --git a/routing_algorithms/tsp_nearest_neighbour.hpp b/routing_algorithms/tsp_nearest_neighbour.hpp index 6a5d059948c..8c182f7792f 100644 --- a/routing_algorithms/tsp_nearest_neighbour.hpp +++ b/routing_algorithms/tsp_nearest_neighbour.hpp @@ -51,8 +51,7 @@ namespace tsp void NearestNeighbourTSP(const std::vector & locations, const PhantomNodeArray & phantom_node_vector, const std::vector & dist_table, - InternalRouteResult & min_route, - std::vector & min_loc_permutation) { + std::vector & route) { ////////////////////////////////////////////////////////////////////////////////////////////////// // START GREEDY NEAREST NEIGHBOUR HERE // 1. grab a random location and mark as starting point @@ -65,25 +64,25 @@ void NearestNeighbourTSP(const std::vector & locations, ////////////////////////////////////////////////////////////////////////////////////////////////// const auto number_of_locations = phantom_node_vector.size(); - const int size_of_component = locations.size(); - min_route.shortest_path_length = std::numeric_limits::max(); + const int component_size = locations.size(); + int shortest_trip_distance = std::numeric_limits::max(); // ALWAYS START AT ANOTHER STARTING POINT for(auto start_node : locations) { int curr_node = start_node; - InternalRouteResult raw_route; - //TODO: Should we always use the same vector or does it not matter at all because of loop scope? - std::vector loc_permutation(number_of_locations, -1); - loc_permutation[start_node] = 0; + + std::vector curr_route; + curr_route.reserve(component_size); + curr_route.push_back(start_node); + // visited[i] indicates whether node i was already visited by the salesman std::vector visited(number_of_locations, false); visited[start_node] = true; - PhantomNodes viapoint; // 3. REPEAT FOR EVERY UNVISITED NODE int trip_dist = 0; - for(int via_point = 1; via_point < size_of_component; ++via_point) + for(int via_point = 1; via_point < component_size; ++via_point) { int min_dist = std::numeric_limits::max(); int min_id = -1; @@ -96,84 +95,23 @@ void NearestNeighbourTSP(const std::vector & locations, min_id = next; } } - loc_permutation[min_id] = via_point; visited[min_id] = true; - viapoint = PhantomNodes{phantom_node_vector[curr_node][0], phantom_node_vector[min_id][0]}; - raw_route.segment_end_coordinates.emplace_back(viapoint); + curr_route.push_back(min_id); trip_dist += min_dist; curr_node = min_id; } - // 4. ROUTE BACK TO STARTING POINT - viapoint = PhantomNodes{raw_route.segment_end_coordinates.back().target_phantom, phantom_node_vector[start_node][0]}; - raw_route.segment_end_coordinates.emplace_back(viapoint); - // check round trip with this starting point is shorter than the shortest round trip found till now - if (trip_dist < min_route.shortest_path_length) { - min_route = raw_route; - min_route.shortest_path_length = trip_dist; - //TODO: this gets copied right? fix this - min_loc_permutation = loc_permutation; + if (trip_dist < shortest_trip_distance) { + shortest_trip_distance = trip_dist; + route = curr_route; } } - - // // ALWAYS START AT ANOTHER STARTING POINT - // for(auto start_node : locations) { - // SimpleLogger().Write() << "STARTING AT " << start_node; - // int curr_node = start_node; - // InternalRouteResult raw_route; - // //TODO: Should we always use the same vector or does it not matter at all because of loop scope? - // std::vector loc_permutation(number_of_locations, -1); - - // // visited[i] indicates whether node i was already visited by the salesman - // std::vector visited(number_of_locations, false); - // visited[start_node] = true; - // loc_permutation[start_node] = 0; - - // PhantomNodes viapoint; - // // 3. REPEAT FOR EVERY UNVISITED NODE - // int trip_dist = 0; - // for(int via_point = 1; via_point < size_of_component; ++via_point) - // { - // int min_dist = std::numeric_limits::max(); - // int min_id = -1; - - // // 2. FIND NEAREST NEIGHBOUR - // for (auto next : locations) { - // if(!visited[next] && - // *(dist_table.begin() + curr_node * number_of_locations + next) < min_dist) { - // min_dist = *(dist_table.begin() + curr_node * number_of_locations + next); - // min_id = next; - // } - // } - - // loc_permutation[min_id] = via_point; - // visited[min_id] = true; - // SimpleLogger().Write() << "MOVING TO " << min_id; - // viapoint = PhantomNodes{phantom_node_vector[curr_node][0], phantom_node_vector[min_id][0]}; - // raw_route.segment_end_coordinates.emplace_back(viapoint); - // trip_dist += min_dist; - // curr_node = min_id; - // } - - // // 4. ROUTE BACK TO STARTING POINT - // viapoint = PhantomNodes{raw_route.segment_end_coordinates.back().target_phantom, phantom_node_vector[start_node][0]}; - // raw_route.segment_end_coordinates.emplace_back(viapoint); - - // // check round trip with this starting point is shorter than the shortest round trip found till now - // if (trip_dist < min_route.shortest_path_length) { - // min_route = raw_route; - // min_route.shortest_path_length = trip_dist; - // //TODO: this gets copied right? fix this - // min_loc_permutation = loc_permutation; - // } - // } } void NearestNeighbourTSP(const PhantomNodeArray & phantom_node_vector, const std::vector & dist_table, - InternalRouteResult & min_route, - std::vector & min_loc_permutation) { + std::vector & route) { ////////////////////////////////////////////////////////////////////////////////////////////////// // START GREEDY NEAREST NEIGHBOUR HERE // 1. grab a random location and mark as starting point @@ -186,21 +124,21 @@ void NearestNeighbourTSP(const PhantomNodeArray & phantom_node_vector, ////////////////////////////////////////////////////////////////////////////////////////////////// const auto number_of_locations = phantom_node_vector.size(); - min_route.shortest_path_length = std::numeric_limits::max(); + int shortest_trip_distance = std::numeric_limits::max(); // ALWAYS START AT ANOTHER STARTING POINT for(int start_node = 0; start_node < number_of_locations; ++start_node) { int curr_node = start_node; - InternalRouteResult raw_route; - //TODO: Should we always use the same vector or does it not matter at all because of loop scope? - std::vector loc_permutation(number_of_locations, -1); - loc_permutation[start_node] = 0; + + std::vector curr_route; + curr_route.reserve(number_of_locations); + curr_route.push_back(start_node); + // visited[i] indicates whether node i was already visited by the salesman std::vector visited(number_of_locations, false); visited[start_node] = true; - PhantomNodes viapoint; // 3. REPEAT FOR EVERY UNVISITED NODE int trip_dist = 0; for(int via_point = 1; via_point < number_of_locations; ++via_point) @@ -212,31 +150,23 @@ void NearestNeighbourTSP(const PhantomNodeArray & phantom_node_vector, auto row_begin_iterator = dist_table.begin() + (curr_node * number_of_locations); auto row_end_iterator = dist_table.begin() + ((curr_node + 1) * number_of_locations); for (auto it = row_begin_iterator; it != row_end_iterator; ++it) { - auto index = std::distance(row_begin_iterator, it); + const auto index = std::distance(row_begin_iterator, it); if (!visited[index] && *it < min_dist) { min_dist = *it; min_id = index; } } - loc_permutation[min_id] = via_point; visited[min_id] = true; - viapoint = PhantomNodes{phantom_node_vector[curr_node][0], phantom_node_vector[min_id][0]}; - raw_route.segment_end_coordinates.emplace_back(viapoint); + curr_route.push_back(min_id); trip_dist += min_dist; curr_node = min_id; } - // 4. ROUTE BACK TO STARTING POINT - viapoint = PhantomNodes{raw_route.segment_end_coordinates.back().target_phantom, phantom_node_vector[start_node][0]}; - raw_route.segment_end_coordinates.emplace_back(viapoint); - // check round trip with this starting point is shorter than the shortest round trip found till now - if (trip_dist < min_route.shortest_path_length) { - min_route = raw_route; - min_route.shortest_path_length = trip_dist; - //TODO: this gets copied right? fix this - min_loc_permutation = loc_permutation; + if (trip_dist < shortest_trip_distance) { + shortest_trip_distance = trip_dist; + route = curr_route; } } } diff --git a/routing_algorithms/tsp_tabu_search.hpp b/routing_algorithms/tsp_tabu_search.hpp new file mode 100644 index 00000000000..4db7dc470e6 --- /dev/null +++ b/routing_algorithms/tsp_tabu_search.hpp @@ -0,0 +1,73 @@ +/* + +Copyright (c) 2015, Project OSRM contributors +All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + +Redistributions of source code must retain the above copyright notice, this list +of conditions and the following disclaimer. +Redistributions in binary form must reproduce the above copyright notice, this +list of conditions and the following disclaimer in the documentation and/or +other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +*/ + +#ifndef TSP_BRUTE_FORCE_HPP +#define TSP_BRUTE_FORCE_HPP + + +#include "../data_structures/search_engine.hpp" +#include "../util/string_util.hpp" + +#include + +#include + +#include +#include +#include +#include +#include +#include "../util/simple_logger.hpp" + + + + +namespace osrm +{ +namespace tsp +{ + +void TabuSearchTSP(std::vector & location, + const PhantomNodeArray & phantom_node_vector, + const std::vector & dist_table, + InternalRouteResult & min_route, + std::vector & min_loc_permutation) { + + +} + +void TabuSearchTSP(const PhantomNodeArray & phantom_node_vector, + const std::vector & dist_table, + InternalRouteResult & min_route, + std::vector & min_loc_permutation) { + + +} + +} +} +#endif // TSP_BRUTE_FORCE_HPP \ No newline at end of file From 3061c8b854abca0840652f1cb9d88c55cee47e8f Mon Sep 17 00:00:00 2001 From: Huyen Chau Nguyen Date: Wed, 19 Aug 2015 10:04:36 +0200 Subject: [PATCH 074/122] solve merge conflicts --- algorithms/tarjan_scc.hpp | 6 +-- data_structures/matrix_graph_wrapper.hpp | 4 +- plugins/round_trip.hpp | 63 ++++++++++++------------ server/api_grammar.hpp | 1 - 4 files changed, 37 insertions(+), 37 deletions(-) diff --git a/algorithms/tarjan_scc.hpp b/algorithms/tarjan_scc.hpp index 05bbb0422a0..8444d785a21 100644 --- a/algorithms/tarjan_scc.hpp +++ b/algorithms/tarjan_scc.hpp @@ -25,8 +25,8 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -#ifndef TINY_COMPONENTS_HPP -#define TINY_COMPONENTS_HPP +#ifndef TARJAN_SCC_HPP +#define TARJAN_SCC_HPP #include "../typedefs.h" #include "../data_structures/deallocating_vector.hpp" @@ -209,4 +209,4 @@ template class TarjanSCC unsigned get_component_id(const NodeID node) const { return components_index[node]; } }; -#endif /* TINY_COMPONENTS_HPP */ +#endif /* TARJAN_SCC_HPP */ diff --git a/data_structures/matrix_graph_wrapper.hpp b/data_structures/matrix_graph_wrapper.hpp index 3bf9259ec17..7e91e8b3314 100644 --- a/data_structures/matrix_graph_wrapper.hpp +++ b/data_structures/matrix_graph_wrapper.hpp @@ -38,7 +38,7 @@ template class MatrixGraphWrapper { MatrixGraphWrapper(std::vector table, const unsigned number_of_nodes) : table_(table), number_of_nodes_(number_of_nodes) {}; - unsigned GetNumberOfNodes() { + unsigned GetNumberOfNodes() const { return number_of_nodes_; } @@ -53,7 +53,7 @@ template class MatrixGraphWrapper { return edges; } - unsigned GetTarget(const unsigned edge) { + unsigned GetTarget(const unsigned edge) const { return edge; } diff --git a/plugins/round_trip.hpp b/plugins/round_trip.hpp index 0670af3d41c..a1a955f9f21 100644 --- a/plugins/round_trip.hpp +++ b/plugins/round_trip.hpp @@ -31,13 +31,15 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "plugin_base.hpp" #include "../algorithms/object_encoder.hpp" -#include "../algorithms/tiny_components.hpp" +#include "../algorithms/tarjan_scc.hpp" #include "../routing_algorithms/tsp_nearest_neighbour.hpp" #include "../routing_algorithms/tsp_farthest_insertion.hpp" #include "../routing_algorithms/tsp_brute_force.hpp" #include "../data_structures/query_edge.hpp" #include "../data_structures/search_engine.hpp" #include "../data_structures/matrix_graph_wrapper.hpp" +#include "../data_structures/restriction.hpp" +#include "../data_structures/restriction_map.hpp" #include "../descriptors/descriptor_base.hpp" #include "../descriptors/json_descriptor.hpp" #include "../util/json_renderer.hpp" @@ -103,21 +105,21 @@ template class RoundTripPlugin final : public BasePlugin void SplitUnaccessibleLocations(PhantomNodeArray & phantom_node_vector, std::vector & result_table, - std::vector> & components) { + std::vector> & components) { // Run TarjanSCC const auto number_of_locations = phantom_node_vector.size(); auto wrapper = std::make_shared>(result_table, number_of_locations); - auto empty_restriction = RestrictionMap(std::vector()); - std::vector empty_vector; - auto scc = TarjanSCC>(wrapper, empty_restriction, empty_vector); + // auto empty_restriction = RestrictionMap(std::vector()); + // std::vector empty_vector; + auto scc = TarjanSCC>(wrapper); scc.run(); - for (int j = 0; j < scc.get_number_of_components(); ++j){ - components.push_back(std::vector()); + for (size_t j = 0; j < scc.get_number_of_components(); ++j){ + components.push_back(std::vector()); } - for (int i = 0; i < number_of_locations; ++i) { + for (size_t i = 0; i < number_of_locations; ++i) { components[scc.get_component_id(i)].push_back(i); } } @@ -147,7 +149,7 @@ template class RoundTripPlugin final : public BasePlugin void ComputeRoute(const PhantomNodeArray & phantom_node_vector, const RouteParameters & route_parameters, - std::vector & trip, + const std::vector & trip, InternalRouteResult & min_route) { // given he final trip, compute total distance and return the route and location permutation PhantomNodes viapoint; @@ -165,7 +167,7 @@ template class RoundTripPlugin final : public BasePlugin void ComputeRoute(const PhantomNodeArray & phantom_node_vector, const RouteParameters & route_parameters, - std::vector> & trip, + std::vector> & trip, std::vector & route) { for (const auto & curr_trip : trip) { InternalRouteResult curr_route; @@ -193,10 +195,12 @@ template class RoundTripPlugin final : public BasePlugin return 400; } - + const constexpr std::size_t BF_MAX_FEASABLE = 10; + auto number_of_locations = phantom_node_vector.size(); BOOST_ASSERT_MSG(result_table->size() > 0, "Distance Table is empty."); + //check if locations are in different strongly connected components (SCC) - const auto maxint = std::numeric_limits::max(); + const auto maxint = INVALID_EDGE_WEIGHT; if (*std::max_element(std::begin(*result_table), std::end(*result_table)) == maxint) { //TODO DELETE @@ -207,45 +211,43 @@ template class RoundTripPlugin final : public BasePlugin TIMER_START(tsp); // Compute all SCC - std::vector> components; + std::vector> components; SplitUnaccessibleLocations(phantom_node_vector, *result_table, components); - // std::vector> res_route (components.size()-1); - std::vector> res_route; - const constexpr std::size_t BF_MAX_FEASABLE = 14; + // std::vector> res_route (components.size()-1); + std::vector> res_route; //run TSP computation for every SCC for(auto k = 0; k < components.size(); ++k) { if (components[k].size() > 1) { - std::vector scc_route; + std::vector scc_route; scc_route.reserve(components[k].size()); // Compute the TSP with the given algorithm if (route_parameters.tsp_algo == "BF" && route_parameters.coordinates.size() < BF_MAX_FEASABLE) { SimpleLogger().Write() << "Running SCC BF"; - osrm::tsp::BruteForceTSP(components[k], phantom_node_vector, *result_table, scc_route); + osrm::tsp::BruteForceTSP(components[k], number_of_locations, *result_table, scc_route); res_route.push_back(scc_route); } else if (route_parameters.tsp_algo == "NN") { SimpleLogger().Write() << "Running SCC NN"; - osrm::tsp::NearestNeighbourTSP(components[k], phantom_node_vector, *result_table, scc_route); + osrm::tsp::NearestNeighbourTSP(components[k], number_of_locations, *result_table, scc_route); res_route.push_back(scc_route); } else if (route_parameters.tsp_algo == "FI") { SimpleLogger().Write() << "Running SCC FI"; - osrm::tsp::FarthestInsertionTSP(components[k], phantom_node_vector, *result_table, scc_route); + osrm::tsp::FarthestInsertionTSP(components[k], number_of_locations, *result_table, scc_route); res_route.push_back(scc_route); } else{ SimpleLogger().Write() << "Running SCC FI"; - osrm::tsp::FarthestInsertionTSP(components[k], phantom_node_vector, *result_table, scc_route); + osrm::tsp::FarthestInsertionTSP(components[k], number_of_locations, *result_table, scc_route); res_route.push_back(scc_route); } } } - SimpleLogger().Write() << "DONE"; std::vector route; ComputeRoute(phantom_node_vector, route_parameters, res_route, route); TIMER_STOP(tsp); SetRuntimeOutput(TIMER_MSEC(tsp), json_result); - + SimpleLogger().Write() << "Computed roundtrip in " << TIMER_MSEC(tsp) << "ms"; // SimpleLogger().Write() << "Route is"; // for (auto x : res_route) { // for (auto y : x) @@ -260,8 +262,7 @@ template class RoundTripPlugin final : public BasePlugin } SetDistanceOutput(dist, json_result); } else { //run TSP computation for all locations - auto number_of_locations = phantom_node_vector.size(); - std::vector res_route; + std::vector res_route; res_route.reserve(number_of_locations); // Compute the TSP with the given algorithm @@ -269,22 +270,21 @@ template class RoundTripPlugin final : public BasePlugin // TODO patrick nach userfreundlichkeit fragen, BF vs bf usw if (route_parameters.tsp_algo == "BF" && route_parameters.coordinates.size() < BF_MAX_FEASABLE) { SimpleLogger().Write() << "Running BF"; - res_route = osrm::tsp::BruteForceTSP(phantom_node_vector, *result_table, res_route); + osrm::tsp::BruteForceTSP(number_of_locations, *result_table, res_route); } else if (route_parameters.tsp_algo == "NN") { SimpleLogger().Write() << "Running NN"; - osrm::tsp::NearestNeighbourTSP(phantom_node_vector, *result_table, res_route); + osrm::tsp::NearestNeighbourTSP(number_of_locations, *result_table, res_route); } else if (route_parameters.tsp_algo == "FI") { SimpleLogger().Write() << "Running FI"; - osrm::tsp::FarthestInsertionTSP(phantom_node_vector, *result_table, res_route); + osrm::tsp::FarthestInsertionTSP(number_of_locations, *result_table, res_route); } else { SimpleLogger().Write() << "Running FI"; - osrm::tsp::FarthestInsertionTSP(phantom_node_vector, *result_table, res_route); - // osrm::tsp::NearestNeighbourTSP(phantom_node_vector, *result_table, res_route); + osrm::tsp::FarthestInsertionTSP(number_of_locations, *result_table, res_route); } // TODO asserts numer of result blablabla size // TODO std::is_permutation // TODO boost range - SimpleLogger().Write() << "DONE"; + InternalRouteResult min_route; @@ -299,6 +299,7 @@ template class RoundTripPlugin final : public BasePlugin //TODO TIMER im LOGGER SetRuntimeOutput(TIMER_MSEC(tsp), json_result); + SimpleLogger().Write() << "Computed roundtrip in " << TIMER_MSEC(tsp) << "ms"; SetLocPermutationOutput(res_route, json_result); //TODO MEHR ASSERTIONS! :O SetDistanceOutput(min_route.shortest_path_length, json_result); diff --git a/server/api_grammar.hpp b/server/api_grammar.hpp index c93518a34b1..8013b82435c 100644 --- a/server/api_grammar.hpp +++ b/server/api_grammar.hpp @@ -99,7 +99,6 @@ template struct APIGrammar : qi::grammar service, zoom, output, string, jsonp, checksum, location, hint, timestamp, stringwithDot, stringwithPercent, language, instruction, geometry, cmp, alt_route, u, uturns, old_API, num_results, matching_beta, gps_precision, classify, locs, stringforPolyline, tsp_algo; - uturns, old_API, num_results, matching_beta, gps_precision, classify, tsp_algo; HandlerT *handler; }; From 7587e97d46f8f31e8de09a88f298be4baa571909 Mon Sep 17 00:00:00 2001 From: Huyen Chau Nguyen Date: Wed, 19 Aug 2015 10:25:32 +0200 Subject: [PATCH 075/122] use typedefs from typedefs.h return roundtrip result as a return parameter and not as an input parameter --- plugins/round_trip.hpp | 39 ++++++------ routing_algorithms/tsp_brute_force.hpp | 41 +++++++----- routing_algorithms/tsp_farthest_insertion.hpp | 63 ++++++++++--------- routing_algorithms/tsp_nearest_neighbour.hpp | 33 +++++----- 4 files changed, 94 insertions(+), 82 deletions(-) diff --git a/plugins/round_trip.hpp b/plugins/round_trip.hpp index a1a955f9f21..14f5b4bfa6f 100644 --- a/plugins/round_trip.hpp +++ b/plugins/round_trip.hpp @@ -103,12 +103,11 @@ template class RoundTripPlugin final : public BasePlugin } } - void SplitUnaccessibleLocations(PhantomNodeArray & phantom_node_vector, + void SplitUnaccessibleLocations(const std::size_t number_of_locations, std::vector & result_table, std::vector> & components) { // Run TarjanSCC - const auto number_of_locations = phantom_node_vector.size(); auto wrapper = std::make_shared>(result_table, number_of_locations); // auto empty_restriction = RestrictionMap(std::vector()); // std::vector empty_vector; @@ -212,7 +211,7 @@ template class RoundTripPlugin final : public BasePlugin TIMER_START(tsp); // Compute all SCC std::vector> components; - SplitUnaccessibleLocations(phantom_node_vector, *result_table, components); + SplitUnaccessibleLocations(number_of_locations, *result_table, components); // std::vector> res_route (components.size()-1); std::vector> res_route; @@ -221,24 +220,23 @@ template class RoundTripPlugin final : public BasePlugin for(auto k = 0; k < components.size(); ++k) { if (components[k].size() > 1) { std::vector scc_route; - scc_route.reserve(components[k].size()); // Compute the TSP with the given algorithm if (route_parameters.tsp_algo == "BF" && route_parameters.coordinates.size() < BF_MAX_FEASABLE) { - SimpleLogger().Write() << "Running SCC BF"; - osrm::tsp::BruteForceTSP(components[k], number_of_locations, *result_table, scc_route); + SimpleLogger().Write() << "Running brute force on multiple SCC"; + scc_route = osrm::tsp::BruteForceTSP(components[k], number_of_locations, *result_table); res_route.push_back(scc_route); } else if (route_parameters.tsp_algo == "NN") { - SimpleLogger().Write() << "Running SCC NN"; - osrm::tsp::NearestNeighbourTSP(components[k], number_of_locations, *result_table, scc_route); + SimpleLogger().Write() << "Running nearest neighbour on multiple SCC"; + scc_route = osrm::tsp::NearestNeighbourTSP(components[k], number_of_locations, *result_table); res_route.push_back(scc_route); } else if (route_parameters.tsp_algo == "FI") { - SimpleLogger().Write() << "Running SCC FI"; - osrm::tsp::FarthestInsertionTSP(components[k], number_of_locations, *result_table, scc_route); + SimpleLogger().Write() << "Running farthest insertion on multiple SCC"; + scc_route = osrm::tsp::FarthestInsertionTSP(components[k], number_of_locations, *result_table); res_route.push_back(scc_route); } else{ - SimpleLogger().Write() << "Running SCC FI"; - osrm::tsp::FarthestInsertionTSP(components[k], number_of_locations, *result_table, scc_route); + SimpleLogger().Write() << "Running farthest insertion on multiple SCC"; + scc_route = osrm::tsp::FarthestInsertionTSP(components[k], number_of_locations, *result_table); res_route.push_back(scc_route); } } @@ -263,23 +261,22 @@ template class RoundTripPlugin final : public BasePlugin SetDistanceOutput(dist, json_result); } else { //run TSP computation for all locations std::vector res_route; - res_route.reserve(number_of_locations); // Compute the TSP with the given algorithm TIMER_START(tsp); // TODO patrick nach userfreundlichkeit fragen, BF vs bf usw if (route_parameters.tsp_algo == "BF" && route_parameters.coordinates.size() < BF_MAX_FEASABLE) { - SimpleLogger().Write() << "Running BF"; - osrm::tsp::BruteForceTSP(number_of_locations, *result_table, res_route); + SimpleLogger().Write() << "Running brute force"; + res_route = osrm::tsp::BruteForceTSP(number_of_locations, *result_table); } else if (route_parameters.tsp_algo == "NN") { - SimpleLogger().Write() << "Running NN"; - osrm::tsp::NearestNeighbourTSP(number_of_locations, *result_table, res_route); + SimpleLogger().Write() << "Running nearest neighbour"; + res_route = osrm::tsp::NearestNeighbourTSP(number_of_locations, *result_table); } else if (route_parameters.tsp_algo == "FI") { - SimpleLogger().Write() << "Running FI"; - osrm::tsp::FarthestInsertionTSP(number_of_locations, *result_table, res_route); + SimpleLogger().Write() << "Running farthest insertion"; + res_route = osrm::tsp::FarthestInsertionTSP(number_of_locations, *result_table); } else { - SimpleLogger().Write() << "Running FI"; - osrm::tsp::FarthestInsertionTSP(number_of_locations, *result_table, res_route); + SimpleLogger().Write() << "Running farthest insertion"; + res_route = osrm::tsp::FarthestInsertionTSP(number_of_locations, *result_table); } // TODO asserts numer of result blablabla size // TODO std::is_permutation diff --git a/routing_algorithms/tsp_brute_force.hpp b/routing_algorithms/tsp_brute_force.hpp index bb83aa888dd..2665d68f2c7 100644 --- a/routing_algorithms/tsp_brute_force.hpp +++ b/routing_algorithms/tsp_brute_force.hpp @@ -54,11 +54,11 @@ namespace tsp template int ReturnDistance(const std::vector & dist_table, const std::vector & location_order, - const int min_route_dist, + const EdgeWeight min_route_dist, const int number_of_locations) { int route_dist = 0; int i = 0; - while (i < location_order.size() - 1 && route_dist < min_route_dist) { + while (i < location_order.size() - 1) { route_dist += *(dist_table.begin() + (location_order[i] * number_of_locations) + location_order[i+1]); ++i; } @@ -67,42 +67,49 @@ int ReturnDistance(const std::vector & dist_table, return route_dist; } -void BruteForceTSP(std::vector & component, - const PhantomNodeArray & phantom_node_vector, - const std::vector & dist_table, - std::vector & route) { +std::vector BruteForceTSP(std::vector & component, + const std::size_t number_of_locations, + const std::vector & dist_table) { - const unsigned component_size = component.size(); - unsigned min_route_dist = std::numeric_limits::max(); + std::vector route; + route.reserve(number_of_locations); + + + EdgeWeight min_route_dist = INVALID_EDGE_WEIGHT; // check length of all possible permutation of the component ids do { - const auto new_distance = ReturnDistance(dist_table, component, min_route_dist, component_size); - if (new_distance < min_route_dist) { + const auto new_distance = ReturnDistance(dist_table, component, min_route_dist, number_of_locations); + if (new_distance <= min_route_dist) { min_route_dist = new_distance; route = component; } } while(std::next_permutation(component.begin(), component.end())); + + return route; } -void BruteForceTSP(const PhantomNodeArray & phantom_node_vector, - const std::vector & dist_table, - std::vector & route) { - const auto number_of_locations = phantom_node_vector.size(); +std::vector BruteForceTSP(const std::size_t number_of_locations, + const std::vector & dist_table) { + std::vector route; + route.reserve(number_of_locations); + // fill a vector with node ids - std::vector location_ids(number_of_locations); + std::vector location_ids(number_of_locations); std::iota(location_ids.begin(), location_ids.end(), 0); - unsigned min_route_dist = std::numeric_limits::max(); + EdgeWeight min_route_dist = INVALID_EDGE_WEIGHT; // check length of all possible permutation of the location ids do { const auto new_distance = ReturnDistance(dist_table, location_ids, min_route_dist, number_of_locations); - if (new_distance < min_route_dist) { + if (new_distance <= min_route_dist) { min_route_dist = new_distance; route = location_ids; } } while(std::next_permutation(location_ids.begin(), location_ids.end())); + + return route; } } diff --git a/routing_algorithms/tsp_farthest_insertion.hpp b/routing_algorithms/tsp_farthest_insertion.hpp index 94755306fa1..5f53447e753 100644 --- a/routing_algorithms/tsp_farthest_insertion.hpp +++ b/routing_algorithms/tsp_farthest_insertion.hpp @@ -52,12 +52,12 @@ namespace tsp void GetShortestRoundTrip(const int current_loc, const std::vector & dist_table, const int number_of_locations, - std::vector & current_trip, + std::vector & route, int & min_trip_distance, - std::vector::iterator & next_insert_point_candidate){ + std::vector::iterator & next_insert_point_candidate){ // for all nodes in the current trip find the best insertion resulting in the shortest path - // assert min 2 nodes in current_trip - for (auto from_node = current_trip.begin(); from_node != std::prev(current_trip.end()); ++from_node) { + // assert min 2 nodes in route + for (auto from_node = route.begin(); from_node != std::prev(route.end()); ++from_node) { const auto to_node = std::next(from_node); const auto dist_from = *(dist_table.begin() + (*from_node * number_of_locations) + current_loc); @@ -71,8 +71,8 @@ void GetShortestRoundTrip(const int current_loc, } } // check insertion between last and first location too - auto from_node = std::prev(current_trip.end()); - auto to_node = current_trip.begin(); + auto from_node = std::prev(route.end()); + auto to_node = route.begin(); auto dist_from = *(dist_table.begin() + (*from_node * number_of_locations) + current_loc); auto dist_to = *(dist_table.begin() + (current_loc * number_of_locations) + *to_node); @@ -84,10 +84,9 @@ void GetShortestRoundTrip(const int current_loc, } // osrm::tsp::FarthestInsertionTSP(components[k], phantom_node_vector, *result_table, scc_route); -void FarthestInsertionTSP(const std::vector & locations, - const PhantomNodeArray & phantom_node_vector, - const std::vector & dist_table, - std::vector & current_trip) { +std::vector FarthestInsertionTSP(const std::vector & locations, + const std::size_t number_of_locations, + const std::vector & dist_table) { ////////////////////////////////////////////////////////////////////////////////////////////////// // START FARTHEST INSERTION HERE // 1. start at a random round trip of 2 locations @@ -96,7 +95,10 @@ void FarthestInsertionTSP(const std::vector & locations, // 4. repeat 2-3 until all locations are visited // 5. DONE! ////////////////////////////////////////////////////////////////////////////////////////////////// - const int number_of_locations = phantom_node_vector.size(); + + std::vector route; + route.reserve(number_of_locations); + const int size_of_component = locations.size(); // tracks which nodes have been already visited std::vector visited(number_of_locations, false); @@ -119,24 +121,24 @@ void FarthestInsertionTSP(const std::vector & locations, visited[max_from] = true; visited[max_to] = true; - current_trip.push_back(max_from); - current_trip.push_back(max_to); + route.push_back(max_from); + route.push_back(max_to); // SimpleLogger().Write() << size_of_component; // add all other nodes missing (two nodes are already in the initial start trip) for (int j = 2; j < size_of_component; ++j) { // SimpleLogger().Write() << j << "/" << size_of_component; auto farthest_distance = 0; auto next_node = -1; - std::vector::iterator next_insert_point; + std::vector::iterator next_insert_point; // find unvisited loc i that is the farthest away from all other visited locs for (auto i : locations) { // find the shortest distance from i to all visited nodes if (!visited[i]) { - auto min_trip_distance = std::numeric_limits::max(); - std::vector::iterator next_insert_point_candidate; + auto min_trip_distance = INVALID_EDGE_WEIGHT; + std::vector::iterator next_insert_point_candidate; - GetShortestRoundTrip(i, dist_table, number_of_locations, current_trip, min_trip_distance, next_insert_point_candidate); + GetShortestRoundTrip(i, dist_table, number_of_locations, route, min_trip_distance, next_insert_point_candidate); // add the location to the current trip such that it results in the shortest total tour // SimpleLogger().Write() << "min_trip_distance " << min_trip_distance; @@ -150,13 +152,13 @@ void FarthestInsertionTSP(const std::vector & locations, // SimpleLogger().Write() << "next node " << next_node; // mark as visited and insert node visited[next_node] = true; - current_trip.insert(next_insert_point, next_node); + route.insert(next_insert_point, next_node); } + return route; } -void FarthestInsertionTSP(const PhantomNodeArray & phantom_node_vector, - const std::vector & dist_table, - std::vector & current_trip) { +std::vector FarthestInsertionTSP(const std::size_t number_of_locations, + const std::vector & dist_table) { ////////////////////////////////////////////////////////////////////////////////////////////////// // START FARTHEST INSERTION HERE // 1. start at a random round trip of 2 locations @@ -166,7 +168,9 @@ void FarthestInsertionTSP(const PhantomNodeArray & phantom_node_vector, // 5. DONE! ////////////////////////////////////////////////////////////////////////////////////////////////// - const auto number_of_locations = phantom_node_vector.size(); + std::vector route; + route.reserve(number_of_locations); + // tracks which nodes have been already visited std::vector visited(number_of_locations, false); @@ -176,23 +180,23 @@ void FarthestInsertionTSP(const PhantomNodeArray & phantom_node_vector, const int max_to = index % number_of_locations; visited[max_from] = true; visited[max_to] = true; - current_trip.push_back(max_from); - current_trip.push_back(max_to); + route.push_back(max_from); + route.push_back(max_to); // add all other nodes missing (two nodes are already in the initial start trip) for (int j = 2; j < number_of_locations; ++j) { auto farthest_distance = 0; auto next_node = -1; //todo move out of loop and overwrite - std::vector::iterator next_insert_point; + std::vector::iterator next_insert_point; // find unvisited loc i that is the farthest away from all other visited locs for (int i = 0; i < number_of_locations; ++i) { if (!visited[i]) { - auto min_trip_distance = std::numeric_limits::max(); - std::vector::iterator next_insert_point_candidate; + auto min_trip_distance = INVALID_EDGE_WEIGHT; + std::vector::iterator next_insert_point_candidate; - GetShortestRoundTrip(i, dist_table, number_of_locations, current_trip, min_trip_distance, next_insert_point_candidate); + GetShortestRoundTrip(i, dist_table, number_of_locations, route, min_trip_distance, next_insert_point_candidate); // add the location to the current trip such that it results in the shortest total tour if (min_trip_distance >= farthest_distance) { @@ -205,8 +209,9 @@ void FarthestInsertionTSP(const PhantomNodeArray & phantom_node_vector, // mark as visited and insert node visited[next_node] = true; - current_trip.insert(next_insert_point, next_node); + route.insert(next_insert_point, next_node); } + return route; } diff --git a/routing_algorithms/tsp_nearest_neighbour.hpp b/routing_algorithms/tsp_nearest_neighbour.hpp index 8c182f7792f..e7820c506b7 100644 --- a/routing_algorithms/tsp_nearest_neighbour.hpp +++ b/routing_algorithms/tsp_nearest_neighbour.hpp @@ -48,10 +48,9 @@ namespace osrm namespace tsp { -void NearestNeighbourTSP(const std::vector & locations, - const PhantomNodeArray & phantom_node_vector, - const std::vector & dist_table, - std::vector & route) { +std::vector NearestNeighbourTSP(const std::vector & locations, + const std::size_t number_of_locations, + const std::vector & dist_table) { ////////////////////////////////////////////////////////////////////////////////////////////////// // START GREEDY NEAREST NEIGHBOUR HERE // 1. grab a random location and mark as starting point @@ -62,17 +61,18 @@ void NearestNeighbourTSP(const std::vector & locations, // 6. repeat 1-5 with different starting points and choose iteration with shortest trip // 7. DONE! ////////////////////////////////////////////////////////////////////////////////////////////////// + std::vector route; + route.reserve(number_of_locations); - const auto number_of_locations = phantom_node_vector.size(); const int component_size = locations.size(); - int shortest_trip_distance = std::numeric_limits::max(); + int shortest_trip_distance = INVALID_EDGE_WEIGHT; // ALWAYS START AT ANOTHER STARTING POINT for(auto start_node : locations) { int curr_node = start_node; - std::vector curr_route; + std::vector curr_route; curr_route.reserve(component_size); curr_route.push_back(start_node); @@ -84,7 +84,7 @@ void NearestNeighbourTSP(const std::vector & locations, int trip_dist = 0; for(int via_point = 1; via_point < component_size; ++via_point) { - int min_dist = std::numeric_limits::max(); + int min_dist = INVALID_EDGE_WEIGHT; int min_id = -1; // 2. FIND NEAREST NEIGHBOUR @@ -107,11 +107,11 @@ void NearestNeighbourTSP(const std::vector & locations, route = curr_route; } } + return route; } -void NearestNeighbourTSP(const PhantomNodeArray & phantom_node_vector, - const std::vector & dist_table, - std::vector & route) { +std::vector NearestNeighbourTSP(const std::size_t number_of_locations, + const std::vector & dist_table) { ////////////////////////////////////////////////////////////////////////////////////////////////// // START GREEDY NEAREST NEIGHBOUR HERE // 1. grab a random location and mark as starting point @@ -123,15 +123,17 @@ void NearestNeighbourTSP(const PhantomNodeArray & phantom_node_vector, // 7. DONE! ////////////////////////////////////////////////////////////////////////////////////////////////// - const auto number_of_locations = phantom_node_vector.size(); - int shortest_trip_distance = std::numeric_limits::max(); + std::vector route; + route.reserve(number_of_locations); + + int shortest_trip_distance = INVALID_EDGE_WEIGHT; // ALWAYS START AT ANOTHER STARTING POINT for(int start_node = 0; start_node < number_of_locations; ++start_node) { int curr_node = start_node; - std::vector curr_route; + std::vector curr_route; curr_route.reserve(number_of_locations); curr_route.push_back(start_node); @@ -143,7 +145,7 @@ void NearestNeighbourTSP(const PhantomNodeArray & phantom_node_vector, int trip_dist = 0; for(int via_point = 1; via_point < number_of_locations; ++via_point) { - int min_dist = std::numeric_limits::max(); + int min_dist = INVALID_EDGE_WEIGHT; int min_id = -1; // 2. FIND NEAREST NEIGHBOUR @@ -169,6 +171,7 @@ void NearestNeighbourTSP(const PhantomNodeArray & phantom_node_vector, route = curr_route; } } + return route; } } From 99cf3219d43faf050d43d6dc27d9643082f8fa9d Mon Sep 17 00:00:00 2001 From: Huyen Chau Nguyen Date: Wed, 19 Aug 2015 11:35:36 +0200 Subject: [PATCH 076/122] have less redundant code for requests with one or multiple SCCs --- plugins/round_trip.hpp | 148 +++++-------- routing_algorithms/tsp_brute_force.hpp | 27 +-- routing_algorithms/tsp_farthest_insertion.hpp | 207 ++++++++++-------- routing_algorithms/tsp_nearest_neighbour.hpp | 64 ------ 4 files changed, 166 insertions(+), 280 deletions(-) diff --git a/plugins/round_trip.hpp b/plugins/round_trip.hpp index 14f5b4bfa6f..e1d95e72cd6 100644 --- a/plugins/round_trip.hpp +++ b/plugins/round_trip.hpp @@ -197,112 +197,66 @@ template class RoundTripPlugin final : public BasePlugin const constexpr std::size_t BF_MAX_FEASABLE = 10; auto number_of_locations = phantom_node_vector.size(); BOOST_ASSERT_MSG(result_table->size() > 0, "Distance Table is empty."); + std::vector> components; //check if locations are in different strongly connected components (SCC) const auto maxint = INVALID_EDGE_WEIGHT; if (*std::max_element(std::begin(*result_table), std::end(*result_table)) == maxint) { - - //TODO DELETE - // JSON output related objects - std::unique_ptr> descriptor; - descriptor = osrm::make_unique>(facade); - descriptor->SetConfig(route_parameters); - - TIMER_START(tsp); // Compute all SCC - std::vector> components; SplitUnaccessibleLocations(number_of_locations, *result_table, components); - // std::vector> res_route (components.size()-1); - std::vector> res_route; - - - //run TSP computation for every SCC - for(auto k = 0; k < components.size(); ++k) { - if (components[k].size() > 1) { - std::vector scc_route; - - // Compute the TSP with the given algorithm - if (route_parameters.tsp_algo == "BF" && route_parameters.coordinates.size() < BF_MAX_FEASABLE) { - SimpleLogger().Write() << "Running brute force on multiple SCC"; - scc_route = osrm::tsp::BruteForceTSP(components[k], number_of_locations, *result_table); - res_route.push_back(scc_route); - } else if (route_parameters.tsp_algo == "NN") { - SimpleLogger().Write() << "Running nearest neighbour on multiple SCC"; - scc_route = osrm::tsp::NearestNeighbourTSP(components[k], number_of_locations, *result_table); - res_route.push_back(scc_route); - } else if (route_parameters.tsp_algo == "FI") { - SimpleLogger().Write() << "Running farthest insertion on multiple SCC"; - scc_route = osrm::tsp::FarthestInsertionTSP(components[k], number_of_locations, *result_table); - res_route.push_back(scc_route); - } else{ - SimpleLogger().Write() << "Running farthest insertion on multiple SCC"; - scc_route = osrm::tsp::FarthestInsertionTSP(components[k], number_of_locations, *result_table); - res_route.push_back(scc_route); - } + } else { + // fill a vector with node ids + std::vector location_ids(number_of_locations); + std::iota(location_ids.begin(), location_ids.end(), 0); + components.push_back(location_ids); + + } + + std::vector> res_route; + TIMER_START(tsp); + //run TSP computation for every SCC + for(auto k = 0; k < components.size(); ++k) { + if (components[k].size() > 1) { + std::vector scc_route; + + // Compute the TSP with the given algorithm + if (route_parameters.tsp_algo == "BF" && route_parameters.coordinates.size() < BF_MAX_FEASABLE) { + SimpleLogger().Write() << "Running brute force"; + scc_route = osrm::tsp::BruteForceTSP(components[k], number_of_locations, *result_table); + res_route.push_back(scc_route); + } else if (route_parameters.tsp_algo == "NN") { + SimpleLogger().Write() << "Running nearest neighbour"; + scc_route = osrm::tsp::NearestNeighbourTSP(components[k], number_of_locations, *result_table); + res_route.push_back(scc_route); + } else if (route_parameters.tsp_algo == "FI") { + SimpleLogger().Write() << "Running farthest insertion"; + scc_route = osrm::tsp::FarthestInsertionTSP(components[k], number_of_locations, *result_table); + res_route.push_back(scc_route); + } else{ + SimpleLogger().Write() << "Running farthest insertion"; + scc_route = osrm::tsp::FarthestInsertionTSP(components[k], number_of_locations, *result_table); + res_route.push_back(scc_route); } } - std::vector route; - ComputeRoute(phantom_node_vector, route_parameters, res_route, route); - TIMER_STOP(tsp); - SetRuntimeOutput(TIMER_MSEC(tsp), json_result); - SimpleLogger().Write() << "Computed roundtrip in " << TIMER_MSEC(tsp) << "ms"; - // SimpleLogger().Write() << "Route is"; - // for (auto x : res_route) { - // for (auto y : x) - // std::cout << y << " "; - // } - // SimpleLogger().Write() << ""; - - auto dist = 0; - for (auto curr_route : route) { - dist += curr_route.shortest_path_length; - SetGeometry(route_parameters, curr_route, json_result); - } - SetDistanceOutput(dist, json_result); - } else { //run TSP computation for all locations - std::vector res_route; - - // Compute the TSP with the given algorithm - TIMER_START(tsp); - // TODO patrick nach userfreundlichkeit fragen, BF vs bf usw - if (route_parameters.tsp_algo == "BF" && route_parameters.coordinates.size() < BF_MAX_FEASABLE) { - SimpleLogger().Write() << "Running brute force"; - res_route = osrm::tsp::BruteForceTSP(number_of_locations, *result_table); - } else if (route_parameters.tsp_algo == "NN") { - SimpleLogger().Write() << "Running nearest neighbour"; - res_route = osrm::tsp::NearestNeighbourTSP(number_of_locations, *result_table); - } else if (route_parameters.tsp_algo == "FI") { - SimpleLogger().Write() << "Running farthest insertion"; - res_route = osrm::tsp::FarthestInsertionTSP(number_of_locations, *result_table); - } else { - SimpleLogger().Write() << "Running farthest insertion"; - res_route = osrm::tsp::FarthestInsertionTSP(number_of_locations, *result_table); - } - // TODO asserts numer of result blablabla size - // TODO std::is_permutation - // TODO boost range - - - - InternalRouteResult min_route; - ComputeRoute(phantom_node_vector, route_parameters, res_route, min_route); - TIMER_STOP(tsp); - - // SimpleLogger().Write() << "Route is"; - // for (auto x : res_route) { - // std::cout << x << " "; - // } - // SimpleLogger().Write() << ""; - - //TODO TIMER im LOGGER - SetRuntimeOutput(TIMER_MSEC(tsp), json_result); - SimpleLogger().Write() << "Computed roundtrip in " << TIMER_MSEC(tsp) << "ms"; - SetLocPermutationOutput(res_route, json_result); - //TODO MEHR ASSERTIONS! :O - SetDistanceOutput(min_route.shortest_path_length, json_result); - SetGeometry(route_parameters, min_route, json_result); - BOOST_ASSERT(min_route.segment_end_coordinates.size() == route_parameters.coordinates.size()); } + std::vector route; + ComputeRoute(phantom_node_vector, route_parameters, res_route, route); + TIMER_STOP(tsp); + SetRuntimeOutput(TIMER_MSEC(tsp), json_result); + SimpleLogger().Write() << "Computed roundtrip in " << TIMER_MSEC(tsp) << "ms"; + // SimpleLogger().Write() << "Route is"; + // for (auto x : res_route) { + // for (auto y : x) + // std::cout << y << " "; + // } + // SimpleLogger().Write() << ""; + + auto dist = 0; + for (auto curr_route : route) { + dist += curr_route.shortest_path_length; + SetGeometry(route_parameters, curr_route, json_result); + } + SetDistanceOutput(dist, json_result); diff --git a/routing_algorithms/tsp_brute_force.hpp b/routing_algorithms/tsp_brute_force.hpp index 2665d68f2c7..022804e5a61 100644 --- a/routing_algorithms/tsp_brute_force.hpp +++ b/routing_algorithms/tsp_brute_force.hpp @@ -89,29 +89,6 @@ std::vector BruteForceTSP(std::vector & component, return route; } -std::vector BruteForceTSP(const std::size_t number_of_locations, - const std::vector & dist_table) { - std::vector route; - route.reserve(number_of_locations); - - // fill a vector with node ids - std::vector location_ids(number_of_locations); - std::iota(location_ids.begin(), location_ids.end(), 0); - - EdgeWeight min_route_dist = INVALID_EDGE_WEIGHT; - // check length of all possible permutation of the location ids - do { - const auto new_distance = ReturnDistance(dist_table, location_ids, min_route_dist, number_of_locations); - - if (new_distance <= min_route_dist) { - min_route_dist = new_distance; - route = location_ids; - } - } while(std::next_permutation(location_ids.begin(), location_ids.end())); - - return route; -} - -} -} +} //end namespace osrm +} //end namespace tsp #endif // TSP_BRUTE_FORCE_HPP \ No newline at end of file diff --git a/routing_algorithms/tsp_farthest_insertion.hpp b/routing_algorithms/tsp_farthest_insertion.hpp index 5f53447e753..03cd451ca77 100644 --- a/routing_algorithms/tsp_farthest_insertion.hpp +++ b/routing_algorithms/tsp_farthest_insertion.hpp @@ -49,12 +49,16 @@ namespace osrm namespace tsp { -void GetShortestRoundTrip(const int current_loc, - const std::vector & dist_table, - const int number_of_locations, - std::vector & route, - int & min_trip_distance, - std::vector::iterator & next_insert_point_candidate){ +using NodeIterator = typename std::vector::iterator; + +std::pair GetShortestRoundTrip(const int current_loc, + const std::vector & dist_table, + const int number_of_locations, + std::vector & route){ + + auto min_trip_distance = INVALID_EDGE_WEIGHT; + NodeIterator next_insert_point_candidate; + // for all nodes in the current trip find the best insertion resulting in the shortest path // assert min 2 nodes in route for (auto from_node = route.begin(); from_node != std::prev(route.end()); ++from_node) { @@ -81,75 +85,50 @@ void GetShortestRoundTrip(const int current_loc, min_trip_distance = trip_dist; next_insert_point_candidate = to_node; } -} -// osrm::tsp::FarthestInsertionTSP(components[k], phantom_node_vector, *result_table, scc_route); -std::vector FarthestInsertionTSP(const std::vector & locations, - const std::size_t number_of_locations, - const std::vector & dist_table) { - ////////////////////////////////////////////////////////////////////////////////////////////////// - // START FARTHEST INSERTION HERE - // 1. start at a random round trip of 2 locations - // 2. find the location that is the farthest away from the visited locations and whose insertion will make the round trip the longest - // 3. add the found location to the current round trip such that round trip is the shortest - // 4. repeat 2-3 until all locations are visited - // 5. DONE! - ////////////////////////////////////////////////////////////////////////////////////////////////// + return std::make_pair(min_trip_distance, next_insert_point_candidate); +} +// given two initial start nodes, find a roundtrip route using the farthest insertion algorithm +std::vector FindRoute(const std::size_t & number_of_locations, + const std::size_t & size_of_component, + const std::vector & locations, + const std::vector & dist_table, + const NodeID & start1, + const NodeID & start2) { std::vector route; route.reserve(number_of_locations); - const int size_of_component = locations.size(); // tracks which nodes have been already visited std::vector visited(number_of_locations, false); - auto max_dist = 0; - auto max_from = -1; - auto max_to = -1; - - //TODO - for (auto x : locations) { - for (auto y : locations) { - auto xy_dist = *(dist_table.begin() + x * number_of_locations + y); - if (xy_dist > max_dist) { - max_dist = xy_dist; - max_from = x; - max_to = y; - } - } - } + visited[start1] = true; + visited[start2] = true; + route.push_back(start1); + route.push_back(start2); - visited[max_from] = true; - visited[max_to] = true; - route.push_back(max_from); - route.push_back(max_to); - // SimpleLogger().Write() << size_of_component; // add all other nodes missing (two nodes are already in the initial start trip) for (int j = 2; j < size_of_component; ++j) { - // SimpleLogger().Write() << j << "/" << size_of_component; + auto farthest_distance = 0; auto next_node = -1; - std::vector::iterator next_insert_point; + NodeIterator next_insert_point; // find unvisited loc i that is the farthest away from all other visited locs for (auto i : locations) { // find the shortest distance from i to all visited nodes if (!visited[i]) { - auto min_trip_distance = INVALID_EDGE_WEIGHT; - std::vector::iterator next_insert_point_candidate; - - GetShortestRoundTrip(i, dist_table, number_of_locations, route, min_trip_distance, next_insert_point_candidate); + auto insert_candidate = GetShortestRoundTrip(i, dist_table, number_of_locations, route); // add the location to the current trip such that it results in the shortest total tour - // SimpleLogger().Write() << "min_trip_distance " << min_trip_distance; - if (min_trip_distance >= farthest_distance) { - farthest_distance = min_trip_distance; + if (insert_candidate.first >= farthest_distance) { + farthest_distance = insert_candidate.first; next_node = i; - next_insert_point = next_insert_point_candidate; + next_insert_point = insert_candidate.second; } } } - // SimpleLogger().Write() << "next node " << next_node; + // mark as visited and insert node visited[next_node] = true; route.insert(next_insert_point, next_node); @@ -157,7 +136,9 @@ std::vector FarthestInsertionTSP(const std::vector & locations, return route; } -std::vector FarthestInsertionTSP(const std::size_t number_of_locations, +// osrm::tsp::FarthestInsertionTSP(components[k], phantom_node_vector, *result_table, scc_route); +std::vector FarthestInsertionTSP(const std::vector & locations, + const std::size_t number_of_locations, const std::vector & dist_table) { ////////////////////////////////////////////////////////////////////////////////////////////////// // START FARTHEST INSERTION HERE @@ -168,54 +149,92 @@ std::vector FarthestInsertionTSP(const std::size_t number_of_locations, // 5. DONE! ////////////////////////////////////////////////////////////////////////////////////////////////// - std::vector route; - route.reserve(number_of_locations); - - // tracks which nodes have been already visited - std::vector visited(number_of_locations, false); - - // find the pair of location with the biggest distance and make the pair the initial start trip - const auto index = std::distance(dist_table.begin(), std::max_element(dist_table.begin(), dist_table.end())); - const int max_from = index / number_of_locations; - const int max_to = index % number_of_locations; - visited[max_from] = true; - visited[max_to] = true; - route.push_back(max_from); - route.push_back(max_to); - - // add all other nodes missing (two nodes are already in the initial start trip) - for (int j = 2; j < number_of_locations; ++j) { - auto farthest_distance = 0; - auto next_node = -1; - //todo move out of loop and overwrite - std::vector::iterator next_insert_point; - - // find unvisited loc i that is the farthest away from all other visited locs - for (int i = 0; i < number_of_locations; ++i) { - if (!visited[i]) { - auto min_trip_distance = INVALID_EDGE_WEIGHT; - std::vector::iterator next_insert_point_candidate; - - GetShortestRoundTrip(i, dist_table, number_of_locations, route, min_trip_distance, next_insert_point_candidate); + const int size_of_component = locations.size(); + auto max_from = -1; + auto max_to = -1; - // add the location to the current trip such that it results in the shortest total tour - if (min_trip_distance >= farthest_distance) { - farthest_distance = min_trip_distance; - next_node = i; - next_insert_point = next_insert_point_candidate; + if (size_of_component == number_of_locations) { + // find the pair of location with the biggest distance and make the pair the initial start trip + const auto index = std::distance(dist_table.begin(), std::max_element(dist_table.begin(), dist_table.end())); + max_from = index / number_of_locations; + max_to = index % number_of_locations; + + } else { + auto max_dist = 0; + for (auto x : locations) { + for (auto y : locations) { + auto xy_dist = *(dist_table.begin() + x * number_of_locations + y); + if (xy_dist > max_dist) { + max_dist = xy_dist; + max_from = x; + max_to = y; } } } - - // mark as visited and insert node - visited[next_node] = true; - route.insert(next_insert_point, next_node); } - return route; -} - -} + return FindRoute(number_of_locations, size_of_component, locations, dist_table, max_from, max_to); } +// std::vector FarthestInsertionTSP(const std::size_t number_of_locations, +// const std::vector & dist_table) { +// ////////////////////////////////////////////////////////////////////////////////////////////////// +// // START FARTHEST INSERTION HERE +// // 1. start at a random round trip of 2 locations +// // 2. find the location that is the farthest away from the visited locations and whose insertion will make the round trip the longest +// // 3. add the found location to the current round trip such that round trip is the shortest +// // 4. repeat 2-3 until all locations are visited +// // 5. DONE! +// ////////////////////////////////////////////////////////////////////////////////////////////////// + +// std::vector route; +// route.reserve(number_of_locations); + +// // tracks which nodes have been already visited +// std::vector visited(number_of_locations, false); + +// // find the pair of location with the biggest distance and make the pair the initial start trip +// const auto index = std::distance(dist_table.begin(), std::max_element(dist_table.begin(), dist_table.end())); +// const int max_from = index / number_of_locations; +// const int max_to = index % number_of_locations; +// visited[max_from] = true; +// visited[max_to] = true; +// route.push_back(max_from); +// route.push_back(max_to); + +// // add all other nodes missing (two nodes are already in the initial start trip) +// for (int j = 2; j < number_of_locations; ++j) { +// auto farthest_distance = 0; +// auto next_node = -1; +// //todo move out of loop and overwrite +// NodeIterator next_insert_point; + +// // find unvisited loc i that is the farthest away from all other visited locs +// for (int i = 0; i < number_of_locations; ++i) { +// if (!visited[i]) { +// auto min_trip_distance = INVALID_EDGE_WEIGHT; +// NodeIterator next_insert_point_candidate; + +// GetShortestRoundTrip(i, dist_table, number_of_locations, route, min_trip_distance, next_insert_point_candidate); + +// // add the location to the current trip such that it results in the shortest total tour +// if (min_trip_distance >= farthest_distance) { +// farthest_distance = min_trip_distance; +// next_node = i; +// next_insert_point = next_insert_point_candidate; +// } +// } +// } + +// // mark as visited and insert node +// visited[next_node] = true; +// route.insert(next_insert_point, next_node); +// } +// return route; +// } + + +} //end namespace osrm +} //end namespace tsp + #endif // TSP_FARTHEST_INSERTION_HPP \ No newline at end of file diff --git a/routing_algorithms/tsp_nearest_neighbour.hpp b/routing_algorithms/tsp_nearest_neighbour.hpp index e7820c506b7..cd2adadec16 100644 --- a/routing_algorithms/tsp_nearest_neighbour.hpp +++ b/routing_algorithms/tsp_nearest_neighbour.hpp @@ -110,70 +110,6 @@ std::vector NearestNeighbourTSP(const std::vector & locations, return route; } -std::vector NearestNeighbourTSP(const std::size_t number_of_locations, - const std::vector & dist_table) { - ////////////////////////////////////////////////////////////////////////////////////////////////// - // START GREEDY NEAREST NEIGHBOUR HERE - // 1. grab a random location and mark as starting point - // 2. find the nearest unvisited neighbour, set it as the current location and mark as visited - // 3. repeat 2 until there is no unvisited location - // 4. return route back to starting point - // 5. compute route - // 6. repeat 1-5 with different starting points and choose iteration with shortest trip - // 7. DONE! - ////////////////////////////////////////////////////////////////////////////////////////////////// - - std::vector route; - route.reserve(number_of_locations); - - int shortest_trip_distance = INVALID_EDGE_WEIGHT; - - // ALWAYS START AT ANOTHER STARTING POINT - for(int start_node = 0; start_node < number_of_locations; ++start_node) - { - int curr_node = start_node; - - std::vector curr_route; - curr_route.reserve(number_of_locations); - curr_route.push_back(start_node); - - // visited[i] indicates whether node i was already visited by the salesman - std::vector visited(number_of_locations, false); - visited[start_node] = true; - - // 3. REPEAT FOR EVERY UNVISITED NODE - int trip_dist = 0; - for(int via_point = 1; via_point < number_of_locations; ++via_point) - { - int min_dist = INVALID_EDGE_WEIGHT; - int min_id = -1; - - // 2. FIND NEAREST NEIGHBOUR - auto row_begin_iterator = dist_table.begin() + (curr_node * number_of_locations); - auto row_end_iterator = dist_table.begin() + ((curr_node + 1) * number_of_locations); - for (auto it = row_begin_iterator; it != row_end_iterator; ++it) { - const auto index = std::distance(row_begin_iterator, it); - if (!visited[index] && *it < min_dist) - { - min_dist = *it; - min_id = index; - } - } - visited[min_id] = true; - curr_route.push_back(min_id); - trip_dist += min_dist; - curr_node = min_id; - } - - // check round trip with this starting point is shorter than the shortest round trip found till now - if (trip_dist < shortest_trip_distance) { - shortest_trip_distance = trip_dist; - route = curr_route; - } - } - return route; -} - } } #endif // TSP_NEAREST_NEIGHBOUR_HPP \ No newline at end of file From 78a8cf6982d512298bb9326c825131eca217e210 Mon Sep 17 00:00:00 2001 From: Huyen Chau Nguyen Date: Wed, 19 Aug 2015 14:04:59 +0200 Subject: [PATCH 077/122] add a wrapper for the distance table for better access --- data_structures/matrix_graph_wrapper.hpp | 23 +++-- plugins/round_trip.hpp | 48 ++++------ routing_algorithms/tsp_brute_force.hpp | 17 ++-- routing_algorithms/tsp_farthest_insertion.hpp | 40 ++++---- routing_algorithms/tsp_nearest_neighbour.hpp | 8 +- util/dist_table_wrapper.hpp | 95 +++++++++++++++++++ 6 files changed, 157 insertions(+), 74 deletions(-) create mode 100644 util/dist_table_wrapper.hpp diff --git a/data_structures/matrix_graph_wrapper.hpp b/data_structures/matrix_graph_wrapper.hpp index 7e91e8b3314..9a6180ce954 100644 --- a/data_structures/matrix_graph_wrapper.hpp +++ b/data_structures/matrix_graph_wrapper.hpp @@ -36,30 +36,29 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. template class MatrixGraphWrapper { public: - MatrixGraphWrapper(std::vector table, const unsigned number_of_nodes) : table_(table), number_of_nodes_(number_of_nodes) {}; + MatrixGraphWrapper(std::vector table, const std::size_t number_of_nodes) : table_(table), number_of_nodes_(number_of_nodes) {}; - unsigned GetNumberOfNodes() const { + std::size_t GetNumberOfNodes() const { return number_of_nodes_; } - std::vector GetAdjacentEdgeRange(const unsigned node) const { - std::vector edges; - const auto maxint = std::numeric_limits::max(); - for (auto i = 0; i < number_of_nodes_; ++i) { - if (*(table_.begin() + node * number_of_nodes_ + i) != maxint) { - edges.push_back(i); - } - } + std::vector GetAdjacentEdgeRange(const NodeID node) const { + std::vector edges; + auto neq_invalid_edge_weight = [](EdgeWeight e){return (e != INVALID_EDGE_WEIGHT);}; + std::copy_if(std::begin(table_), + std::end(table_), + std::back_inserter(edges), + neq_invalid_edge_weight); return edges; } - unsigned GetTarget(const unsigned edge) const { + EdgeWeight GetTarget(const EdgeWeight edge) const { return edge; } private: std::vector table_; - const unsigned number_of_nodes_; + const std::size_t number_of_nodes_; }; diff --git a/plugins/round_trip.hpp b/plugins/round_trip.hpp index e1d95e72cd6..d2d5045abb6 100644 --- a/plugins/round_trip.hpp +++ b/plugins/round_trip.hpp @@ -47,6 +47,7 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "../util/string_util.hpp" #include "../util/timing_util.hpp" #include "../util/simple_logger.hpp" +#include "../util/dist_table_wrapper.hpp" #include #include @@ -56,6 +57,7 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include #include #include +#include #include #include #include @@ -97,20 +99,18 @@ template class RoundTripPlugin final : public BasePlugin facade->IncrementalFindPhantomNodeForCoordinate(route_parameters.coordinates[i], phantom_node_vector[i], 1); if (phantom_node_vector[i].size() > 1) { - phantom_node_vector[i].erase(phantom_node_vector[i].begin()); + phantom_node_vector[i].erase(std::begin(phantom_node_vector[i])); } BOOST_ASSERT(phantom_node_vector[i].front().is_valid(facade->GetNumberOfNodes())); } } void SplitUnaccessibleLocations(const std::size_t number_of_locations, - std::vector & result_table, + const DistTableWrapper & result_table, std::vector> & components) { // Run TarjanSCC - auto wrapper = std::make_shared>(result_table, number_of_locations); - // auto empty_restriction = RestrictionMap(std::vector()); - // std::vector empty_vector; + auto wrapper = std::make_shared>(result_table.GetTable(), number_of_locations); auto scc = TarjanSCC>(wrapper); scc.run(); @@ -126,7 +126,7 @@ template class RoundTripPlugin final : public BasePlugin template void SetLocPermutationOutput(const std::vector & loc_permutation, osrm::json::Object & json_result){ osrm::json::Array json_loc_permutation; - json_loc_permutation.values.insert(json_loc_permutation.values.end(), loc_permutation.begin(), loc_permutation.end()); + json_loc_permutation.values.insert(std::end(json_loc_permutation.values), std::begin(loc_permutation), std::end(loc_permutation)); json_result.values["loc_permutation"] = json_loc_permutation; } @@ -152,14 +152,14 @@ template class RoundTripPlugin final : public BasePlugin InternalRouteResult & min_route) { // given he final trip, compute total distance and return the route and location permutation PhantomNodes viapoint; - for (auto it = trip.begin(); it != std::prev(trip.end()); ++it) { + for (auto it = std::begin(trip); it != std::prev(std::end(trip)); ++it) { auto from_node = *it; auto to_node = *std::next(it); viapoint = PhantomNodes{phantom_node_vector[from_node][0], phantom_node_vector[to_node][0]}; min_route.segment_end_coordinates.emplace_back(viapoint); } // check dist between last and first location too - viapoint = PhantomNodes{phantom_node_vector[*std::prev(trip.end())][0], phantom_node_vector[trip.front()][0]}; + viapoint = PhantomNodes{phantom_node_vector[*std::prev(std::end(trip))][0], phantom_node_vector[trip.front()][0]}; min_route.segment_end_coordinates.emplace_back(viapoint); search_engine_ptr->shortest_path(min_route.segment_end_coordinates, route_parameters.uturns, min_route); } @@ -186,29 +186,29 @@ template class RoundTripPlugin final : public BasePlugin PhantomNodeArray phantom_node_vector(route_parameters.coordinates.size()); GetPhantomNodes(route_parameters, phantom_node_vector); + auto number_of_locations = phantom_node_vector.size(); // compute the distance table of all phantom nodes - const std::shared_ptr> result_table = - search_engine_ptr->distance_table(phantom_node_vector); - if (!result_table){ + const auto result_table = DistTableWrapper(*search_engine_ptr->distance_table(phantom_node_vector), + number_of_locations); + if (result_table.size() == 0){ return 400; } const constexpr std::size_t BF_MAX_FEASABLE = 10; - auto number_of_locations = phantom_node_vector.size(); - BOOST_ASSERT_MSG(result_table->size() > 0, "Distance Table is empty."); + BOOST_ASSERT_MSG(result_table.size() > 0, "Distance Table is empty."); std::vector> components; //check if locations are in different strongly connected components (SCC) const auto maxint = INVALID_EDGE_WEIGHT; - if (*std::max_element(std::begin(*result_table), std::end(*result_table)) == maxint) { + if (*std::max_element(result_table.begin(), result_table.end()) == maxint) { // Compute all SCC - SplitUnaccessibleLocations(number_of_locations, *result_table, components); + SplitUnaccessibleLocations(number_of_locations, result_table, components); } else { // fill a vector with node ids std::vector location_ids(number_of_locations); - std::iota(location_ids.begin(), location_ids.end(), 0); - components.push_back(location_ids); + std::iota(std::begin(location_ids), std::end(location_ids), 0); + components.push_back(std::move(location_ids)); } @@ -222,19 +222,19 @@ template class RoundTripPlugin final : public BasePlugin // Compute the TSP with the given algorithm if (route_parameters.tsp_algo == "BF" && route_parameters.coordinates.size() < BF_MAX_FEASABLE) { SimpleLogger().Write() << "Running brute force"; - scc_route = osrm::tsp::BruteForceTSP(components[k], number_of_locations, *result_table); + scc_route = osrm::tsp::BruteForceTSP(components[k], number_of_locations, result_table); res_route.push_back(scc_route); } else if (route_parameters.tsp_algo == "NN") { SimpleLogger().Write() << "Running nearest neighbour"; - scc_route = osrm::tsp::NearestNeighbourTSP(components[k], number_of_locations, *result_table); + scc_route = osrm::tsp::NearestNeighbourTSP(components[k], number_of_locations, result_table); res_route.push_back(scc_route); } else if (route_parameters.tsp_algo == "FI") { SimpleLogger().Write() << "Running farthest insertion"; - scc_route = osrm::tsp::FarthestInsertionTSP(components[k], number_of_locations, *result_table); + scc_route = osrm::tsp::FarthestInsertionTSP(components[k], number_of_locations, result_table); res_route.push_back(scc_route); } else{ SimpleLogger().Write() << "Running farthest insertion"; - scc_route = osrm::tsp::FarthestInsertionTSP(components[k], number_of_locations, *result_table); + scc_route = osrm::tsp::FarthestInsertionTSP(components[k], number_of_locations, result_table); res_route.push_back(scc_route); } } @@ -244,12 +244,6 @@ template class RoundTripPlugin final : public BasePlugin TIMER_STOP(tsp); SetRuntimeOutput(TIMER_MSEC(tsp), json_result); SimpleLogger().Write() << "Computed roundtrip in " << TIMER_MSEC(tsp) << "ms"; - // SimpleLogger().Write() << "Route is"; - // for (auto x : res_route) { - // for (auto y : x) - // std::cout << y << " "; - // } - // SimpleLogger().Write() << ""; auto dist = 0; for (auto curr_route : route) { diff --git a/routing_algorithms/tsp_brute_force.hpp b/routing_algorithms/tsp_brute_force.hpp index 022804e5a61..d92859147db 100644 --- a/routing_algorithms/tsp_brute_force.hpp +++ b/routing_algorithms/tsp_brute_force.hpp @@ -31,6 +31,7 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "../data_structures/search_engine.hpp" #include "../util/string_util.hpp" +#include "../util/dist_table_wrapper.hpp" #include @@ -52,24 +53,22 @@ namespace tsp { template -int ReturnDistance(const std::vector & dist_table, +int ReturnDistance(const DistTableWrapper & dist_table, const std::vector & location_order, const EdgeWeight min_route_dist, - const int number_of_locations) { - int route_dist = 0; + const std::size_t number_of_locations) { + EdgeWeight route_dist = 0; int i = 0; - while (i < location_order.size() - 1) { - route_dist += *(dist_table.begin() + (location_order[i] * number_of_locations) + location_order[i+1]); + while (i < location_order.size()) { + route_dist += dist_table(location_order[i], location_order[(i+1) % number_of_locations]); ++i; } - //get distance from last location to first location - route_dist += *(dist_table.begin() + (location_order[location_order.size()-1] * number_of_locations) + location_order[0]); return route_dist; } std::vector BruteForceTSP(std::vector & component, const std::size_t number_of_locations, - const std::vector & dist_table) { + const DistTableWrapper & dist_table) { std::vector route; route.reserve(number_of_locations); @@ -84,7 +83,7 @@ std::vector BruteForceTSP(std::vector & component, min_route_dist = new_distance; route = component; } - } while(std::next_permutation(component.begin(), component.end())); + } while(std::next_permutation(std::begin(component), std::end(component))); return route; } diff --git a/routing_algorithms/tsp_farthest_insertion.hpp b/routing_algorithms/tsp_farthest_insertion.hpp index 03cd451ca77..c34c91ddb8e 100644 --- a/routing_algorithms/tsp_farthest_insertion.hpp +++ b/routing_algorithms/tsp_farthest_insertion.hpp @@ -31,6 +31,7 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "../data_structures/search_engine.hpp" #include "../util/string_util.hpp" +#include "../util/dist_table_wrapper.hpp" #include "../tools/tsp_logs.hpp" #include @@ -51,8 +52,10 @@ namespace tsp using NodeIterator = typename std::vector::iterator; -std::pair GetShortestRoundTrip(const int current_loc, - const std::vector & dist_table, +// given a route and a new location, find the best place of insertion and +// check the distance of roundtrip when the new location is additionally visited +std::pair GetShortestRoundTrip(const int new_loc, + const DistTableWrapper & dist_table, const int number_of_locations, std::vector & route){ @@ -61,12 +64,15 @@ std::pair GetShortestRoundTrip(const int current_loc, // for all nodes in the current trip find the best insertion resulting in the shortest path // assert min 2 nodes in route - for (auto from_node = route.begin(); from_node != std::prev(route.end()); ++from_node) { - const auto to_node = std::next(from_node); + for (auto from_node = std::begin(route); from_node != std::end(route); ++from_node) { + auto to_node = std::next(from_node); + if (to_node == std::end(route)) { + to_node = std::begin(route); + } - const auto dist_from = *(dist_table.begin() + (*from_node * number_of_locations) + current_loc); - const auto dist_to = *(dist_table.begin() + (current_loc * number_of_locations) + *to_node); - const auto trip_dist = dist_from + dist_to - *(dist_table.begin() + (*from_node * number_of_locations) + *to_node); + const auto dist_from = dist_table(*from_node, new_loc); + const auto dist_to = dist_table(new_loc, *to_node); + const auto trip_dist = dist_from + dist_to - dist_table(*from_node, *to_node);; // from all possible insertions to the current trip, choose the shortest of all insertions if (trip_dist < min_trip_distance) { @@ -74,17 +80,6 @@ std::pair GetShortestRoundTrip(const int current_loc, next_insert_point_candidate = to_node; } } - // check insertion between last and first location too - auto from_node = std::prev(route.end()); - auto to_node = route.begin(); - - auto dist_from = *(dist_table.begin() + (*from_node * number_of_locations) + current_loc); - auto dist_to = *(dist_table.begin() + (current_loc * number_of_locations) + *to_node); - auto trip_dist = dist_from + dist_to - *(dist_table.begin() + (*from_node * number_of_locations) + *to_node); - if (trip_dist < min_trip_distance) { - min_trip_distance = trip_dist; - next_insert_point_candidate = to_node; - } return std::make_pair(min_trip_distance, next_insert_point_candidate); } @@ -93,7 +88,7 @@ std::pair GetShortestRoundTrip(const int current_loc, std::vector FindRoute(const std::size_t & number_of_locations, const std::size_t & size_of_component, const std::vector & locations, - const std::vector & dist_table, + const DistTableWrapper & dist_table, const NodeID & start1, const NodeID & start2) { std::vector route; @@ -136,10 +131,9 @@ std::vector FindRoute(const std::size_t & number_of_locations, return route; } -// osrm::tsp::FarthestInsertionTSP(components[k], phantom_node_vector, *result_table, scc_route); std::vector FarthestInsertionTSP(const std::vector & locations, const std::size_t number_of_locations, - const std::vector & dist_table) { + const DistTableWrapper & dist_table) { ////////////////////////////////////////////////////////////////////////////////////////////////// // START FARTHEST INSERTION HERE // 1. start at a random round trip of 2 locations @@ -149,7 +143,7 @@ std::vector FarthestInsertionTSP(const std::vector & locations, // 5. DONE! ////////////////////////////////////////////////////////////////////////////////////////////////// - const int size_of_component = locations.size(); + const auto size_of_component = locations.size(); auto max_from = -1; auto max_to = -1; @@ -163,7 +157,7 @@ std::vector FarthestInsertionTSP(const std::vector & locations, auto max_dist = 0; for (auto x : locations) { for (auto y : locations) { - auto xy_dist = *(dist_table.begin() + x * number_of_locations + y); + auto xy_dist = dist_table(x, y); if (xy_dist > max_dist) { max_dist = xy_dist; max_from = x; diff --git a/routing_algorithms/tsp_nearest_neighbour.hpp b/routing_algorithms/tsp_nearest_neighbour.hpp index cd2adadec16..3fde321746a 100644 --- a/routing_algorithms/tsp_nearest_neighbour.hpp +++ b/routing_algorithms/tsp_nearest_neighbour.hpp @@ -32,6 +32,7 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "../data_structures/search_engine.hpp" #include "../util/string_util.hpp" #include "../util/simple_logger.hpp" +#include "../util/dist_table_wrapper.hpp" #include @@ -50,7 +51,7 @@ namespace tsp std::vector NearestNeighbourTSP(const std::vector & locations, const std::size_t number_of_locations, - const std::vector & dist_table) { + const DistTableWrapper & dist_table) { ////////////////////////////////////////////////////////////////////////////////////////////////// // START GREEDY NEAREST NEIGHBOUR HERE // 1. grab a random location and mark as starting point @@ -89,9 +90,10 @@ std::vector NearestNeighbourTSP(const std::vector & locations, // 2. FIND NEAREST NEIGHBOUR for (auto next : locations) { + auto curr_dist = dist_table(curr_node, next); if(!visited[next] && - *(dist_table.begin() + curr_node * number_of_locations + next) < min_dist) { - min_dist = *(dist_table.begin() + curr_node * number_of_locations + next); + curr_dist < min_dist) { + min_dist = curr_dist; min_id = next; } } diff --git a/util/dist_table_wrapper.hpp b/util/dist_table_wrapper.hpp new file mode 100644 index 00000000000..a626cfbc9fe --- /dev/null +++ b/util/dist_table_wrapper.hpp @@ -0,0 +1,95 @@ +/* + +Copyright (c) 2015, Project OSRM contributors +All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + +Redistributions of source code must retain the above copyright notice, this list +of conditions and the following disclaimer. +Redistributions in binary form must reproduce the above copyright notice, this +list of conditions and the following disclaimer in the documentation and/or +other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +*/ + +#ifndef DIST_TABLE_WRAPPER_H +#define DIST_TABLE_WRAPPER_H + +#include +#include +#include + +//This Wrapper provides an easier access to a distance table that is given as an linear vector + +template class DistTableWrapper { +public: + + using Iterator = typename std::vector::iterator; + using ConstIterator = typename std::vector::const_iterator; + + DistTableWrapper(std::vector table, std::size_t number_of_nodes) + : table_(std::move(table)), number_of_nodes_(number_of_nodes) { + BOOST_ASSERT_MSG(table.size() == 0, "table is empty"); + BOOST_ASSERT_MSG(number_of_nodes_ * number_of_nodes_ < table_, "number_of_nodes_ is invalid"); + }; + + std::size_t GetNumberOfNodes() const { + return number_of_nodes_; + } + + std::size_t size() const { + return table_.size(); + } + + EdgeWeight operator() (NodeID from, NodeID to) const { + BOOST_ASSERT_MSG(from < number_of_nodes_, "from ID is out of bound"); + BOOST_ASSERT_MSG(to < number_of_nodes_, "to ID is out of bound"); + const auto index = from * number_of_nodes_ + to; + BOOST_ASSERT_MSG(index < table_size(), "index is out of bound"); + return table_[index]; + } + + ConstIterator begin() const{ + return std::begin(table_); + } + + Iterator begin() { + return std::begin(table_); + } + + ConstIterator end() const{ + return std::end(table_); + } + + Iterator end() { + return std::end(table_); + } + + NodeID GetIndexOfMaxValue() const { + return std::distance(table_.begin(), std::max_element(table_.begin(), table_.end())); + } + + std::vector GetTable() const { + return table_; + } + +private: + std::vector table_; + const std::size_t number_of_nodes_; +}; + + +#endif // DIST_TABLE_WRAPPER_H From 2de3fc9f6f5bc688ac06c2cc7d3098adb73133ef Mon Sep 17 00:00:00 2001 From: Huyen Chau Nguyen Date: Wed, 19 Aug 2015 19:44:11 +0200 Subject: [PATCH 078/122] fix GetAdjacendEdgeRange of matrix wrapper for tarjan scc and fix wrongly solved merge conflict --- algorithms/tarjan_scc.hpp | 9 ++++----- data_structures/matrix_graph_wrapper.hpp | 14 +++++++------- 2 files changed, 11 insertions(+), 12 deletions(-) diff --git a/algorithms/tarjan_scc.hpp b/algorithms/tarjan_scc.hpp index 8444d785a21..6f5bc1563a3 100644 --- a/algorithms/tarjan_scc.hpp +++ b/algorithms/tarjan_scc.hpp @@ -52,6 +52,7 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include #include #include +#include template class TarjanSCC { @@ -192,6 +193,7 @@ template class TarjanSCC }); } + std::size_t get_number_of_components() const { return component_size_vector.size(); } std::size_t get_size_one_count() const { return size_one_counter; } @@ -201,12 +203,9 @@ template class TarjanSCC return component_size_vector[component_id]; } - unsigned get_component_size_by_id(const unsigned component_id) const - { - return component_size_vector[component_id]; - } - unsigned get_component_id(const NodeID node) const { return components_index[node]; } + + }; #endif /* TARJAN_SCC_HPP */ diff --git a/data_structures/matrix_graph_wrapper.hpp b/data_structures/matrix_graph_wrapper.hpp index 9a6180ce954..95b0e4c860a 100644 --- a/data_structures/matrix_graph_wrapper.hpp +++ b/data_structures/matrix_graph_wrapper.hpp @@ -42,13 +42,13 @@ template class MatrixGraphWrapper { return number_of_nodes_; } - std::vector GetAdjacentEdgeRange(const NodeID node) const { - std::vector edges; - auto neq_invalid_edge_weight = [](EdgeWeight e){return (e != INVALID_EDGE_WEIGHT);}; - std::copy_if(std::begin(table_), - std::end(table_), - std::back_inserter(edges), - neq_invalid_edge_weight); + std::vector GetAdjacentEdgeRange(const NodeID node) const { + std::vector edges; + for (auto i = 0; i < number_of_nodes_; ++i) { + if (*(std::begin(table_) + node * number_of_nodes_ + i) != INVALID_EDGE_WEIGHT) { + edges.push_back(i); + } + } return edges; } From 93835b9b94f241cdab8f2903979f3dd0a1f42aa2 Mon Sep 17 00:00:00 2001 From: Huyen Chau Nguyen Date: Thu, 20 Aug 2015 01:41:36 +0200 Subject: [PATCH 079/122] change input param for tsp algos from a vector to a begin and an end iterator --- plugins/round_trip.hpp | 145 ++++++++++++------ routing_algorithms/tsp_brute_force.hpp | 30 ++-- routing_algorithms/tsp_farthest_insertion.hpp | 110 ++++--------- routing_algorithms/tsp_nearest_neighbour.hpp | 37 ++--- 4 files changed, 157 insertions(+), 165 deletions(-) diff --git a/plugins/round_trip.hpp b/plugins/round_trip.hpp index d2d5045abb6..9fc288bd769 100644 --- a/plugins/round_trip.hpp +++ b/plugins/round_trip.hpp @@ -60,6 +60,7 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include #include #include +#include #include #include @@ -105,26 +106,56 @@ template class RoundTripPlugin final : public BasePlugin } } - void SplitUnaccessibleLocations(const std::size_t number_of_locations, - const DistTableWrapper & result_table, - std::vector> & components) { + struct SCC_Component{ + SCC_Component(std::vector in_component, + std::vector in_component_range) + : component(in_component), component_range(in_component_range) { + component_range.push_back(in_component.size()); + }; + + SCC_Component(std::vector in_component) + : component(in_component), component_range({0, in_component.size()}) { + }; + + std::size_t GetNumberOfComponents() const{ + return component_range.size() - 1; + } + + const std::vector component; + std::vector component_range; + }; + + SCC_Component SplitUnaccessibleLocations(const std::size_t number_of_locations, + const DistTableWrapper & result_table) { // Run TarjanSCC auto wrapper = std::make_shared>(result_table.GetTable(), number_of_locations); auto scc = TarjanSCC>(wrapper); scc.run(); + std::vector range_insertion; + std::vector component_range; + range_insertion.reserve(scc.get_number_of_components()); + component_range.reserve(scc.get_number_of_components()); + + std::vector components(number_of_locations, 0); + + auto prefix = 0; for (size_t j = 0; j < scc.get_number_of_components(); ++j){ - components.push_back(std::vector()); + range_insertion.push_back(prefix); + component_range.push_back(prefix); + prefix += scc.get_component_size(j); } for (size_t i = 0; i < number_of_locations; ++i) { - components[scc.get_component_id(i)].push_back(i); + components[range_insertion[scc.get_component_id(i)]] = i; + ++range_insertion[scc.get_component_id(i)]; } + + return SCC_Component(components, component_range); } - template - void SetLocPermutationOutput(const std::vector & loc_permutation, osrm::json::Object & json_result){ + void SetLocPermutationOutput(const std::vector & loc_permutation, osrm::json::Object & json_result){ osrm::json::Array json_loc_permutation; json_loc_permutation.values.insert(std::end(json_loc_permutation.values), std::begin(loc_permutation), std::end(loc_permutation)); json_result.values["loc_permutation"] = json_loc_permutation; @@ -137,7 +168,9 @@ template class RoundTripPlugin final : public BasePlugin json_result.values["runtime"] = runtime; } - void SetGeometry(const RouteParameters &route_parameters, const InternalRouteResult & min_route, osrm::json::Object & json_result) { + void SetGeometry(const RouteParameters &route_parameters, + const InternalRouteResult & min_route, + osrm::json::Object & json_result) { // return geometry result to json std::unique_ptr> descriptor; descriptor = osrm::make_unique>(facade); @@ -152,28 +185,14 @@ template class RoundTripPlugin final : public BasePlugin InternalRouteResult & min_route) { // given he final trip, compute total distance and return the route and location permutation PhantomNodes viapoint; - for (auto it = std::begin(trip); it != std::prev(std::end(trip)); ++it) { - auto from_node = *it; - auto to_node = *std::next(it); + for (auto it = std::begin(trip); it != std::end(trip); ++it) { + const auto from_node = *it; + const auto to_node = std::next(it) != std::end(trip) ? *std::next(it) : *std::begin(trip); viapoint = PhantomNodes{phantom_node_vector[from_node][0], phantom_node_vector[to_node][0]}; min_route.segment_end_coordinates.emplace_back(viapoint); } - // check dist between last and first location too - viapoint = PhantomNodes{phantom_node_vector[*std::prev(std::end(trip))][0], phantom_node_vector[trip.front()][0]}; - min_route.segment_end_coordinates.emplace_back(viapoint); - search_engine_ptr->shortest_path(min_route.segment_end_coordinates, route_parameters.uturns, min_route); - } - void ComputeRoute(const PhantomNodeArray & phantom_node_vector, - const RouteParameters & route_parameters, - std::vector> & trip, - std::vector & route) { - for (const auto & curr_trip : trip) { - InternalRouteResult curr_route; - ComputeRoute(phantom_node_vector, route_parameters, curr_trip, curr_route); - route.push_back(curr_route); - search_engine_ptr->shortest_path(route.back().segment_end_coordinates, route_parameters.uturns, route.back()); - } + search_engine_ptr->shortest_path(min_route.segment_end_coordinates, route_parameters.uturns, min_route); } int HandleRequest(const RouteParameters &route_parameters, @@ -184,6 +203,7 @@ template class RoundTripPlugin final : public BasePlugin return 400; } + // get phantom nodes PhantomNodeArray phantom_node_vector(route_parameters.coordinates.size()); GetPhantomNodes(route_parameters, phantom_node_vector); auto number_of_locations = phantom_node_vector.size(); @@ -197,58 +217,81 @@ template class RoundTripPlugin final : public BasePlugin const constexpr std::size_t BF_MAX_FEASABLE = 10; BOOST_ASSERT_MSG(result_table.size() > 0, "Distance Table is empty."); - std::vector> components; - - //check if locations are in different strongly connected components (SCC) - const auto maxint = INVALID_EDGE_WEIGHT; - if (*std::max_element(result_table.begin(), result_table.end()) == maxint) { - // Compute all SCC - SplitUnaccessibleLocations(number_of_locations, result_table, components); - } else { - // fill a vector with node ids - std::vector location_ids(number_of_locations); - std::iota(std::begin(location_ids), std::end(location_ids), 0); - components.push_back(std::move(location_ids)); - } + // get scc components + SCC_Component scc = [&](){ + if (*std::max_element(result_table.begin(), result_table.end()) == INVALID_EDGE_WEIGHT) { + // compute all scc with tarjan + return SplitUnaccessibleLocations(number_of_locations, result_table); + } else { + // whole graph is one scc + std::vector location_ids(number_of_locations); + std::iota(std::begin(location_ids), std::end(location_ids), 0); + return SCC_Component(location_ids); + } + }(); + + + using NodeIDIterator = typename std::vector::const_iterator; std::vector> res_route; TIMER_START(tsp); //run TSP computation for every SCC - for(auto k = 0; k < components.size(); ++k) { - if (components[k].size() > 1) { + for(auto k = 0; k < scc.GetNumberOfComponents(); ++k) { + const auto component_size = scc.component_range[k+1] - scc.component_range[k]; + if (component_size > 1) { std::vector scc_route; + NodeIDIterator start = std::begin(scc.component) + scc.component_range[k]; + NodeIDIterator end = std::begin(scc.component) + scc.component_range[k+1]; // Compute the TSP with the given algorithm if (route_parameters.tsp_algo == "BF" && route_parameters.coordinates.size() < BF_MAX_FEASABLE) { SimpleLogger().Write() << "Running brute force"; - scc_route = osrm::tsp::BruteForceTSP(components[k], number_of_locations, result_table); + scc_route = osrm::tsp::BruteForceTSP(start, end, number_of_locations, result_table); res_route.push_back(scc_route); } else if (route_parameters.tsp_algo == "NN") { SimpleLogger().Write() << "Running nearest neighbour"; - scc_route = osrm::tsp::NearestNeighbourTSP(components[k], number_of_locations, result_table); + scc_route = osrm::tsp::NearestNeighbourTSP(start, end, number_of_locations, result_table); res_route.push_back(scc_route); } else if (route_parameters.tsp_algo == "FI") { SimpleLogger().Write() << "Running farthest insertion"; - scc_route = osrm::tsp::FarthestInsertionTSP(components[k], number_of_locations, result_table); + scc_route = osrm::tsp::FarthestInsertionTSP(start, end, number_of_locations, result_table); res_route.push_back(scc_route); } else{ SimpleLogger().Write() << "Running farthest insertion"; - scc_route = osrm::tsp::FarthestInsertionTSP(components[k], number_of_locations, result_table); + scc_route = osrm::tsp::FarthestInsertionTSP(start, end, number_of_locations, result_table); res_route.push_back(scc_route); } + + SimpleLogger().Write() << "Route #" + << k << ": " + << [&scc_route](){ + std::string s = ""; + for (auto x : scc_route) { + s += std::to_string(x) + " "; + } + return s; + }(); } } - std::vector route; - ComputeRoute(phantom_node_vector, route_parameters, res_route, route); + std::vector comp_route (res_route.size()); + for (auto r = 0; r < res_route.size(); ++r) { + ComputeRoute(phantom_node_vector, route_parameters, res_route[r], comp_route[r]); + } TIMER_STOP(tsp); + SetRuntimeOutput(TIMER_MSEC(tsp), json_result); - SimpleLogger().Write() << "Computed roundtrip in " << TIMER_MSEC(tsp) << "ms"; + //TODO + SetLocPermutationOutput(res_route[0], json_result); + std::unique_ptr> descriptor; + descriptor = osrm::make_unique>(facade); + descriptor->SetConfig(route_parameters); auto dist = 0; - for (auto curr_route : route) { - dist += curr_route.shortest_path_length; - SetGeometry(route_parameters, curr_route, json_result); + for (auto r : comp_route) { + dist += r.shortest_path_length; + // SetGeometry(route_parameters, r, json_result); + descriptor->Run(r, json_result); } SetDistanceOutput(dist, json_result); diff --git a/routing_algorithms/tsp_brute_force.hpp b/routing_algorithms/tsp_brute_force.hpp index d92859147db..fb6849529fc 100644 --- a/routing_algorithms/tsp_brute_force.hpp +++ b/routing_algorithms/tsp_brute_force.hpp @@ -52,42 +52,44 @@ namespace osrm namespace tsp { -template -int ReturnDistance(const DistTableWrapper & dist_table, - const std::vector & location_order, - const EdgeWeight min_route_dist, - const std::size_t number_of_locations) { +EdgeWeight ReturnDistance(const DistTableWrapper & dist_table, + const std::vector & location_order, + const EdgeWeight min_route_dist, + const std::size_t component_size) { EdgeWeight route_dist = 0; - int i = 0; + std::size_t i = 0; while (i < location_order.size()) { - route_dist += dist_table(location_order[i], location_order[(i+1) % number_of_locations]); + route_dist += dist_table(location_order[i], location_order[(i+1) % component_size]); ++i; } return route_dist; } -std::vector BruteForceTSP(std::vector & component, +template +std::vector BruteForceTSP(const NodeIDIterator start, + const NodeIDIterator end, const std::size_t number_of_locations, const DistTableWrapper & dist_table) { + const auto component_size = std::distance(start, end); + std::vector perm(start, end); std::vector route; - route.reserve(number_of_locations); - + route.reserve(component_size); EdgeWeight min_route_dist = INVALID_EDGE_WEIGHT; // check length of all possible permutation of the component ids do { - const auto new_distance = ReturnDistance(dist_table, component, min_route_dist, number_of_locations); + const auto new_distance = ReturnDistance(dist_table, perm, min_route_dist, component_size); if (new_distance <= min_route_dist) { min_route_dist = new_distance; - route = component; + route = perm; } - } while(std::next_permutation(std::begin(component), std::end(component))); + } while(std::next_permutation(std::begin(perm), std::end(perm))); return route; } -} //end namespace osrm } //end namespace tsp +} //end namespace osrm #endif // TSP_BRUTE_FORCE_HPP \ No newline at end of file diff --git a/routing_algorithms/tsp_farthest_insertion.hpp b/routing_algorithms/tsp_farthest_insertion.hpp index c34c91ddb8e..309471bef3a 100644 --- a/routing_algorithms/tsp_farthest_insertion.hpp +++ b/routing_algorithms/tsp_farthest_insertion.hpp @@ -50,17 +50,16 @@ namespace osrm namespace tsp { -using NodeIterator = typename std::vector::iterator; - // given a route and a new location, find the best place of insertion and // check the distance of roundtrip when the new location is additionally visited -std::pair GetShortestRoundTrip(const int new_loc, - const DistTableWrapper & dist_table, - const int number_of_locations, - std::vector & route){ +using NodeIDIter = typename std::vector::iterator; +std::pair GetShortestRoundTrip(const NodeID new_loc, + const DistTableWrapper & dist_table, + const std::size_t number_of_locations, + std::vector & route){ auto min_trip_distance = INVALID_EDGE_WEIGHT; - NodeIterator next_insert_point_candidate; + NodeIDIter next_insert_point_candidate; // for all nodes in the current trip find the best insertion resulting in the shortest path // assert min 2 nodes in route @@ -84,10 +83,12 @@ std::pair GetShortestRoundTrip(const int new_loc, return std::make_pair(min_trip_distance, next_insert_point_candidate); } +template // given two initial start nodes, find a roundtrip route using the farthest insertion algorithm std::vector FindRoute(const std::size_t & number_of_locations, const std::size_t & size_of_component, - const std::vector & locations, + const NodeIDIterator & start, + const NodeIDIterator & end, const DistTableWrapper & dist_table, const NodeID & start1, const NodeID & start2) { @@ -103,22 +104,22 @@ std::vector FindRoute(const std::size_t & number_of_locations, route.push_back(start2); // add all other nodes missing (two nodes are already in the initial start trip) - for (int j = 2; j < size_of_component; ++j) { + for (std::size_t j = 2; j < size_of_component; ++j) { auto farthest_distance = 0; auto next_node = -1; - NodeIterator next_insert_point; + NodeIDIter next_insert_point; // find unvisited loc i that is the farthest away from all other visited locs - for (auto i : locations) { + for (auto i = start; i != end; ++i) { // find the shortest distance from i to all visited nodes - if (!visited[i]) { - auto insert_candidate = GetShortestRoundTrip(i, dist_table, number_of_locations, route); + if (!visited[*i]) { + auto insert_candidate = GetShortestRoundTrip(*i, dist_table, number_of_locations, route); // add the location to the current trip such that it results in the shortest total tour if (insert_candidate.first >= farthest_distance) { farthest_distance = insert_candidate.first; - next_node = i; + next_node = *i; next_insert_point = insert_candidate.second; } } @@ -131,7 +132,9 @@ std::vector FindRoute(const std::size_t & number_of_locations, return route; } -std::vector FarthestInsertionTSP(const std::vector & locations, +template +std::vector FarthestInsertionTSP(const NodeIDIterator & start, + const NodeIDIterator & end, const std::size_t number_of_locations, const DistTableWrapper & dist_table) { ////////////////////////////////////////////////////////////////////////////////////////////////// @@ -143,92 +146,35 @@ std::vector FarthestInsertionTSP(const std::vector & locations, // 5. DONE! ////////////////////////////////////////////////////////////////////////////////////////////////// - const auto size_of_component = locations.size(); + const auto component_size = std::distance(start, end); auto max_from = -1; auto max_to = -1; - if (size_of_component == number_of_locations) { + if (component_size == number_of_locations) { // find the pair of location with the biggest distance and make the pair the initial start trip - const auto index = std::distance(dist_table.begin(), std::max_element(dist_table.begin(), dist_table.end())); + const auto index = std::distance(std::begin(dist_table), std::max_element(std::begin(dist_table), std::end(dist_table))); max_from = index / number_of_locations; max_to = index % number_of_locations; } else { auto max_dist = 0; - for (auto x : locations) { - for (auto y : locations) { - auto xy_dist = dist_table(x, y); + for (auto x = start; x != end; ++x) { + for (auto y = start; y != end; ++y) { + const auto xy_dist = dist_table(*x, *y); if (xy_dist > max_dist) { max_dist = xy_dist; - max_from = x; - max_to = y; + max_from = *x; + max_to = *y; } } } } - return FindRoute(number_of_locations, size_of_component, locations, dist_table, max_from, max_to); + return FindRoute(number_of_locations, component_size, start, end, dist_table, max_from, max_to); } -// std::vector FarthestInsertionTSP(const std::size_t number_of_locations, -// const std::vector & dist_table) { -// ////////////////////////////////////////////////////////////////////////////////////////////////// -// // START FARTHEST INSERTION HERE -// // 1. start at a random round trip of 2 locations -// // 2. find the location that is the farthest away from the visited locations and whose insertion will make the round trip the longest -// // 3. add the found location to the current round trip such that round trip is the shortest -// // 4. repeat 2-3 until all locations are visited -// // 5. DONE! -// ////////////////////////////////////////////////////////////////////////////////////////////////// - -// std::vector route; -// route.reserve(number_of_locations); - -// // tracks which nodes have been already visited -// std::vector visited(number_of_locations, false); - -// // find the pair of location with the biggest distance and make the pair the initial start trip -// const auto index = std::distance(dist_table.begin(), std::max_element(dist_table.begin(), dist_table.end())); -// const int max_from = index / number_of_locations; -// const int max_to = index % number_of_locations; -// visited[max_from] = true; -// visited[max_to] = true; -// route.push_back(max_from); -// route.push_back(max_to); - -// // add all other nodes missing (two nodes are already in the initial start trip) -// for (int j = 2; j < number_of_locations; ++j) { -// auto farthest_distance = 0; -// auto next_node = -1; -// //todo move out of loop and overwrite -// NodeIterator next_insert_point; - -// // find unvisited loc i that is the farthest away from all other visited locs -// for (int i = 0; i < number_of_locations; ++i) { -// if (!visited[i]) { -// auto min_trip_distance = INVALID_EDGE_WEIGHT; -// NodeIterator next_insert_point_candidate; - -// GetShortestRoundTrip(i, dist_table, number_of_locations, route, min_trip_distance, next_insert_point_candidate); - -// // add the location to the current trip such that it results in the shortest total tour -// if (min_trip_distance >= farthest_distance) { -// farthest_distance = min_trip_distance; -// next_node = i; -// next_insert_point = next_insert_point_candidate; -// } -// } -// } - -// // mark as visited and insert node -// visited[next_node] = true; -// route.insert(next_insert_point, next_node); -// } -// return route; -// } - -} //end namespace osrm } //end namespace tsp +} //end namespace osrm #endif // TSP_FARTHEST_INSERTION_HPP \ No newline at end of file diff --git a/routing_algorithms/tsp_nearest_neighbour.hpp b/routing_algorithms/tsp_nearest_neighbour.hpp index 3fde321746a..899eec60d2c 100644 --- a/routing_algorithms/tsp_nearest_neighbour.hpp +++ b/routing_algorithms/tsp_nearest_neighbour.hpp @@ -48,8 +48,9 @@ namespace osrm { namespace tsp { - -std::vector NearestNeighbourTSP(const std::vector & locations, +template +std::vector NearestNeighbourTSP(const NodeIDIterator & start, + const NodeIDIterator & end, const std::size_t number_of_locations, const DistTableWrapper & dist_table) { ////////////////////////////////////////////////////////////////////////////////////////////////// @@ -65,36 +66,36 @@ std::vector NearestNeighbourTSP(const std::vector & locations, std::vector route; route.reserve(number_of_locations); - const int component_size = locations.size(); - int shortest_trip_distance = INVALID_EDGE_WEIGHT; + const auto component_size = std::distance(start, end); + auto shortest_trip_distance = INVALID_EDGE_WEIGHT; // ALWAYS START AT ANOTHER STARTING POINT - for(auto start_node : locations) + for(auto start_node = start; start_node != end; ++start_node) { - int curr_node = start_node; + NodeID curr_node = *start_node; std::vector curr_route; curr_route.reserve(component_size); - curr_route.push_back(start_node); + curr_route.push_back(*start_node); // visited[i] indicates whether node i was already visited by the salesman std::vector visited(number_of_locations, false); - visited[start_node] = true; + visited[*start_node] = true; // 3. REPEAT FOR EVERY UNVISITED NODE - int trip_dist = 0; - for(int via_point = 1; via_point < component_size; ++via_point) + EdgeWeight trip_dist = 0; + for(auto via_point = 1; via_point < component_size; ++via_point) { - int min_dist = INVALID_EDGE_WEIGHT; - int min_id = -1; + EdgeWeight min_dist = INVALID_EDGE_WEIGHT; + NodeID min_id = SPECIAL_NODEID; // 2. FIND NEAREST NEIGHBOUR - for (auto next : locations) { - auto curr_dist = dist_table(curr_node, next); - if(!visited[next] && + for (auto next = start; next != end; ++next) { + auto curr_dist = dist_table(curr_node, *next); + if(!visited[*next] && curr_dist < min_dist) { min_dist = curr_dist; - min_id = next; + min_id = *next; } } visited[min_id] = true; @@ -112,6 +113,6 @@ std::vector NearestNeighbourTSP(const std::vector & locations, return route; } -} -} +} //end namespace tsp +} //end namespace osrm #endif // TSP_NEAREST_NEIGHBOUR_HPP \ No newline at end of file From 47fbd2a2b5a7349d946eb623bc87ebf0a1aeaad2 Mon Sep 17 00:00:00 2001 From: Huyen Chau Nguyen Date: Thu, 20 Aug 2015 14:07:14 +0200 Subject: [PATCH 080/122] fix json output such that each trip returns a json object with all information of the trip --- plugins/round_trip.hpp | 129 +++++++++++++++++++++++------------------ 1 file changed, 72 insertions(+), 57 deletions(-) diff --git a/plugins/round_trip.hpp b/plugins/round_trip.hpp index 9fc288bd769..460fe249dce 100644 --- a/plugins/round_trip.hpp +++ b/plugins/round_trip.hpp @@ -106,27 +106,43 @@ template class RoundTripPlugin final : public BasePlugin } } + // Object to hold all strongly connected components (scc) of a graph + // to access all graphs with component ID i, get the iterators by: + // auto start = std::begin(scc_component.component) + scc_component.range[i]; + // auto end = std::begin(scc_component.component) + scc_component.range[i+1]; struct SCC_Component{ + // in_component: all NodeIDs sorted by component ID + // in_range: index where a new component starts + // + // example: NodeID 0, 1, 2, 4, 5 are in component 0 + // NodeID 3, 6, 7, 8 are in component 1 + // => in_component = [0, 1, 2, 4, 5, 3, 6, 7, 8] + // => in_range = [0, 5] SCC_Component(std::vector in_component, - std::vector in_component_range) - : component(in_component), component_range(in_component_range) { - component_range.push_back(in_component.size()); + std::vector in_range) + : component(in_component), range(in_range) { + range.push_back(in_component.size()); }; + // constructor to use when whole graph is one single scc SCC_Component(std::vector in_component) - : component(in_component), component_range({0, in_component.size()}) { + : component(in_component), range({0, in_component.size()}) { }; std::size_t GetNumberOfComponents() const{ - return component_range.size() - 1; + return range.size() - 1; } const std::vector component; - std::vector component_range; + // component range = in_range + [component.size()] + std::vector range; }; + // takes the number of locations and its distance matrix, + // identifies and splits the graph in its strongly connected components (scc) + // and returns an SCC_Component SCC_Component SplitUnaccessibleLocations(const std::size_t number_of_locations, - const DistTableWrapper & result_table) { + const DistTableWrapper & result_table) { // Run TarjanSCC auto wrapper = std::make_shared>(result_table.GetTable(), number_of_locations); @@ -134,16 +150,16 @@ template class RoundTripPlugin final : public BasePlugin scc.run(); std::vector range_insertion; - std::vector component_range; + std::vector range; range_insertion.reserve(scc.get_number_of_components()); - component_range.reserve(scc.get_number_of_components()); + range.reserve(scc.get_number_of_components()); std::vector components(number_of_locations, 0); auto prefix = 0; for (size_t j = 0; j < scc.get_number_of_components(); ++j){ range_insertion.push_back(prefix); - component_range.push_back(prefix); + range.push_back(prefix); prefix += scc.get_component_size(j); } @@ -152,31 +168,16 @@ template class RoundTripPlugin final : public BasePlugin ++range_insertion[scc.get_component_id(i)]; } - return SCC_Component(components, component_range); + return SCC_Component(components, range); } - void SetLocPermutationOutput(const std::vector & loc_permutation, osrm::json::Object & json_result){ - osrm::json::Array json_loc_permutation; - json_loc_permutation.values.insert(std::end(json_loc_permutation.values), std::begin(loc_permutation), std::end(loc_permutation)); - json_result.values["loc_permutation"] = json_loc_permutation; - } - - void SetDistanceOutput(const int distance, osrm::json::Object & json_result) { - json_result.values["distance"] = distance; - } - void SetRuntimeOutput(const float runtime, osrm::json::Object & json_result) { - json_result.values["runtime"] = runtime; - } - - void SetGeometry(const RouteParameters &route_parameters, - const InternalRouteResult & min_route, - osrm::json::Object & json_result) { - // return geometry result to json - std::unique_ptr> descriptor; - descriptor = osrm::make_unique>(facade); - - descriptor->SetConfig(route_parameters); - descriptor->Run(min_route, json_result); + void SetLocPermutationOutput(const std::vector & permutation, + osrm::json::Object & json_result){ + osrm::json::Array json_permutation; + json_permutation.values.insert(std::end(json_permutation.values), + std::begin(permutation), + std::end(permutation)); + json_result.values["permutation"] = json_permutation; } void ComputeRoute(const PhantomNodeArray & phantom_node_vector, @@ -187,6 +188,7 @@ template class RoundTripPlugin final : public BasePlugin PhantomNodes viapoint; for (auto it = std::begin(trip); it != std::end(trip); ++it) { const auto from_node = *it; + // if from_node is the last node, compute the route from the last to the first location const auto to_node = std::next(it) != std::end(trip) ? *std::next(it) : *std::begin(trip); viapoint = PhantomNodes{phantom_node_vector[from_node][0], phantom_node_vector[to_node][0]}; min_route.segment_end_coordinates.emplace_back(viapoint); @@ -234,33 +236,34 @@ template class RoundTripPlugin final : public BasePlugin using NodeIDIterator = typename std::vector::const_iterator; - std::vector> res_route; + std::vector> route_result; + route_result.reserve(scc.GetNumberOfComponents()); TIMER_START(tsp); //run TSP computation for every SCC for(auto k = 0; k < scc.GetNumberOfComponents(); ++k) { - const auto component_size = scc.component_range[k+1] - scc.component_range[k]; + const auto component_size = scc.range[k+1] - scc.range[k]; if (component_size > 1) { std::vector scc_route; - NodeIDIterator start = std::begin(scc.component) + scc.component_range[k]; - NodeIDIterator end = std::begin(scc.component) + scc.component_range[k+1]; + NodeIDIterator start = std::begin(scc.component) + scc.range[k]; + NodeIDIterator end = std::begin(scc.component) + scc.range[k+1]; // Compute the TSP with the given algorithm if (route_parameters.tsp_algo == "BF" && route_parameters.coordinates.size() < BF_MAX_FEASABLE) { SimpleLogger().Write() << "Running brute force"; scc_route = osrm::tsp::BruteForceTSP(start, end, number_of_locations, result_table); - res_route.push_back(scc_route); + route_result.push_back(scc_route); } else if (route_parameters.tsp_algo == "NN") { SimpleLogger().Write() << "Running nearest neighbour"; scc_route = osrm::tsp::NearestNeighbourTSP(start, end, number_of_locations, result_table); - res_route.push_back(scc_route); + route_result.push_back(scc_route); } else if (route_parameters.tsp_algo == "FI") { SimpleLogger().Write() << "Running farthest insertion"; scc_route = osrm::tsp::FarthestInsertionTSP(start, end, number_of_locations, result_table); - res_route.push_back(scc_route); + route_result.push_back(scc_route); } else{ SimpleLogger().Write() << "Running farthest insertion"; scc_route = osrm::tsp::FarthestInsertionTSP(start, end, number_of_locations, result_table); - res_route.push_back(scc_route); + route_result.push_back(scc_route); } SimpleLogger().Write() << "Route #" @@ -272,28 +275,40 @@ template class RoundTripPlugin final : public BasePlugin } return s; }(); + } else { + route_result.push_back({scc.component[scc.range[k]]}); } } - std::vector comp_route (res_route.size()); - for (auto r = 0; r < res_route.size(); ++r) { - ComputeRoute(phantom_node_vector, route_parameters, res_route[r], comp_route[r]); + + // compute all round trip routes + std::vector comp_route (route_result.size()); + for (auto r = 0; r < route_result.size(); ++r) { + ComputeRoute(phantom_node_vector, route_parameters, route_result[r], comp_route[r]); } + TIMER_STOP(tsp); - SetRuntimeOutput(TIMER_MSEC(tsp), json_result); - //TODO - SetLocPermutationOutput(res_route[0], json_result); - - std::unique_ptr> descriptor; - descriptor = osrm::make_unique>(facade); - descriptor->SetConfig(route_parameters); - auto dist = 0; - for (auto r : comp_route) { - dist += r.shortest_path_length; - // SetGeometry(route_parameters, r, json_result); - descriptor->Run(r, json_result); + // prepare JSON output + + + // create a json object for every trip + osrm::json::Array trip; + for (auto i = 0; i < route_result.size(); ++i) { + std::unique_ptr> descriptor; + descriptor = osrm::make_unique>(facade); + descriptor->SetConfig(route_parameters); + + osrm::json::Object scc_trip; + + // set permutation output + SetLocPermutationOutput(route_result[i], scc_trip); + // set viaroute output + descriptor->Run(comp_route[i], scc_trip); + + trip.values.push_back(scc_trip); } - SetDistanceOutput(dist, json_result); + + json_result.values["trips"] = trip; From 8429a1e792724c7ed50e04fe6b7a119c625013ba Mon Sep 17 00:00:00 2001 From: Huyen Chau Nguyen Date: Thu, 20 Aug 2015 15:32:28 +0200 Subject: [PATCH 081/122] add assertions --- plugins/round_trip.hpp | 37 ++++++++++++++++--- routing_algorithms/tsp_brute_force.hpp | 9 +++++ routing_algorithms/tsp_farthest_insertion.hpp | 24 ++++++++++-- routing_algorithms/tsp_nearest_neighbour.hpp | 6 ++- util/dist_table_wrapper.hpp | 7 +++- 5 files changed, 71 insertions(+), 12 deletions(-) diff --git a/plugins/round_trip.hpp b/plugins/round_trip.hpp index 460fe249dce..5cbff46a760 100644 --- a/plugins/round_trip.hpp +++ b/plugins/round_trip.hpp @@ -120,9 +120,25 @@ template class RoundTripPlugin final : public BasePlugin // => in_range = [0, 5] SCC_Component(std::vector in_component, std::vector in_range) - : component(in_component), range(in_range) { - range.push_back(in_component.size()); - }; + : component(in_component), + range(in_range) { + range.push_back(in_component.size()); + BOOST_ASSERT_MSG(in_component.size() >= in_range.size(), + "scc component and its ranges do not match"); + BOOST_ASSERT_MSG(*std::max_element(in_range.begin(), in_range.end()) < in_component.size(), + "scc component ranges are out of bound"); + BOOST_ASSERT_MSG(*std::min_element(in_range.begin(), in_range.end()) >= 0, + "invalid scc component range"); + BOOST_ASSERT_MSG([&in_range](){ + for (std::size_t r = 0; r < in_range.size() - 1; ++r) { + if (in_range[r] > in_range[r+1]) { + return false; + } + } + return true; + }(), + "invalid component ranges"); + }; // constructor to use when whole graph is one single scc SCC_Component(std::vector in_component) @@ -190,11 +206,15 @@ template class RoundTripPlugin final : public BasePlugin const auto from_node = *it; // if from_node is the last node, compute the route from the last to the first location const auto to_node = std::next(it) != std::end(trip) ? *std::next(it) : *std::begin(trip); + viapoint = PhantomNodes{phantom_node_vector[from_node][0], phantom_node_vector[to_node][0]}; min_route.segment_end_coordinates.emplace_back(viapoint); } search_engine_ptr->shortest_path(min_route.segment_end_coordinates, route_parameters.uturns, min_route); + + BOOST_ASSERT_MSG(min_route.shortest_path_length < INVALID_EDGE_WEIGHT, + "unroutable route"); } int HandleRequest(const RouteParameters &route_parameters, @@ -240,8 +260,11 @@ template class RoundTripPlugin final : public BasePlugin route_result.reserve(scc.GetNumberOfComponents()); TIMER_START(tsp); //run TSP computation for every SCC - for(auto k = 0; k < scc.GetNumberOfComponents(); ++k) { + for (std::size_t k = 0; k < scc.GetNumberOfComponents(); ++k) { const auto component_size = scc.range[k+1] - scc.range[k]; + + BOOST_ASSERT_MSG(component_size >= 0,"invalid component size"); + if (component_size > 1) { std::vector scc_route; NodeIDIterator start = std::begin(scc.component) + scc.range[k]; @@ -266,6 +289,7 @@ template class RoundTripPlugin final : public BasePlugin route_result.push_back(scc_route); } + // use this if output if debugging of route is needed: SimpleLogger().Write() << "Route #" << k << ": " << [&scc_route](){ @@ -276,13 +300,14 @@ template class RoundTripPlugin final : public BasePlugin return s; }(); } else { + // if component only consists of one node, add it to the result routes route_result.push_back({scc.component[scc.range[k]]}); } } // compute all round trip routes std::vector comp_route (route_result.size()); - for (auto r = 0; r < route_result.size(); ++r) { + for (std::size_t r = 0; r < route_result.size(); ++r) { ComputeRoute(phantom_node_vector, route_parameters, route_result[r], comp_route[r]); } @@ -293,7 +318,7 @@ template class RoundTripPlugin final : public BasePlugin // create a json object for every trip osrm::json::Array trip; - for (auto i = 0; i < route_result.size(); ++i) { + for (std::size_t i = 0; i < route_result.size(); ++i) { std::unique_ptr> descriptor; descriptor = osrm::make_unique>(facade); descriptor->SetConfig(route_parameters); diff --git a/routing_algorithms/tsp_brute_force.hpp b/routing_algorithms/tsp_brute_force.hpp index fb6849529fc..e02ac325267 100644 --- a/routing_algorithms/tsp_brute_force.hpp +++ b/routing_algorithms/tsp_brute_force.hpp @@ -52,6 +52,7 @@ namespace osrm namespace tsp { +// computes the distance of a given permutation EdgeWeight ReturnDistance(const DistTableWrapper & dist_table, const std::vector & location_order, const EdgeWeight min_route_dist, @@ -62,9 +63,13 @@ EdgeWeight ReturnDistance(const DistTableWrapper & dist_table, route_dist += dist_table(location_order[i], location_order[(i+1) % component_size]); ++i; } + + BOOST_ASSERT_MSG(route_dist != INVALID_EDGE_WEIGHT, "invalid route found"); + return route_dist; } +// computes the route by computing all permutations and selecting the shortest template std::vector BruteForceTSP(const NodeIDIterator start, const NodeIDIterator end, @@ -79,6 +84,10 @@ std::vector BruteForceTSP(const NodeIDIterator start, EdgeWeight min_route_dist = INVALID_EDGE_WEIGHT; // check length of all possible permutation of the component ids + + BOOST_ASSERT_MSG(*(std::max_element(std::begin(perm), std::end(perm))) < number_of_locations, "invalid node id"); + BOOST_ASSERT_MSG(*(std::min_element(std::begin(perm), std::end(perm))) >= 0, "invalid node id"); + do { const auto new_distance = ReturnDistance(dist_table, perm, min_route_dist, component_size); if (new_distance <= min_route_dist) { diff --git a/routing_algorithms/tsp_farthest_insertion.hpp b/routing_algorithms/tsp_farthest_insertion.hpp index 309471bef3a..9ffc445443b 100644 --- a/routing_algorithms/tsp_farthest_insertion.hpp +++ b/routing_algorithms/tsp_farthest_insertion.hpp @@ -73,12 +73,21 @@ std::pair GetShortestRoundTrip(const NodeID new_loc, const auto dist_to = dist_table(new_loc, *to_node); const auto trip_dist = dist_from + dist_to - dist_table(*from_node, *to_node);; + BOOST_ASSERT_MSG(dist_from != INVALID_EDGE_WEIGHT, + "distance has invalid edge weight"); + BOOST_ASSERT_MSG(dist_to != INVALID_EDGE_WEIGHT, + "distance has invalid edge weight"); + BOOST_ASSERT_MSG(trip_dist >= 0, + "previous trip was not minimal. something's wrong"); + // from all possible insertions to the current trip, choose the shortest of all insertions if (trip_dist < min_trip_distance) { min_trip_distance = trip_dist; next_insert_point_candidate = to_node; } } + BOOST_ASSERT_MSG(min_trip_distance != INVALID_EDGE_WEIGHT, + "trip has invalid edge weight"); return std::make_pair(min_trip_distance, next_insert_point_candidate); } @@ -86,12 +95,15 @@ std::pair GetShortestRoundTrip(const NodeID new_loc, template // given two initial start nodes, find a roundtrip route using the farthest insertion algorithm std::vector FindRoute(const std::size_t & number_of_locations, - const std::size_t & size_of_component, + const std::size_t & component_size, const NodeIDIterator & start, const NodeIDIterator & end, const DistTableWrapper & dist_table, const NodeID & start1, const NodeID & start2) { + BOOST_ASSERT_MSG(number_of_locations >= component_size, + "component size bigger than total number of locations"); + std::vector route; route.reserve(number_of_locations); @@ -104,7 +116,7 @@ std::vector FindRoute(const std::size_t & number_of_locations, route.push_back(start2); // add all other nodes missing (two nodes are already in the initial start trip) - for (std::size_t j = 2; j < size_of_component; ++j) { + for (std::size_t j = 2; j < component_size; ++j) { auto farthest_distance = 0; auto next_node = -1; @@ -116,6 +128,9 @@ std::vector FindRoute(const std::size_t & number_of_locations, if (!visited[*i]) { auto insert_candidate = GetShortestRoundTrip(*i, dist_table, number_of_locations, route); + BOOST_ASSERT_MSG(insert_candidate.first != INVALID_EDGE_WEIGHT, + "shortest round trip is invalid"); + // add the location to the current trip such that it results in the shortest total tour if (insert_candidate.first >= farthest_distance) { farthest_distance = insert_candidate.first; @@ -125,6 +140,8 @@ std::vector FindRoute(const std::size_t & number_of_locations, } } + BOOST_ASSERT_MSG(next_node >= 0, "next node to visit is invalid"); + // mark as visited and insert node visited[next_node] = true; route.insert(next_insert_point, next_node); @@ -169,7 +186,8 @@ std::vector FarthestInsertionTSP(const NodeIDIterator & start, } } } - + BOOST_ASSERT_MSG(max_from < number_of_locations && max_from >= 0, "start node"); + BOOST_ASSERT_MSG(max_to < number_of_locations && max_to >= 0, "start node"); return FindRoute(number_of_locations, component_size, start, end, dist_table, max_from, max_to); } diff --git a/routing_algorithms/tsp_nearest_neighbour.hpp b/routing_algorithms/tsp_nearest_neighbour.hpp index 899eec60d2c..0f42b451416 100644 --- a/routing_algorithms/tsp_nearest_neighbour.hpp +++ b/routing_algorithms/tsp_nearest_neighbour.hpp @@ -84,7 +84,7 @@ std::vector NearestNeighbourTSP(const NodeIDIterator & start, // 3. REPEAT FOR EVERY UNVISITED NODE EdgeWeight trip_dist = 0; - for(auto via_point = 1; via_point < component_size; ++via_point) + for(std::size_t via_point = 1; via_point < component_size; ++via_point) { EdgeWeight min_dist = INVALID_EDGE_WEIGHT; NodeID min_id = SPECIAL_NODEID; @@ -92,12 +92,16 @@ std::vector NearestNeighbourTSP(const NodeIDIterator & start, // 2. FIND NEAREST NEIGHBOUR for (auto next = start; next != end; ++next) { auto curr_dist = dist_table(curr_node, *next); + BOOST_ASSERT_MSG(curr_dist != INVALID_EDGE_WEIGHT, "invalid distance found"); if(!visited[*next] && curr_dist < min_dist) { min_dist = curr_dist; min_id = *next; } } + + BOOST_ASSERT_MSG(min_id != SPECIAL_NODEID, "no next node found"); + visited[min_id] = true; curr_route.push_back(min_id); trip_dist += min_dist; diff --git a/util/dist_table_wrapper.hpp b/util/dist_table_wrapper.hpp index a626cfbc9fe..eda3878c714 100644 --- a/util/dist_table_wrapper.hpp +++ b/util/dist_table_wrapper.hpp @@ -43,7 +43,7 @@ template class DistTableWrapper { DistTableWrapper(std::vector table, std::size_t number_of_nodes) : table_(std::move(table)), number_of_nodes_(number_of_nodes) { BOOST_ASSERT_MSG(table.size() == 0, "table is empty"); - BOOST_ASSERT_MSG(number_of_nodes_ * number_of_nodes_ < table_, "number_of_nodes_ is invalid"); + BOOST_ASSERT_MSG(number_of_nodes_ * number_of_nodes_ <= table_.size(), "number_of_nodes_ is invalid"); }; std::size_t GetNumberOfNodes() const { @@ -57,8 +57,11 @@ template class DistTableWrapper { EdgeWeight operator() (NodeID from, NodeID to) const { BOOST_ASSERT_MSG(from < number_of_nodes_, "from ID is out of bound"); BOOST_ASSERT_MSG(to < number_of_nodes_, "to ID is out of bound"); + const auto index = from * number_of_nodes_ + to; - BOOST_ASSERT_MSG(index < table_size(), "index is out of bound"); + + BOOST_ASSERT_MSG(index < table_.size(), "index is out of bound"); + return table_[index]; } From e6eea67eeb4734149253a54a227db29e0295bd17 Mon Sep 17 00:00:00 2001 From: Huyen Chau Nguyen Date: Thu, 20 Aug 2015 16:07:56 +0200 Subject: [PATCH 082/122] rename all names with round_trip, trip or tsp to trip to standardize the naming --- data_structures/route_parameters.cpp | 4 +- include/osrm/route_parameters.hpp | 6 +- library/osrm_impl.cpp | 2 +- plugins/{round_trip.hpp => trip.hpp} | 34 +++--- ...p_brute_force.hpp => trip_brute_force.hpp} | 12 +- ...ertion.hpp => trip_farthest_insertion.hpp} | 19 ++-- ...ighbour.hpp => trip_nearest_neighbour.hpp} | 18 +-- ...p_tabu_search.hpp => trip_tabu_search.hpp} | 12 +- server/api_grammar.hpp | 8 +- tools/tsp_logs.hpp | 103 ------------------ unit_tests/routing_algorithms/tsp.cpp | 4 +- 11 files changed, 59 insertions(+), 163 deletions(-) rename plugins/{round_trip.hpp => trip.hpp} (92%) rename routing_algorithms/{tsp_brute_force.hpp => trip_brute_force.hpp} (94%) rename routing_algorithms/{tsp_farthest_insertion.hpp => trip_farthest_insertion.hpp} (94%) rename routing_algorithms/{tsp_nearest_neighbour.hpp => trip_nearest_neighbour.hpp} (90%) rename routing_algorithms/{tsp_tabu_search.hpp => trip_tabu_search.hpp} (90%) delete mode 100644 tools/tsp_logs.hpp diff --git a/data_structures/route_parameters.cpp b/data_structures/route_parameters.cpp index 31e91c1fa73..e48d246eb22 100644 --- a/data_structures/route_parameters.cpp +++ b/data_structures/route_parameters.cpp @@ -122,9 +122,9 @@ void RouteParameters::setLanguage(const std::string &language_string) language = language_string; } -void RouteParameters::setTSPAlgo(const std::string &tsp_algo_string) +void RouteParameters::setTripAlgo(const std::string &trip_algo_string) { - tsp_algo = tsp_algo_string; + trip_algo = trip_algo_string; } void RouteParameters::setGeometryFlag(const bool flag) { geometry = flag; } diff --git a/include/osrm/route_parameters.hpp b/include/osrm/route_parameters.hpp index 94ed3a33600..5f5e09f8012 100644 --- a/include/osrm/route_parameters.hpp +++ b/include/osrm/route_parameters.hpp @@ -78,10 +78,10 @@ struct RouteParameters void setCompressionFlag(const bool flag); void addCoordinate(const boost::fusion::vector &received_coordinates); - + void getCoordinatesFromGeometry(const std::string geometry_string); - void setTSPAlgo(const std::string &tsp_algo); + void setTripAlgo(const std::string &trip_algo); short zoom_level; bool print_instructions; @@ -99,7 +99,7 @@ struct RouteParameters std::string output_format; std::string jsonp_parameter; std::string language; - std::string tsp_algo; + std::string trip_algo; std::vector hints; std::vector timestamps; std::vector uturns; diff --git a/library/osrm_impl.cpp b/library/osrm_impl.cpp index 8057069eb88..075c65f790c 100644 --- a/library/osrm_impl.cpp +++ b/library/osrm_impl.cpp @@ -41,7 +41,7 @@ class named_mutex; #include "../plugins/locate.hpp" #include "../plugins/nearest.hpp" #include "../plugins/timestamp.hpp" -#include "../plugins/round_trip.hpp" +#include "../plugins/trip.hpp" #include "../plugins/viaroute.hpp" #include "../plugins/match.hpp" #include "../server/data_structures/datafacade_base.hpp" diff --git a/plugins/round_trip.hpp b/plugins/trip.hpp similarity index 92% rename from plugins/round_trip.hpp rename to plugins/trip.hpp index 5cbff46a760..f2cc89dca73 100644 --- a/plugins/round_trip.hpp +++ b/plugins/trip.hpp @@ -25,16 +25,16 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -#ifndef ROUND_TRIP_HPP -#define ROUND_TRIP_HPP +#ifndef TRIP_HPP +#define TRIP_HPP #include "plugin_base.hpp" #include "../algorithms/object_encoder.hpp" #include "../algorithms/tarjan_scc.hpp" -#include "../routing_algorithms/tsp_nearest_neighbour.hpp" -#include "../routing_algorithms/tsp_farthest_insertion.hpp" -#include "../routing_algorithms/tsp_brute_force.hpp" +#include "../routing_algorithms/trip_nearest_neighbour.hpp" +#include "../routing_algorithms/trip_farthest_insertion.hpp" +#include "../routing_algorithms/trip_brute_force.hpp" #include "../data_structures/query_edge.hpp" #include "../data_structures/search_engine.hpp" #include "../data_structures/matrix_graph_wrapper.hpp" @@ -258,8 +258,8 @@ template class RoundTripPlugin final : public BasePlugin std::vector> route_result; route_result.reserve(scc.GetNumberOfComponents()); - TIMER_START(tsp); - //run TSP computation for every SCC + TIMER_START(trip); + //run Trip computation for every SCC for (std::size_t k = 0; k < scc.GetNumberOfComponents(); ++k) { const auto component_size = scc.range[k+1] - scc.range[k]; @@ -270,22 +270,22 @@ template class RoundTripPlugin final : public BasePlugin NodeIDIterator start = std::begin(scc.component) + scc.range[k]; NodeIDIterator end = std::begin(scc.component) + scc.range[k+1]; - // Compute the TSP with the given algorithm - if (route_parameters.tsp_algo == "BF" && route_parameters.coordinates.size() < BF_MAX_FEASABLE) { + // Compute the Trip with the given algorithm + if (route_parameters.trip_algo == "BF" && route_parameters.coordinates.size() < BF_MAX_FEASABLE) { SimpleLogger().Write() << "Running brute force"; - scc_route = osrm::tsp::BruteForceTSP(start, end, number_of_locations, result_table); + scc_route = osrm::trip::BruteForceTrip(start, end, number_of_locations, result_table); route_result.push_back(scc_route); - } else if (route_parameters.tsp_algo == "NN") { + } else if (route_parameters.trip_algo == "NN") { SimpleLogger().Write() << "Running nearest neighbour"; - scc_route = osrm::tsp::NearestNeighbourTSP(start, end, number_of_locations, result_table); + scc_route = osrm::trip::NearestNeighbourTrip(start, end, number_of_locations, result_table); route_result.push_back(scc_route); - } else if (route_parameters.tsp_algo == "FI") { + } else if (route_parameters.trip_algo == "FI") { SimpleLogger().Write() << "Running farthest insertion"; - scc_route = osrm::tsp::FarthestInsertionTSP(start, end, number_of_locations, result_table); + scc_route = osrm::trip::FarthestInsertionTrip(start, end, number_of_locations, result_table); route_result.push_back(scc_route); } else{ SimpleLogger().Write() << "Running farthest insertion"; - scc_route = osrm::tsp::FarthestInsertionTSP(start, end, number_of_locations, result_table); + scc_route = osrm::trip::FarthestInsertionTrip(start, end, number_of_locations, result_table); route_result.push_back(scc_route); } @@ -311,7 +311,7 @@ template class RoundTripPlugin final : public BasePlugin ComputeRoute(phantom_node_vector, route_parameters, route_result[r], comp_route[r]); } - TIMER_STOP(tsp); + TIMER_STOP(trip); // prepare JSON output @@ -342,4 +342,4 @@ template class RoundTripPlugin final : public BasePlugin }; -#endif // ROUND_TRIP_HPP +#endif // TRIP_HPP diff --git a/routing_algorithms/tsp_brute_force.hpp b/routing_algorithms/trip_brute_force.hpp similarity index 94% rename from routing_algorithms/tsp_brute_force.hpp rename to routing_algorithms/trip_brute_force.hpp index e02ac325267..24319d58360 100644 --- a/routing_algorithms/tsp_brute_force.hpp +++ b/routing_algorithms/trip_brute_force.hpp @@ -25,8 +25,8 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -#ifndef TSP_BRUTE_FORCE_HPP -#define TSP_BRUTE_FORCE_HPP +#ifndef Trip_BRUTE_FORCE_HPP +#define Trip_BRUTE_FORCE_HPP #include "../data_structures/search_engine.hpp" @@ -49,7 +49,7 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. namespace osrm { -namespace tsp +namespace trip { // computes the distance of a given permutation @@ -71,7 +71,7 @@ EdgeWeight ReturnDistance(const DistTableWrapper & dist_table, // computes the route by computing all permutations and selecting the shortest template -std::vector BruteForceTSP(const NodeIDIterator start, +std::vector BruteForceTrip(const NodeIDIterator start, const NodeIDIterator end, const std::size_t number_of_locations, const DistTableWrapper & dist_table) { @@ -99,6 +99,6 @@ std::vector BruteForceTSP(const NodeIDIterator start, return route; } -} //end namespace tsp +} //end namespace trip } //end namespace osrm -#endif // TSP_BRUTE_FORCE_HPP \ No newline at end of file +#endif // Trip_BRUTE_FORCE_HPP \ No newline at end of file diff --git a/routing_algorithms/tsp_farthest_insertion.hpp b/routing_algorithms/trip_farthest_insertion.hpp similarity index 94% rename from routing_algorithms/tsp_farthest_insertion.hpp rename to routing_algorithms/trip_farthest_insertion.hpp index 9ffc445443b..2368192fbc3 100644 --- a/routing_algorithms/tsp_farthest_insertion.hpp +++ b/routing_algorithms/trip_farthest_insertion.hpp @@ -25,14 +25,13 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -#ifndef TSP_FARTHEST_INSERTION_HPP -#define TSP_FARTHEST_INSERTION_HPP +#ifndef TRIP_FARTHEST_INSERTION_HPP +#define TRIP_FARTHEST_INSERTION_HPP #include "../data_structures/search_engine.hpp" #include "../util/string_util.hpp" #include "../util/dist_table_wrapper.hpp" -#include "../tools/tsp_logs.hpp" #include @@ -47,7 +46,7 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. namespace osrm { -namespace tsp +namespace trip { // given a route and a new location, find the best place of insertion and @@ -150,10 +149,10 @@ std::vector FindRoute(const std::size_t & number_of_locations, } template -std::vector FarthestInsertionTSP(const NodeIDIterator & start, - const NodeIDIterator & end, - const std::size_t number_of_locations, - const DistTableWrapper & dist_table) { +std::vector FarthestInsertionTrip(const NodeIDIterator & start, + const NodeIDIterator & end, + const std::size_t number_of_locations, + const DistTableWrapper & dist_table) { ////////////////////////////////////////////////////////////////////////////////////////////////// // START FARTHEST INSERTION HERE // 1. start at a random round trip of 2 locations @@ -192,7 +191,7 @@ std::vector FarthestInsertionTSP(const NodeIDIterator & start, } -} //end namespace tsp +} //end namespace trip } //end namespace osrm -#endif // TSP_FARTHEST_INSERTION_HPP \ No newline at end of file +#endif // TRIP_FARTHEST_INSERTION_HPP \ No newline at end of file diff --git a/routing_algorithms/tsp_nearest_neighbour.hpp b/routing_algorithms/trip_nearest_neighbour.hpp similarity index 90% rename from routing_algorithms/tsp_nearest_neighbour.hpp rename to routing_algorithms/trip_nearest_neighbour.hpp index 0f42b451416..510e8743625 100644 --- a/routing_algorithms/tsp_nearest_neighbour.hpp +++ b/routing_algorithms/trip_nearest_neighbour.hpp @@ -25,8 +25,8 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -#ifndef TSP_NEAREST_NEIGHBOUR_HPP -#define TSP_NEAREST_NEIGHBOUR_HPP +#ifndef TRIP_NEAREST_NEIGHBOUR_HPP +#define TRIP_NEAREST_NEIGHBOUR_HPP #include "../data_structures/search_engine.hpp" @@ -46,13 +46,13 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. namespace osrm { -namespace tsp +namespace trip { template -std::vector NearestNeighbourTSP(const NodeIDIterator & start, - const NodeIDIterator & end, - const std::size_t number_of_locations, - const DistTableWrapper & dist_table) { +std::vector NearestNeighbourTrip(const NodeIDIterator & start, + const NodeIDIterator & end, + const std::size_t number_of_locations, + const DistTableWrapper & dist_table) { ////////////////////////////////////////////////////////////////////////////////////////////////// // START GREEDY NEAREST NEIGHBOUR HERE // 1. grab a random location and mark as starting point @@ -117,6 +117,6 @@ std::vector NearestNeighbourTSP(const NodeIDIterator & start, return route; } -} //end namespace tsp +} //end namespace trip } //end namespace osrm -#endif // TSP_NEAREST_NEIGHBOUR_HPP \ No newline at end of file +#endif // TRIP_NEAREST_NEIGHBOUR_HPP \ No newline at end of file diff --git a/routing_algorithms/tsp_tabu_search.hpp b/routing_algorithms/trip_tabu_search.hpp similarity index 90% rename from routing_algorithms/tsp_tabu_search.hpp rename to routing_algorithms/trip_tabu_search.hpp index 4db7dc470e6..02e063a0e53 100644 --- a/routing_algorithms/tsp_tabu_search.hpp +++ b/routing_algorithms/trip_tabu_search.hpp @@ -25,8 +25,8 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -#ifndef TSP_BRUTE_FORCE_HPP -#define TSP_BRUTE_FORCE_HPP +#ifndef TRIP_BRUTE_FORCE_HPP +#define TRIP_BRUTE_FORCE_HPP #include "../data_structures/search_engine.hpp" @@ -48,10 +48,10 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. namespace osrm { -namespace tsp +namespace trip { -void TabuSearchTSP(std::vector & location, +void TabuSearchTrip(std::vector & location, const PhantomNodeArray & phantom_node_vector, const std::vector & dist_table, InternalRouteResult & min_route, @@ -60,7 +60,7 @@ void TabuSearchTSP(std::vector & location, } -void TabuSearchTSP(const PhantomNodeArray & phantom_node_vector, +void TabuSearchTrip(const PhantomNodeArray & phantom_node_vector, const std::vector & dist_table, InternalRouteResult & min_route, std::vector & min_loc_permutation) { @@ -70,4 +70,4 @@ void TabuSearchTSP(const PhantomNodeArray & phantom_node_vector, } } -#endif // TSP_BRUTE_FORCE_HPP \ No newline at end of file +#endif // TRIP_BRUTE_FORCE_HPP \ No newline at end of file diff --git a/server/api_grammar.hpp b/server/api_grammar.hpp index 8013b82435c..e775c2e8850 100644 --- a/server/api_grammar.hpp +++ b/server/api_grammar.hpp @@ -42,7 +42,7 @@ template struct APIGrammar : qi::grammar> -(uturns); query = ('?') >> (+(zoom | output | jsonp | checksum | location | hint | timestamp | u | cmp | language | instruction | geometry | alt_route | old_API | num_results | - matching_beta | gps_precision | classify | tsp_algo | locs)); + matching_beta | gps_precision | classify | trip_algo | locs)); zoom = (-qi::lit('&')) >> qi::lit('z') >> '=' >> qi::short_[boost::bind(&HandlerT::setZoomLevel, handler, ::_1)]; @@ -85,8 +85,8 @@ template struct APIGrammar : qi::grammar> qi::lit("locs") >> '=' >> stringforPolyline[boost::bind(&HandlerT::getCoordinatesFromGeometry, handler, ::_1)]; - tsp_algo = (-qi::lit('&')) >> qi::lit("tsp_algo") >> '=' >> - string[boost::bind(&HandlerT::setTSPAlgo, handler, ::_1)]; + trip_algo = (-qi::lit('&')) >> qi::lit("trip_algo") >> '=' >> + string[boost::bind(&HandlerT::setTripAlgo, handler, ::_1)]; string = +(qi::char_("a-zA-Z")); stringwithDot = +(qi::char_("a-zA-Z0-9_.-")); @@ -98,7 +98,7 @@ template struct APIGrammar : qi::grammar api_call, query; qi::rule service, zoom, output, string, jsonp, checksum, location, hint, timestamp, stringwithDot, stringwithPercent, language, instruction, geometry, cmp, alt_route, u, - uturns, old_API, num_results, matching_beta, gps_precision, classify, locs, stringforPolyline, tsp_algo; + uturns, old_API, num_results, matching_beta, gps_precision, classify, locs, stringforPolyline, trip_algo; HandlerT *handler; }; diff --git a/tools/tsp_logs.hpp b/tools/tsp_logs.hpp deleted file mode 100644 index fcc9af09666..00000000000 --- a/tools/tsp_logs.hpp +++ /dev/null @@ -1,103 +0,0 @@ -/* - -Copyright (c) 2015, Project OSRM contributors -All rights reserved. - -Redistribution and use in source and binary forms, with or without modification, -are permitted provided that the following conditions are met: - -Redistributions of source code must retain the above copyright notice, this list -of conditions and the following disclaimer. -Redistributions in binary form must reproduce the above copyright notice, this -list of conditions and the following disclaimer in the documentation and/or -other materials provided with the distribution. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR -ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES -(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON -ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -*/ - -#ifndef TSP_LOGS_HPP -#define TSP_LOGS_HPP - - -#include "../data_structures/search_engine.hpp" -#include "../util/string_util.hpp" - -#include - -#include - -#include -#include -#include -#include -#include -#include "../util/simple_logger.hpp" - - -namespace osrm -{ -namespace tsp -{ - - -inline void PrintDistTable(const std::vector & dist_table, const int number_of_locations) { - int j = 0; - for (auto i = dist_table.begin(); i != dist_table.end(); ++i){ - if (j % number_of_locations == 0) { - std::cout << std::endl; - } - std::cout << std::setw(6) << *i << " "; - ++j; - } - std::cout << std::endl; -} - -bool CheckSymmetricTable(const std::vector & dist_table, const int number_of_locations) { - bool is_quadratic = true; - for (int i = 0; i < number_of_locations; ++i) { - for(int j = 0; j < number_of_locations; ++j) { - int a = *(dist_table.begin() + (i * number_of_locations) + j); - int b = *(dist_table.begin() + (j * number_of_locations) + i); - if (a !=b) { - is_quadratic = false; - } - } - } - return is_quadratic; -} - -void LogRoute(std::vector location_ids){ - SimpleLogger().Write() << "LOC ORDER"; - for (auto x : location_ids) { - SimpleLogger().Write() << x; - } -} - -int ReturnDistanceFI(const std::vector & dist_table, std::list current_trip, const int number_of_locations) { - int route_dist = 0; - - // compute length and stop if length is longer than route already found - for (auto i = current_trip.begin(); i != std::prev(current_trip.end()); ++i) { - //get distance from location i to location i+1 - route_dist += *(dist_table.begin() + (*i * number_of_locations) + *std::next(i)); - } - //get distance from last location to first location - route_dist += *(dist_table.begin() + (*std::prev(current_trip.end()) * number_of_locations) + current_trip.front()); - - return route_dist; -} - - -} -} -#endif // TSP_LOGS_HPP \ No newline at end of file diff --git a/unit_tests/routing_algorithms/tsp.cpp b/unit_tests/routing_algorithms/tsp.cpp index 0c6a52462f7..fde22295c05 100644 --- a/unit_tests/routing_algorithms/tsp.cpp +++ b/unit_tests/routing_algorithms/tsp.cpp @@ -25,7 +25,7 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -#include "../../routing_algorithms/tsp_brute_force.hpp" +#include "../../routing_algorithms/trip_brute_force.hpp" #include #include @@ -35,7 +35,7 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include -BOOST_AUTO_TEST_SUITE(tsp) +BOOST_AUTO_TEST_SUITE(trip) // BOOST_AUTO_TEST_CASE(check_distance_computation) // { From e773a80b068bfd524b0dd2121ff8491430c2dd32 Mon Sep 17 00:00:00 2001 From: Huyen Chau Nguyen Date: Thu, 20 Aug 2015 16:42:50 +0200 Subject: [PATCH 083/122] remove possibility to choose algorithm but only use brute force and farthest insertion --- data_structures/route_parameters.cpp | 4 ---- include/osrm/route_parameters.hpp | 3 --- plugins/trip.hpp | 15 ++------------- routing_algorithms/trip_brute_force.hpp | 6 +++--- server/api_grammar.hpp | 6 ++---- 5 files changed, 7 insertions(+), 27 deletions(-) diff --git a/data_structures/route_parameters.cpp b/data_structures/route_parameters.cpp index e48d246eb22..52e900ec6ff 100644 --- a/data_structures/route_parameters.cpp +++ b/data_structures/route_parameters.cpp @@ -122,10 +122,6 @@ void RouteParameters::setLanguage(const std::string &language_string) language = language_string; } -void RouteParameters::setTripAlgo(const std::string &trip_algo_string) -{ - trip_algo = trip_algo_string; -} void RouteParameters::setGeometryFlag(const bool flag) { geometry = flag; } diff --git a/include/osrm/route_parameters.hpp b/include/osrm/route_parameters.hpp index 5f5e09f8012..5e16b919f28 100644 --- a/include/osrm/route_parameters.hpp +++ b/include/osrm/route_parameters.hpp @@ -81,8 +81,6 @@ struct RouteParameters void getCoordinatesFromGeometry(const std::string geometry_string); - void setTripAlgo(const std::string &trip_algo); - short zoom_level; bool print_instructions; bool alternate_route; @@ -99,7 +97,6 @@ struct RouteParameters std::string output_format; std::string jsonp_parameter; std::string language; - std::string trip_algo; std::vector hints; std::vector timestamps; std::vector uturns; diff --git a/plugins/trip.hpp b/plugins/trip.hpp index f2cc89dca73..1ae6bc57ace 100644 --- a/plugins/trip.hpp +++ b/plugins/trip.hpp @@ -270,21 +270,10 @@ template class RoundTripPlugin final : public BasePlugin NodeIDIterator start = std::begin(scc.component) + scc.range[k]; NodeIDIterator end = std::begin(scc.component) + scc.range[k+1]; - // Compute the Trip with the given algorithm - if (route_parameters.trip_algo == "BF" && route_parameters.coordinates.size() < BF_MAX_FEASABLE) { - SimpleLogger().Write() << "Running brute force"; + if (component_size < BF_MAX_FEASABLE) { scc_route = osrm::trip::BruteForceTrip(start, end, number_of_locations, result_table); route_result.push_back(scc_route); - } else if (route_parameters.trip_algo == "NN") { - SimpleLogger().Write() << "Running nearest neighbour"; - scc_route = osrm::trip::NearestNeighbourTrip(start, end, number_of_locations, result_table); - route_result.push_back(scc_route); - } else if (route_parameters.trip_algo == "FI") { - SimpleLogger().Write() << "Running farthest insertion"; - scc_route = osrm::trip::FarthestInsertionTrip(start, end, number_of_locations, result_table); - route_result.push_back(scc_route); - } else{ - SimpleLogger().Write() << "Running farthest insertion"; + } else { scc_route = osrm::trip::FarthestInsertionTrip(start, end, number_of_locations, result_table); route_result.push_back(scc_route); } diff --git a/routing_algorithms/trip_brute_force.hpp b/routing_algorithms/trip_brute_force.hpp index 24319d58360..0bd1c725c78 100644 --- a/routing_algorithms/trip_brute_force.hpp +++ b/routing_algorithms/trip_brute_force.hpp @@ -55,11 +55,11 @@ namespace trip // computes the distance of a given permutation EdgeWeight ReturnDistance(const DistTableWrapper & dist_table, const std::vector & location_order, - const EdgeWeight min_route_dist, - const std::size_t component_size) { + const EdgeWeight & min_route_dist, + const std::size_t & component_size) { EdgeWeight route_dist = 0; std::size_t i = 0; - while (i < location_order.size()) { + while (i < location_order.size() && (route_dist < min_route_dist)) { route_dist += dist_table(location_order[i], location_order[(i+1) % component_size]); ++i; } diff --git a/server/api_grammar.hpp b/server/api_grammar.hpp index e775c2e8850..325f4c356c1 100644 --- a/server/api_grammar.hpp +++ b/server/api_grammar.hpp @@ -42,7 +42,7 @@ template struct APIGrammar : qi::grammar> -(uturns); query = ('?') >> (+(zoom | output | jsonp | checksum | location | hint | timestamp | u | cmp | language | instruction | geometry | alt_route | old_API | num_results | - matching_beta | gps_precision | classify | trip_algo | locs)); + matching_beta | gps_precision | classify | locs)); zoom = (-qi::lit('&')) >> qi::lit('z') >> '=' >> qi::short_[boost::bind(&HandlerT::setZoomLevel, handler, ::_1)]; @@ -85,8 +85,6 @@ template struct APIGrammar : qi::grammar> qi::lit("locs") >> '=' >> stringforPolyline[boost::bind(&HandlerT::getCoordinatesFromGeometry, handler, ::_1)]; - trip_algo = (-qi::lit('&')) >> qi::lit("trip_algo") >> '=' >> - string[boost::bind(&HandlerT::setTripAlgo, handler, ::_1)]; string = +(qi::char_("a-zA-Z")); stringwithDot = +(qi::char_("a-zA-Z0-9_.-")); @@ -98,7 +96,7 @@ template struct APIGrammar : qi::grammar api_call, query; qi::rule service, zoom, output, string, jsonp, checksum, location, hint, timestamp, stringwithDot, stringwithPercent, language, instruction, geometry, cmp, alt_route, u, - uturns, old_API, num_results, matching_beta, gps_precision, classify, locs, stringforPolyline, trip_algo; + uturns, old_API, num_results, matching_beta, gps_precision, classify, locs, stringforPolyline; HandlerT *handler; }; From 74e00cf652afc69b9fd5f40c3ff64a7353550714 Mon Sep 17 00:00:00 2001 From: Huyen Chau Nguyen Date: Fri, 28 Aug 2015 20:18:15 +0200 Subject: [PATCH 084/122] fix some small issues: remove empty unit test remove compiler directives move trip related files from routing_algorithms to algorithms run clang-format on files fix all std::size_t related issues improve code by adding std::move()s clean up includes fixing several code stye and improvement issues add several small code improvements return single scc in SplitUnaccessibleLocations() when theres only one change ComputeRoute() to return an InternalRouteResult by value improve some code style issues --- algorithms/tarjan_scc.hpp | 13 +- .../trip_brute_force.hpp | 58 ++-- .../trip_farthest_insertion.hpp | 117 ++++---- .../trip_nearest_neighbour.hpp | 36 +-- .../trip_tabu_search.hpp | 35 +-- data_structures/matrix_graph_wrapper.hpp | 39 +-- data_structures/route_parameters.cpp | 1 - library/osrm_impl.hpp | 4 - plugins/trip.hpp | 273 ++++++++++-------- unit_tests/routing_algorithms/tsp.cpp | 45 --- util/dist_table_wrapper.hpp | 54 ++-- 11 files changed, 320 insertions(+), 355 deletions(-) rename {routing_algorithms => algorithms}/trip_brute_force.hpp (70%) rename {routing_algorithms => algorithms}/trip_farthest_insertion.hpp (70%) rename {routing_algorithms => algorithms}/trip_nearest_neighbour.hpp (83%) rename {routing_algorithms => algorithms}/trip_tabu_search.hpp (72%) delete mode 100644 unit_tests/routing_algorithms/tsp.cpp diff --git a/algorithms/tarjan_scc.hpp b/algorithms/tarjan_scc.hpp index 6f5bc1563a3..f3fab7fd57c 100644 --- a/algorithms/tarjan_scc.hpp +++ b/algorithms/tarjan_scc.hpp @@ -40,19 +40,12 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "../util/timing_util.hpp" #include - #include - -#include - #include #include #include -#include -#include #include -#include template class TarjanSCC { @@ -153,7 +146,8 @@ template class TarjanSCC else { processing_node_before_recursion[v] = true; - tarjan_node_list[u].low_link = std::min(tarjan_node_list[u].low_link, tarjan_node_list[v].low_link); + tarjan_node_list[u].low_link = + std::min(tarjan_node_list[u].low_link, tarjan_node_list[v].low_link); // after recursion, lets do cycle checking // Check if we found a cycle. This is the bottom part of the recursion if (tarjan_node_list[v].low_link == tarjan_node_list[v].index) @@ -193,7 +187,6 @@ template class TarjanSCC }); } - std::size_t get_number_of_components() const { return component_size_vector.size(); } std::size_t get_size_one_count() const { return size_one_counter; } @@ -204,8 +197,6 @@ template class TarjanSCC } unsigned get_component_id(const NodeID node) const { return components_index[node]; } - - }; #endif /* TARJAN_SCC_HPP */ diff --git a/routing_algorithms/trip_brute_force.hpp b/algorithms/trip_brute_force.hpp similarity index 70% rename from routing_algorithms/trip_brute_force.hpp rename to algorithms/trip_brute_force.hpp index 0bd1c725c78..5c8eb03fa14 100644 --- a/routing_algorithms/trip_brute_force.hpp +++ b/algorithms/trip_brute_force.hpp @@ -25,27 +25,21 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -#ifndef Trip_BRUTE_FORCE_HPP -#define Trip_BRUTE_FORCE_HPP - +#ifndef TRIP_BRUTE_FORCE_HPP +#define TRIP_BRUTE_FORCE_HPP #include "../data_structures/search_engine.hpp" -#include "../util/string_util.hpp" #include "../util/dist_table_wrapper.hpp" +#include "../util/simple_logger.hpp" #include #include - #include #include +#include #include #include -#include -#include "../util/simple_logger.hpp" - - - namespace osrm { @@ -53,28 +47,31 @@ namespace trip { // computes the distance of a given permutation -EdgeWeight ReturnDistance(const DistTableWrapper & dist_table, - const std::vector & location_order, - const EdgeWeight & min_route_dist, - const std::size_t & component_size) { +EdgeWeight ReturnDistance(const DistTableWrapper &dist_table, + const std::vector &location_order, + const EdgeWeight min_route_dist, + const std::size_t component_size) +{ EdgeWeight route_dist = 0; std::size_t i = 0; - while (i < location_order.size() && (route_dist < min_route_dist)) { - route_dist += dist_table(location_order[i], location_order[(i+1) % component_size]); + while (i < location_order.size() && (route_dist < min_route_dist)) + { + route_dist += dist_table(location_order[i], location_order[(i + 1) % component_size]); + BOOST_ASSERT_MSG(dist_table(location_order[i], location_order[(i + 1) % component_size]) + != INVALID_EDGE_WEIGHT, "invalid route found"); ++i; } - BOOST_ASSERT_MSG(route_dist != INVALID_EDGE_WEIGHT, "invalid route found"); - return route_dist; } // computes the route by computing all permutations and selecting the shortest template std::vector BruteForceTrip(const NodeIDIterator start, - const NodeIDIterator end, - const std::size_t number_of_locations, - const DistTableWrapper & dist_table) { + const NodeIDIterator end, + const std::size_t number_of_locations, + const DistTableWrapper &dist_table) +{ const auto component_size = std::distance(start, end); std::vector perm(start, end); @@ -85,20 +82,25 @@ std::vector BruteForceTrip(const NodeIDIterator start, // check length of all possible permutation of the component ids - BOOST_ASSERT_MSG(*(std::max_element(std::begin(perm), std::end(perm))) < number_of_locations, "invalid node id"); + BOOST_ASSERT_MSG(perm.size() > 0, + "no permutation given"); + BOOST_ASSERT_MSG(*(std::max_element(std::begin(perm), std::end(perm))) < number_of_locations, + "invalid node id"); BOOST_ASSERT_MSG(*(std::min_element(std::begin(perm), std::end(perm))) >= 0, "invalid node id"); - do { + do + { const auto new_distance = ReturnDistance(dist_table, perm, min_route_dist, component_size); - if (new_distance <= min_route_dist) { + if (new_distance <= min_route_dist) + { min_route_dist = new_distance; route = perm; } - } while(std::next_permutation(std::begin(perm), std::end(perm))); + } while (std::next_permutation(std::begin(perm), std::end(perm))); return route; } -} //end namespace trip -} //end namespace osrm -#endif // Trip_BRUTE_FORCE_HPP \ No newline at end of file +} // end namespace trip +} // end namespace osrm +#endif // TRIP_BRUTE_FORCE_HPP \ No newline at end of file diff --git a/routing_algorithms/trip_farthest_insertion.hpp b/algorithms/trip_farthest_insertion.hpp similarity index 70% rename from routing_algorithms/trip_farthest_insertion.hpp rename to algorithms/trip_farthest_insertion.hpp index 2368192fbc3..8f5c59ebbce 100644 --- a/routing_algorithms/trip_farthest_insertion.hpp +++ b/algorithms/trip_farthest_insertion.hpp @@ -28,22 +28,17 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #ifndef TRIP_FARTHEST_INSERTION_HPP #define TRIP_FARTHEST_INSERTION_HPP - #include "../data_structures/search_engine.hpp" -#include "../util/string_util.hpp" #include "../util/dist_table_wrapper.hpp" #include #include - #include #include #include #include -#include - namespace osrm { namespace trip @@ -51,55 +46,59 @@ namespace trip // given a route and a new location, find the best place of insertion and // check the distance of roundtrip when the new location is additionally visited -using NodeIDIter = typename std::vector::iterator; -std::pair GetShortestRoundTrip(const NodeID new_loc, - const DistTableWrapper & dist_table, - const std::size_t number_of_locations, - std::vector & route){ +using NodeIDIter = std::vector::iterator; +std::pair +GetShortestRoundTrip(const NodeID new_loc, + const DistTableWrapper &dist_table, + const std::size_t number_of_locations, + std::vector &route) +{ auto min_trip_distance = INVALID_EDGE_WEIGHT; NodeIDIter next_insert_point_candidate; // for all nodes in the current trip find the best insertion resulting in the shortest path // assert min 2 nodes in route - for (auto from_node = std::begin(route); from_node != std::end(route); ++from_node) { + const auto start = std::begin(route); + const auto end = std::end(route); + for (auto from_node = start; from_node != end; ++from_node) + { auto to_node = std::next(from_node); - if (to_node == std::end(route)) { - to_node = std::begin(route); + if (to_node == end) + { + to_node = start; } const auto dist_from = dist_table(*from_node, new_loc); const auto dist_to = dist_table(new_loc, *to_node); - const auto trip_dist = dist_from + dist_to - dist_table(*from_node, *to_node);; + const auto trip_dist = dist_from + dist_to - dist_table(*from_node, *to_node); - BOOST_ASSERT_MSG(dist_from != INVALID_EDGE_WEIGHT, - "distance has invalid edge weight"); - BOOST_ASSERT_MSG(dist_to != INVALID_EDGE_WEIGHT, - "distance has invalid edge weight"); - BOOST_ASSERT_MSG(trip_dist >= 0, - "previous trip was not minimal. something's wrong"); + BOOST_ASSERT_MSG(dist_from != INVALID_EDGE_WEIGHT, "distance has invalid edge weight"); + BOOST_ASSERT_MSG(dist_to != INVALID_EDGE_WEIGHT, "distance has invalid edge weight"); + BOOST_ASSERT_MSG(trip_dist >= 0, "previous trip was not minimal. something's wrong"); // from all possible insertions to the current trip, choose the shortest of all insertions - if (trip_dist < min_trip_distance) { + if (trip_dist < min_trip_distance) + { min_trip_distance = trip_dist; next_insert_point_candidate = to_node; } } - BOOST_ASSERT_MSG(min_trip_distance != INVALID_EDGE_WEIGHT, - "trip has invalid edge weight"); + BOOST_ASSERT_MSG(min_trip_distance != INVALID_EDGE_WEIGHT, "trip has invalid edge weight"); return std::make_pair(min_trip_distance, next_insert_point_candidate); } template // given two initial start nodes, find a roundtrip route using the farthest insertion algorithm -std::vector FindRoute(const std::size_t & number_of_locations, - const std::size_t & component_size, - const NodeIDIterator & start, - const NodeIDIterator & end, - const DistTableWrapper & dist_table, - const NodeID & start1, - const NodeID & start2) { +std::vector FindRoute(const std::size_t &number_of_locations, + const std::size_t &component_size, + const NodeIDIterator &start, + const NodeIDIterator &end, + const DistTableWrapper &dist_table, + const NodeID &start1, + const NodeID &start2) +{ BOOST_ASSERT_MSG(number_of_locations >= component_size, "component size bigger than total number of locations"); @@ -115,23 +114,29 @@ std::vector FindRoute(const std::size_t & number_of_locations, route.push_back(start2); // add all other nodes missing (two nodes are already in the initial start trip) - for (std::size_t j = 2; j < component_size; ++j) { + for (std::size_t j = 2; j < component_size; ++j) + { auto farthest_distance = 0; auto next_node = -1; NodeIDIter next_insert_point; // find unvisited loc i that is the farthest away from all other visited locs - for (auto i = start; i != end; ++i) { + for (auto i = start; i != end; ++i) + { // find the shortest distance from i to all visited nodes - if (!visited[*i]) { - auto insert_candidate = GetShortestRoundTrip(*i, dist_table, number_of_locations, route); + if (!visited[*i]) + { + const auto insert_candidate = + GetShortestRoundTrip(*i, dist_table, number_of_locations, route); BOOST_ASSERT_MSG(insert_candidate.first != INVALID_EDGE_WEIGHT, - "shortest round trip is invalid"); + "shortest round trip is invalid"); - // add the location to the current trip such that it results in the shortest total tour - if (insert_candidate.first >= farthest_distance) { + // add the location to the current trip such that it results in the shortest total + // tour + if (insert_candidate.first >= farthest_distance) + { farthest_distance = insert_candidate.first; next_node = *i; next_insert_point = insert_candidate.second; @@ -149,14 +154,16 @@ std::vector FindRoute(const std::size_t & number_of_locations, } template -std::vector FarthestInsertionTrip(const NodeIDIterator & start, - const NodeIDIterator & end, +std::vector FarthestInsertionTrip(const NodeIDIterator &start, + const NodeIDIterator &end, const std::size_t number_of_locations, - const DistTableWrapper & dist_table) { + const DistTableWrapper &dist_table) +{ ////////////////////////////////////////////////////////////////////////////////////////////////// // START FARTHEST INSERTION HERE // 1. start at a random round trip of 2 locations - // 2. find the location that is the farthest away from the visited locations and whose insertion will make the round trip the longest + // 2. find the location that is the farthest away from the visited locations and whose insertion + // will make the round trip the longest // 3. add the found location to the current round trip such that round trip is the shortest // 4. repeat 2-3 until all locations are visited // 5. DONE! @@ -166,18 +173,25 @@ std::vector FarthestInsertionTrip(const NodeIDIterator & start, auto max_from = -1; auto max_to = -1; - if (component_size == number_of_locations) { - // find the pair of location with the biggest distance and make the pair the initial start trip - const auto index = std::distance(std::begin(dist_table), std::max_element(std::begin(dist_table), std::end(dist_table))); + if (component_size == number_of_locations) + { + // find the pair of location with the biggest distance and make the pair the initial start + // trip + const auto index = std::distance( + std::begin(dist_table), std::max_element(std::begin(dist_table), std::end(dist_table))); max_from = index / number_of_locations; max_to = index % number_of_locations; - - } else { + } + else + { auto max_dist = 0; - for (auto x = start; x != end; ++x) { - for (auto y = start; y != end; ++y) { + for (auto x = start; x != end; ++x) + { + for (auto y = start; y != end; ++y) + { const auto xy_dist = dist_table(*x, *y); - if (xy_dist > max_dist) { + if (xy_dist > max_dist) + { max_dist = xy_dist; max_from = *x; max_to = *y; @@ -190,8 +204,7 @@ std::vector FarthestInsertionTrip(const NodeIDIterator & start, return FindRoute(number_of_locations, component_size, start, end, dist_table, max_from, max_to); } - -} //end namespace trip -} //end namespace osrm +} // end namespace trip +} // end namespace osrm #endif // TRIP_FARTHEST_INSERTION_HPP \ No newline at end of file diff --git a/routing_algorithms/trip_nearest_neighbour.hpp b/algorithms/trip_nearest_neighbour.hpp similarity index 83% rename from routing_algorithms/trip_nearest_neighbour.hpp rename to algorithms/trip_nearest_neighbour.hpp index 510e8743625..0ae179284de 100644 --- a/routing_algorithms/trip_nearest_neighbour.hpp +++ b/algorithms/trip_nearest_neighbour.hpp @@ -28,9 +28,7 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #ifndef TRIP_NEAREST_NEIGHBOUR_HPP #define TRIP_NEAREST_NEIGHBOUR_HPP - #include "../data_structures/search_engine.hpp" -#include "../util/string_util.hpp" #include "../util/simple_logger.hpp" #include "../util/dist_table_wrapper.hpp" @@ -42,17 +40,16 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include #include - - namespace osrm { namespace trip { template -std::vector NearestNeighbourTrip(const NodeIDIterator & start, - const NodeIDIterator & end, +std::vector NearestNeighbourTrip(const NodeIDIterator &start, + const NodeIDIterator &end, const std::size_t number_of_locations, - const DistTableWrapper & dist_table) { + const DistTableWrapper &dist_table) +{ ////////////////////////////////////////////////////////////////////////////////////////////////// // START GREEDY NEAREST NEIGHBOUR HERE // 1. grab a random location and mark as starting point @@ -70,7 +67,7 @@ std::vector NearestNeighbourTrip(const NodeIDIterator & start, auto shortest_trip_distance = INVALID_EDGE_WEIGHT; // ALWAYS START AT ANOTHER STARTING POINT - for(auto start_node = start; start_node != end; ++start_node) + for (auto start_node = start; start_node != end; ++start_node) { NodeID curr_node = *start_node; @@ -84,17 +81,18 @@ std::vector NearestNeighbourTrip(const NodeIDIterator & start, // 3. REPEAT FOR EVERY UNVISITED NODE EdgeWeight trip_dist = 0; - for(std::size_t via_point = 1; via_point < component_size; ++via_point) + for (std::size_t via_point = 1; via_point < component_size; ++via_point) { EdgeWeight min_dist = INVALID_EDGE_WEIGHT; NodeID min_id = SPECIAL_NODEID; // 2. FIND NEAREST NEIGHBOUR - for (auto next = start; next != end; ++next) { - auto curr_dist = dist_table(curr_node, *next); + for (auto next = start; next != end; ++next) + { + const auto curr_dist = dist_table(curr_node, *next); BOOST_ASSERT_MSG(curr_dist != INVALID_EDGE_WEIGHT, "invalid distance found"); - if(!visited[*next] && - curr_dist < min_dist) { + if (!visited[*next] && curr_dist < min_dist) + { min_dist = curr_dist; min_id = *next; } @@ -108,15 +106,17 @@ std::vector NearestNeighbourTrip(const NodeIDIterator & start, curr_node = min_id; } - // check round trip with this starting point is shorter than the shortest round trip found till now - if (trip_dist < shortest_trip_distance) { + // check round trip with this starting point is shorter than the shortest round trip found + // till now + if (trip_dist < shortest_trip_distance) + { shortest_trip_distance = trip_dist; - route = curr_route; + route = std::move(curr_route); } } return route; } -} //end namespace trip -} //end namespace osrm +} // end namespace trip +} // end namespace osrm #endif // TRIP_NEAREST_NEIGHBOUR_HPP \ No newline at end of file diff --git a/routing_algorithms/trip_tabu_search.hpp b/algorithms/trip_tabu_search.hpp similarity index 72% rename from routing_algorithms/trip_tabu_search.hpp rename to algorithms/trip_tabu_search.hpp index 02e063a0e53..32f50fc59f8 100644 --- a/routing_algorithms/trip_tabu_search.hpp +++ b/algorithms/trip_tabu_search.hpp @@ -28,46 +28,37 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #ifndef TRIP_BRUTE_FORCE_HPP #define TRIP_BRUTE_FORCE_HPP - #include "../data_structures/search_engine.hpp" -#include "../util/string_util.hpp" +#include "../util/simple_logger.hpp" #include #include - #include #include #include #include -#include -#include "../util/simple_logger.hpp" - - - namespace osrm { namespace trip { -void TabuSearchTrip(std::vector & location, - const PhantomNodeArray & phantom_node_vector, - const std::vector & dist_table, - InternalRouteResult & min_route, - std::vector & min_loc_permutation) { - - +// todo: yet to be implemented +void TabuSearchTrip(std::vector &location, + const PhantomNodeArray &phantom_node_vector, + const std::vector &dist_table, + InternalRouteResult &min_route, + std::vector &min_loc_permutation) +{ } -void TabuSearchTrip(const PhantomNodeArray & phantom_node_vector, - const std::vector & dist_table, - InternalRouteResult & min_route, - std::vector & min_loc_permutation) { - - +void TabuSearchTrip(const PhantomNodeArray &phantom_node_vector, + const std::vector &dist_table, + InternalRouteResult &min_route, + std::vector &min_loc_permutation) +{ } - } } #endif // TRIP_BRUTE_FORCE_HPP \ No newline at end of file diff --git a/data_structures/matrix_graph_wrapper.hpp b/data_structures/matrix_graph_wrapper.hpp index 95b0e4c860a..f8b3e65bb2f 100644 --- a/data_structures/matrix_graph_wrapper.hpp +++ b/data_structures/matrix_graph_wrapper.hpp @@ -29,37 +29,42 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #define MATRIX_GRAPH_WRAPPER_H #include +#include +#include -//This Wrapper provides all methods that are needed for TarjanSCC, when the graph is given in a -//matrix representation (e.g. as output from a distance table call) +#include "../typedefs.h" -template class MatrixGraphWrapper { -public: +// This Wrapper provides all methods that are needed for TarjanSCC, when the graph is given in a +// matrix representation (e.g. as output from a distance table call) - MatrixGraphWrapper(std::vector table, const std::size_t number_of_nodes) : table_(table), number_of_nodes_(number_of_nodes) {}; +template class MatrixGraphWrapper +{ + public: + MatrixGraphWrapper(std::vector table, const std::size_t number_of_nodes) + : table_(std::move(table)), number_of_nodes_(number_of_nodes){}; - std::size_t GetNumberOfNodes() const { - return number_of_nodes_; - } + std::size_t GetNumberOfNodes() const { return number_of_nodes_; } + + std::vector GetAdjacentEdgeRange(const NodeID node) const + { - std::vector GetAdjacentEdgeRange(const NodeID node) const { std::vector edges; - for (auto i = 0; i < number_of_nodes_; ++i) { - if (*(std::begin(table_) + node * number_of_nodes_ + i) != INVALID_EDGE_WEIGHT) { + // find all valid adjacent edges and move to vector `edges` + for (std::size_t i = 0; i < number_of_nodes_; ++i) + { + if (*(std::begin(table_) + node * number_of_nodes_ + i) != INVALID_EDGE_WEIGHT) + { edges.push_back(i); } } return edges; } - EdgeWeight GetTarget(const EdgeWeight edge) const { - return edge; - } + EdgeWeight GetTarget(const EdgeWeight edge) const { return edge; } -private: - std::vector table_; + private: + const std::vector table_; const std::size_t number_of_nodes_; }; - #endif // MATRIX_GRAPH_WRAPPER_H diff --git a/data_structures/route_parameters.cpp b/data_structures/route_parameters.cpp index 52e900ec6ff..86b9553580c 100644 --- a/data_structures/route_parameters.cpp +++ b/data_structures/route_parameters.cpp @@ -122,7 +122,6 @@ void RouteParameters::setLanguage(const std::string &language_string) language = language_string; } - void RouteParameters::setGeometryFlag(const bool flag) { geometry = flag; } void RouteParameters::setCompressionFlag(const bool flag) { compression = flag; } diff --git a/library/osrm_impl.hpp b/library/osrm_impl.hpp index 6984c5ff27b..a736c042f6e 100644 --- a/library/osrm_impl.hpp +++ b/library/osrm_impl.hpp @@ -28,10 +28,6 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #ifndef OSRM_IMPL_HPP #define OSRM_IMPL_HPP -// #if __cplusplus > 199711L -// #define register // Deprecated in C++11. -// #endif // #if __cplusplus > 199711L - class BasePlugin; struct RouteParameters; diff --git a/plugins/trip.hpp b/plugins/trip.hpp index 1ae6bc57ace..860f036452f 100644 --- a/plugins/trip.hpp +++ b/plugins/trip.hpp @@ -32,22 +32,19 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "../algorithms/object_encoder.hpp" #include "../algorithms/tarjan_scc.hpp" -#include "../routing_algorithms/trip_nearest_neighbour.hpp" -#include "../routing_algorithms/trip_farthest_insertion.hpp" -#include "../routing_algorithms/trip_brute_force.hpp" -#include "../data_structures/query_edge.hpp" +#include "../algorithms/trip_nearest_neighbour.hpp" +#include "../algorithms/trip_farthest_insertion.hpp" +#include "../algorithms/trip_brute_force.hpp" #include "../data_structures/search_engine.hpp" -#include "../data_structures/matrix_graph_wrapper.hpp" -#include "../data_structures/restriction.hpp" -#include "../data_structures/restriction_map.hpp" -#include "../descriptors/descriptor_base.hpp" -#include "../descriptors/json_descriptor.hpp" -#include "../util/json_renderer.hpp" +#include "../data_structures/matrix_graph_wrapper.hpp" // wrapper to use tarjan + // scc on dist table +#include "../descriptors/descriptor_base.hpp" // to make json output +#include "../descriptors/json_descriptor.hpp" // to make json output #include "../util/make_unique.hpp" -#include "../util/string_util.hpp" -#include "../util/timing_util.hpp" -#include "../util/simple_logger.hpp" -#include "../util/dist_table_wrapper.hpp" +#include "../util/timing_util.hpp" // to time runtime +#include "../util/simple_logger.hpp" // for logging output +#include "../util/dist_table_wrapper.hpp" // to access the dist + // table more easily #include #include @@ -55,16 +52,11 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include #include #include -#include #include #include #include -#include -#include #include -#include - template class RoundTripPlugin final : public BasePlugin { private: @@ -73,22 +65,25 @@ template class RoundTripPlugin final : public BasePlugin std::unique_ptr> search_engine_ptr; public: - explicit RoundTripPlugin(DataFacadeT *facade) - : descriptor_string("trip"), facade(facade) + explicit RoundTripPlugin(DataFacadeT *facade) : descriptor_string("trip"), facade(facade) { search_engine_ptr = osrm::make_unique>(facade); } const std::string GetDescriptor() const override final { return descriptor_string; } - void GetPhantomNodes(const RouteParameters &route_parameters, PhantomNodeArray & phantom_node_vector) { + void GetPhantomNodes(const RouteParameters &route_parameters, + PhantomNodeArray &phantom_node_vector) + { const bool checksum_OK = (route_parameters.check_sum == facade->GetCheckSum()); // find phantom nodes for all input coords - for (const auto i : osrm::irange(0, route_parameters.coordinates.size())) { + for (const auto i : osrm::irange(0, route_parameters.coordinates.size())) + { // if client hints are helpful, encode hints if (checksum_OK && i < route_parameters.hints.size() && - !route_parameters.hints[i].empty()) { + !route_parameters.hints[i].empty()) + { PhantomNode current_phantom_node; ObjectEncoder::DecodeFromBase64(route_parameters.hints[i], current_phantom_node); if (current_phantom_node.is_valid(facade->GetNumberOfNodes())) @@ -99,7 +94,8 @@ template class RoundTripPlugin final : public BasePlugin } facade->IncrementalFindPhantomNodeForCoordinate(route_parameters.coordinates[i], phantom_node_vector[i], 1); - if (phantom_node_vector[i].size() > 1) { + if (phantom_node_vector[i].size() > 1) + { phantom_node_vector[i].erase(std::begin(phantom_node_vector[i])); } BOOST_ASSERT(phantom_node_vector[i].front().is_valid(facade->GetNumberOfNodes())); @@ -110,7 +106,8 @@ template class RoundTripPlugin final : public BasePlugin // to access all graphs with component ID i, get the iterators by: // auto start = std::begin(scc_component.component) + scc_component.range[i]; // auto end = std::begin(scc_component.component) + scc_component.range[i+1]; - struct SCC_Component{ + struct SCC_Component + { // in_component: all NodeIDs sorted by component ID // in_range: index where a new component starts // @@ -118,198 +115,225 @@ template class RoundTripPlugin final : public BasePlugin // NodeID 3, 6, 7, 8 are in component 1 // => in_component = [0, 1, 2, 4, 5, 3, 6, 7, 8] // => in_range = [0, 5] - SCC_Component(std::vector in_component, - std::vector in_range) - : component(in_component), - range(in_range) { + SCC_Component(std::vector in_component, std::vector in_range) + : component(in_component), range(in_range) + { range.push_back(in_component.size()); + BOOST_ASSERT_MSG(in_component.size() >= in_range.size(), "scc component and its ranges do not match"); - BOOST_ASSERT_MSG(*std::max_element(in_range.begin(), in_range.end()) < in_component.size(), + BOOST_ASSERT_MSG(in_component.size() > 0, + "there's no scc component"); + BOOST_ASSERT_MSG(*std::max_element(in_range.begin(), in_range.end()) <= + in_component.size(), "scc component ranges are out of bound"); BOOST_ASSERT_MSG(*std::min_element(in_range.begin(), in_range.end()) >= 0, "invalid scc component range"); - BOOST_ASSERT_MSG([&in_range](){ - for (std::size_t r = 0; r < in_range.size() - 1; ++r) { - if (in_range[r] > in_range[r+1]) { - return false; - } - } - return true; - }(), + BOOST_ASSERT_MSG(std::is_sorted(std::begin(in_range), std::end(in_range)), "invalid component ranges"); - }; + }; // constructor to use when whole graph is one single scc SCC_Component(std::vector in_component) - : component(in_component), range({0, in_component.size()}) { - }; + : component(in_component), range({0, in_component.size()}){}; - std::size_t GetNumberOfComponents() const{ + std::size_t GetNumberOfComponents() const { + BOOST_ASSERT_MSG(range.size() > 0, + "there's no range"); return range.size() - 1; } const std::vector component; - // component range = in_range + [component.size()] - std::vector range; + std::vector range; }; // takes the number of locations and its distance matrix, // identifies and splits the graph in its strongly connected components (scc) // and returns an SCC_Component SCC_Component SplitUnaccessibleLocations(const std::size_t number_of_locations, - const DistTableWrapper & result_table) { + const DistTableWrapper &result_table) + { + + if (std::find(std::begin(result_table), std::end(result_table), INVALID_EDGE_WEIGHT) == std::end(result_table)) + { + // whole graph is one scc + std::vector location_ids(number_of_locations); + std::iota(std::begin(location_ids), std::end(location_ids), 0); + return SCC_Component(std::move(location_ids)); + } + // Run TarjanSCC - auto wrapper = std::make_shared>(result_table.GetTable(), number_of_locations); + auto wrapper = std::make_shared>(result_table.GetTable(), + number_of_locations); auto scc = TarjanSCC>(wrapper); scc.run(); - std::vector range_insertion; - std::vector range; - range_insertion.reserve(scc.get_number_of_components()); - range.reserve(scc.get_number_of_components()); + const auto number_of_components = scc.get_number_of_components(); + + std::vector range_insertion; + std::vector range; + range_insertion.reserve(number_of_components); + range.reserve(number_of_components); std::vector components(number_of_locations, 0); - auto prefix = 0; - for (size_t j = 0; j < scc.get_number_of_components(); ++j){ + std::size_t prefix = 0; + for (std::size_t j = 0; j < number_of_components; ++j) + { range_insertion.push_back(prefix); range.push_back(prefix); prefix += scc.get_component_size(j); } - for (size_t i = 0; i < number_of_locations; ++i) { + for (std::size_t i = 0; i < number_of_locations; ++i) + { components[range_insertion[scc.get_component_id(i)]] = i; ++range_insertion[scc.get_component_id(i)]; } - return SCC_Component(components, range); + return SCC_Component(std::move(components), std::move(range)); } - void SetLocPermutationOutput(const std::vector & permutation, - osrm::json::Object & json_result){ + void SetLocPermutationOutput(const std::vector &permutation, + osrm::json::Object &json_result) + { osrm::json::Array json_permutation; - json_permutation.values.insert(std::end(json_permutation.values), - std::begin(permutation), + json_permutation.values.insert(std::end(json_permutation.values), std::begin(permutation), std::end(permutation)); json_result.values["permutation"] = json_permutation; } - void ComputeRoute(const PhantomNodeArray & phantom_node_vector, - const RouteParameters & route_parameters, - const std::vector & trip, - InternalRouteResult & min_route) { + InternalRouteResult ComputeRoute(const PhantomNodeArray &phantom_node_vector, + const RouteParameters &route_parameters, + const std::vector &trip) + { + InternalRouteResult min_route; // given he final trip, compute total distance and return the route and location permutation PhantomNodes viapoint; - for (auto it = std::begin(trip); it != std::end(trip); ++it) { + const auto start = std::begin(trip); + const auto end = std::end(trip); + for (auto it = start; it != end; ++it) + { const auto from_node = *it; // if from_node is the last node, compute the route from the last to the first location - const auto to_node = std::next(it) != std::end(trip) ? *std::next(it) : *std::begin(trip); + const auto to_node = + std::next(it) != end ? *std::next(it) : *start; - viapoint = PhantomNodes{phantom_node_vector[from_node][0], phantom_node_vector[to_node][0]}; + viapoint = + PhantomNodes{phantom_node_vector[from_node][0], phantom_node_vector[to_node][0]}; min_route.segment_end_coordinates.emplace_back(viapoint); } - search_engine_ptr->shortest_path(min_route.segment_end_coordinates, route_parameters.uturns, min_route); + search_engine_ptr->shortest_path(min_route.segment_end_coordinates, route_parameters.uturns, + min_route); - BOOST_ASSERT_MSG(min_route.shortest_path_length < INVALID_EDGE_WEIGHT, - "unroutable route"); + BOOST_ASSERT_MSG(min_route.shortest_path_length < INVALID_EDGE_WEIGHT, "unroutable route"); + return min_route; } int HandleRequest(const RouteParameters &route_parameters, osrm::json::Object &json_result) override final { // check if all inputs are coordinates - if (!check_all_coordinates(route_parameters.coordinates)) { + if (!check_all_coordinates(route_parameters.coordinates)) + { return 400; } // get phantom nodes PhantomNodeArray phantom_node_vector(route_parameters.coordinates.size()); GetPhantomNodes(route_parameters, phantom_node_vector); - auto number_of_locations = phantom_node_vector.size(); + const auto number_of_locations = phantom_node_vector.size(); // compute the distance table of all phantom nodes - const auto result_table = DistTableWrapper(*search_engine_ptr->distance_table(phantom_node_vector), - number_of_locations); - if (result_table.size() == 0){ + const auto result_table = DistTableWrapper( + *search_engine_ptr->distance_table(phantom_node_vector), number_of_locations); + + if (result_table.size() == 0) + { return 400; } const constexpr std::size_t BF_MAX_FEASABLE = 10; - BOOST_ASSERT_MSG(result_table.size() > 0, "Distance Table is empty."); + BOOST_ASSERT_MSG(result_table.size() == number_of_locations * number_of_locations, + "Distance Table has wrong size."); // get scc components - SCC_Component scc = [&](){ - if (*std::max_element(result_table.begin(), result_table.end()) == INVALID_EDGE_WEIGHT) { - // compute all scc with tarjan - return SplitUnaccessibleLocations(number_of_locations, result_table); - } else { - // whole graph is one scc - std::vector location_ids(number_of_locations); - std::iota(std::begin(location_ids), std::end(location_ids), 0); - return SCC_Component(location_ids); - } - }(); - + SCC_Component scc = SplitUnaccessibleLocations(number_of_locations, result_table); using NodeIDIterator = typename std::vector::const_iterator; std::vector> route_result; route_result.reserve(scc.GetNumberOfComponents()); - TIMER_START(trip); - //run Trip computation for every SCC - for (std::size_t k = 0; k < scc.GetNumberOfComponents(); ++k) { - const auto component_size = scc.range[k+1] - scc.range[k]; + TIMER_START(TRIP_TIMER); + // run Trip computation for every SCC + for (std::size_t k = 0; k < scc.GetNumberOfComponents(); ++k) + { + const auto component_size = scc.range[k + 1] - scc.range[k]; - BOOST_ASSERT_MSG(component_size >= 0,"invalid component size"); + BOOST_ASSERT_MSG(component_size >= 0, "invalid component size"); - if (component_size > 1) { + if (component_size > 1) + { std::vector scc_route; NodeIDIterator start = std::begin(scc.component) + scc.range[k]; - NodeIDIterator end = std::begin(scc.component) + scc.range[k+1]; - - if (component_size < BF_MAX_FEASABLE) { - scc_route = osrm::trip::BruteForceTrip(start, end, number_of_locations, result_table); - route_result.push_back(scc_route); - } else { - scc_route = osrm::trip::FarthestInsertionTrip(start, end, number_of_locations, result_table); - route_result.push_back(scc_route); + NodeIDIterator end = std::begin(scc.component) + scc.range[k + 1]; + + if (component_size < BF_MAX_FEASABLE) + { + scc_route = + osrm::trip::BruteForceTrip(start, + end, + number_of_locations, + result_table); + } + else + { + scc_route = osrm::trip::FarthestInsertionTrip(start, + end, + number_of_locations, + result_table); } - // use this if output if debugging of route is needed: - SimpleLogger().Write() << "Route #" - << k << ": " - << [&scc_route](){ - std::string s = ""; - for (auto x : scc_route) { - s += std::to_string(x) + " "; - } - return s; - }(); - } else { + // use this output if debugging of route is needed: + // SimpleLogger().Write() << "Route #" << k << ": " << [&scc_route]() + // { + // std::string s = ""; + // for (auto x : scc_route) + // { + // s += std::to_string(x) + " "; + // } + // return s; + // }(); + + route_result.push_back(std::move(scc_route)); + } + else + { // if component only consists of one node, add it to the result routes - route_result.push_back({scc.component[scc.range[k]]}); + route_result.emplace_back(scc.component[scc.range[k]]); } } // compute all round trip routes - std::vector comp_route (route_result.size()); - for (std::size_t r = 0; r < route_result.size(); ++r) { - ComputeRoute(phantom_node_vector, route_parameters, route_result[r], comp_route[r]); + std::vector comp_route; + comp_route.reserve(route_result.size()); + for (std::size_t r = 0; r < route_result.size(); ++r) + { + comp_route.push_back(ComputeRoute(phantom_node_vector, route_parameters, route_result[r])); } - TIMER_STOP(trip); - - // prepare JSON output + TIMER_STOP(TRIP_TIMER); + SimpleLogger().Write() << "Trip calculation took: " << TIMER_MSEC(TRIP_TIMER) / 1000. << "s"; + // prepare JSON output // create a json object for every trip osrm::json::Array trip; - for (std::size_t i = 0; i < route_result.size(); ++i) { - std::unique_ptr> descriptor; - descriptor = osrm::make_unique>(facade); + for (std::size_t i = 0; i < route_result.size(); ++i) + { + std::unique_ptr> descriptor = osrm::make_unique>(facade); descriptor->SetConfig(route_parameters); osrm::json::Object scc_trip; @@ -319,16 +343,15 @@ template class RoundTripPlugin final : public BasePlugin // set viaroute output descriptor->Run(comp_route[i], scc_trip); - trip.values.push_back(scc_trip); + trip.values.push_back(std::move(scc_trip)); } - json_result.values["trips"] = trip; + json_result.values["trips"] = std::move(trip); return 200; } - }; #endif // TRIP_HPP diff --git a/unit_tests/routing_algorithms/tsp.cpp b/unit_tests/routing_algorithms/tsp.cpp deleted file mode 100644 index fde22295c05..00000000000 --- a/unit_tests/routing_algorithms/tsp.cpp +++ /dev/null @@ -1,45 +0,0 @@ -/* - -Copyright (c) 2014, Project OSRM contributors -All rights reserved. - -Redistribution and use in source and binary forms, with or without modification, -are permitted provided that the following conditions are met: - -Redistributions of source code must retain the above copyright notice, this list -of conditions and the following disclaimer. -Redistributions in binary form must reproduce the above copyright notice, this -list of conditions and the following disclaimer in the documentation and/or -other materials provided with the distribution. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR -ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES -(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON -ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -*/ - -#include "../../routing_algorithms/trip_brute_force.hpp" - -#include -#include -#include - -#include - -#include - -BOOST_AUTO_TEST_SUITE(trip) - -// BOOST_AUTO_TEST_CASE(check_distance_computation) -// { -// BOOST_CHECK_EQUAL(true, true); -// } - -BOOST_AUTO_TEST_SUITE_END() diff --git a/util/dist_table_wrapper.hpp b/util/dist_table_wrapper.hpp index eda3878c714..15550fafe2c 100644 --- a/util/dist_table_wrapper.hpp +++ b/util/dist_table_wrapper.hpp @@ -31,30 +31,30 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include #include #include +#include -//This Wrapper provides an easier access to a distance table that is given as an linear vector - -template class DistTableWrapper { -public: +// This Wrapper provides an easier access to a distance table that is given as an linear vector +template class DistTableWrapper +{ + public: using Iterator = typename std::vector::iterator; using ConstIterator = typename std::vector::const_iterator; DistTableWrapper(std::vector table, std::size_t number_of_nodes) - : table_(std::move(table)), number_of_nodes_(number_of_nodes) { + : table_(std::move(table)), number_of_nodes_(number_of_nodes) + { BOOST_ASSERT_MSG(table.size() == 0, "table is empty"); - BOOST_ASSERT_MSG(number_of_nodes_ * number_of_nodes_ <= table_.size(), "number_of_nodes_ is invalid"); + BOOST_ASSERT_MSG(number_of_nodes_ * number_of_nodes_ <= table_.size(), + "number_of_nodes_ is invalid"); }; - std::size_t GetNumberOfNodes() const { - return number_of_nodes_; - } + std::size_t GetNumberOfNodes() const { return number_of_nodes_; } - std::size_t size() const { - return table_.size(); - } + std::size_t size() const { return table_.size(); } - EdgeWeight operator() (NodeID from, NodeID to) const { + EdgeWeight operator()(NodeID from, NodeID to) const + { BOOST_ASSERT_MSG(from < number_of_nodes_, "from ID is out of bound"); BOOST_ASSERT_MSG(to < number_of_nodes_, "to ID is out of bound"); @@ -62,37 +62,27 @@ template class DistTableWrapper { BOOST_ASSERT_MSG(index < table_.size(), "index is out of bound"); - return table_[index]; + return table_[index]; } - ConstIterator begin() const{ - return std::begin(table_); - } + ConstIterator begin() const { return std::begin(table_); } - Iterator begin() { - return std::begin(table_); - } + Iterator begin() { return std::begin(table_); } - ConstIterator end() const{ - return std::end(table_); - } + ConstIterator end() const { return std::end(table_); } - Iterator end() { - return std::end(table_); - } + Iterator end() { return std::end(table_); } - NodeID GetIndexOfMaxValue() const { + NodeID GetIndexOfMaxValue() const + { return std::distance(table_.begin(), std::max_element(table_.begin(), table_.end())); } - std::vector GetTable() const { - return table_; - } + std::vector GetTable() const { return table_; } -private: + private: std::vector table_; const std::size_t number_of_nodes_; }; - #endif // DIST_TABLE_WRAPPER_H From a71159667dc0d14c78ea247fcb71944d2a227283 Mon Sep 17 00:00:00 2001 From: Huyen Chau Nguyen Date: Sun, 30 Aug 2015 23:12:40 +0200 Subject: [PATCH 085/122] add cucumber test for the trip plugin --- features/step_definitions/matching.rb | 3 + features/step_definitions/trip.rb | 121 ++++++++++++++++++++++++++ features/support/trip.rb | 14 +++ features/testbot/trip.feature | 86 ++++++++++++++++++ 4 files changed, 224 insertions(+) create mode 100644 features/step_definitions/trip.rb create mode 100644 features/support/trip.rb create mode 100644 features/testbot/trip.feature diff --git a/features/step_definitions/matching.rb b/features/step_definitions/matching.rb index e9b723460b9..607efedd975 100644 --- a/features/step_definitions/matching.rb +++ b/features/step_definitions/matching.rb @@ -58,6 +58,9 @@ end end + puts table + puts sub_matchings + ok = true encoded_result = "" extended_target = "" diff --git a/features/step_definitions/trip.rb b/features/step_definitions/trip.rb new file mode 100644 index 00000000000..007717e040b --- /dev/null +++ b/features/step_definitions/trip.rb @@ -0,0 +1,121 @@ +When /^I plan a trip I should get$/ do |table| + reprocess + actual = [] + OSRMLoader.load(self,"#{prepared_file}.osrm") do + table.hashes.each_with_index do |row,ri| + if row['request'] + got = {'request' => row['request'] } + response = request_url row['request'] + else + params = @query_params + waypoints = [] + if row['from'] and row['to'] + node = find_node_by_name(row['from']) + raise "*** unknown from-node '#{row['from']}" unless node + waypoints << node + + node = find_node_by_name(row['to']) + raise "*** unknown to-node '#{row['to']}" unless node + waypoints << node + + got = {'from' => row['from'], 'to' => row['to'] } + response = request_trip waypoints, params + elsif row['waypoints'] + row['waypoints'].split(',').each do |n| + node = find_node_by_name(n.strip) + raise "*** unknown waypoint node '#{n.strip}" unless node + waypoints << node + end + got = {'waypoints' => row['waypoints'] } + response = request_trip waypoints, params + else + raise "*** no waypoints" + end + end + + row.each_pair do |k,v| + if k =~ /param:(.*)/ + if v=='(nil)' + params[$1]=nil + elsif v!=nil + params[$1]=v + end + got[k]=v + end + end + + if response.body.empty? == false + json = JSON.parse response.body + end + + if table.headers.include? 'status' + got['status'] = json['status'].to_s + end + if table.headers.include? 'message' + got['message'] = json['status_message'] + end + if table.headers.include? '#' # comment column + got['#'] = row['#'] # copy value so it always match + end + + if response.code == "200" + if table.headers.include? 'trips' + sub_trips = json['trips'].compact.map { |sub| sub['via_points']} + end + end + + ###################### + ok = true + encoded_result = "" + extended_target = "" + row['trips'].split(',').each_with_index do |sub, sub_idx| + if sub_idx >= sub_trips.length + ok = false + break + end + + ok = false; + #TODO: Check all rotations of the round trip + sub.length.times do |node_idx| + node = find_node_by_name(sub[node_idx]) + out_node = sub_trips[sub_idx][node_idx] + if FuzzyMatch.match_location out_node, node + encoded_result += sub[node_idx] + extended_target += sub[node_idx] + ok = true + else + encoded_result += "? [#{out_node[0]},#{out_node[1]}]" + extended_target += "#{sub[node_idx]} [#{node.lat},#{node.lon}]" + end + end + end + + if ok + got['trips'] = row['trips'] + got['via_points'] = row['via_points'] + else + got['trips'] = encoded_result + row['trips'] = extended_target + log_fail row,got, { 'trip' => {:query => @query, :response => response} } + end + + + ok = true + row.keys.each do |key| + if FuzzyMatch.match got[key], row[key] + got[key] = row[key] + else + ok = false + end + end + + unless ok + log_fail row,got, { 'trip' => {:query => @query, :response => response} } + end + + actual << got + end + end + table.diff! actual +end + diff --git a/features/support/trip.rb b/features/support/trip.rb new file mode 100644 index 00000000000..6ea90109466 --- /dev/null +++ b/features/support/trip.rb @@ -0,0 +1,14 @@ +require 'net/http' +HOST = "http://127.0.0.1:#{OSRM_PORT}" + +def request_trip waypoints=[], params={} + defaults = { 'output' => 'json' } + locs = waypoints.compact.map { |w| "loc=#{w.lat},#{w.lon}" } + + params = (locs + defaults.merge(params).to_param).join('&') + params = nil if params=="" + + uri = generate_request_url ("trip" + '?' + params) + response = send_request uri, waypoints, params +end + diff --git a/features/testbot/trip.feature b/features/testbot/trip.feature new file mode 100644 index 00000000000..c9f944b2694 --- /dev/null +++ b/features/testbot/trip.feature @@ -0,0 +1,86 @@ +@trip @testbot +Feature: Basic trip planning + + Background: + Given the profile "testbot" + Given a grid size of 10 meters + + Scenario: Testbot - Trip planning with less than 10 nodes + Given the node map + | a | b | + | d | c | + + And the ways + | nodes | + | ab | + | bc | + | cb | + | da | + + When I plan a trip I should get + | waypoints | trips | + | a,b,c,d | dcba | + + Scenario: Testbot - Trip planning with more than 10 nodes + Given the node map + | a | b | c | d | + | l | | | e | + | k | | | f | + | j | i | h | g | + + And the ways + | nodes | + | ab | + | bc | + | cb | + | de | + | ef | + | fg | + | gh | + | hi | + | ij | + | jk | + | kl | + | la | + + + When I plan a trip I should get + | waypoints | trips | + | a,b,c,d,e,f,g,h,i,j,k,l | cbalkjihgfedc | + + Scenario: Testbot - Trip planning with multiple scc + Given the node map + | a | b | c | d | + | l | | | e | + | k | | | f | + | j | i | h | g | + | | | | | + | m | n | | | + | p | o | | | + + And the ways + | nodes | + | ab | + | bc | + | cb | + | de | + | ef | + | fg | + | gh | + | hi | + | ij | + | jk | + | kl | + | la | + | mn | + | no | + | op | + | pm | + + + When I plan a trip I should get + | waypoints | trips | + | a,b,c,d,e,f,g,h,i,j,k,l,m,n,o,p | cbalkjihgfedc,ponm | + + + From f6a90e9b429b4e2fdfa231580ae404d7c0696b14 Mon Sep 17 00:00:00 2001 From: Huyen Chau Nguyen Date: Mon, 31 Aug 2015 18:02:07 +0200 Subject: [PATCH 086/122] add missing include and clang-format --- algorithms/tarjan_scc.hpp | 2 ++ algorithms/trip_brute_force.hpp | 8 ++--- plugins/trip.hpp | 52 +++++++++++++++------------------ 3 files changed, 30 insertions(+), 32 deletions(-) diff --git a/algorithms/tarjan_scc.hpp b/algorithms/tarjan_scc.hpp index f3fab7fd57c..59077322d60 100644 --- a/algorithms/tarjan_scc.hpp +++ b/algorithms/tarjan_scc.hpp @@ -44,6 +44,8 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include #include +#include +#include #include #include diff --git a/algorithms/trip_brute_force.hpp b/algorithms/trip_brute_force.hpp index 5c8eb03fa14..1b451e165dd 100644 --- a/algorithms/trip_brute_force.hpp +++ b/algorithms/trip_brute_force.hpp @@ -57,8 +57,9 @@ EdgeWeight ReturnDistance(const DistTableWrapper &dist_table, while (i < location_order.size() && (route_dist < min_route_dist)) { route_dist += dist_table(location_order[i], location_order[(i + 1) % component_size]); - BOOST_ASSERT_MSG(dist_table(location_order[i], location_order[(i + 1) % component_size]) - != INVALID_EDGE_WEIGHT, "invalid route found"); + BOOST_ASSERT_MSG(dist_table(location_order[i], location_order[(i + 1) % component_size]) != + INVALID_EDGE_WEIGHT, + "invalid route found"); ++i; } @@ -82,8 +83,7 @@ std::vector BruteForceTrip(const NodeIDIterator start, // check length of all possible permutation of the component ids - BOOST_ASSERT_MSG(perm.size() > 0, - "no permutation given"); + BOOST_ASSERT_MSG(perm.size() > 0, "no permutation given"); BOOST_ASSERT_MSG(*(std::max_element(std::begin(perm), std::end(perm))) < number_of_locations, "invalid node id"); BOOST_ASSERT_MSG(*(std::min_element(std::begin(perm), std::end(perm))) >= 0, "invalid node id"); diff --git a/plugins/trip.hpp b/plugins/trip.hpp index 860f036452f..2567db4940b 100644 --- a/plugins/trip.hpp +++ b/plugins/trip.hpp @@ -36,15 +36,15 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "../algorithms/trip_farthest_insertion.hpp" #include "../algorithms/trip_brute_force.hpp" #include "../data_structures/search_engine.hpp" -#include "../data_structures/matrix_graph_wrapper.hpp" // wrapper to use tarjan - // scc on dist table -#include "../descriptors/descriptor_base.hpp" // to make json output -#include "../descriptors/json_descriptor.hpp" // to make json output +#include "../data_structures/matrix_graph_wrapper.hpp" // wrapper to use tarjan + // scc on dist table +#include "../descriptors/descriptor_base.hpp" // to make json output +#include "../descriptors/json_descriptor.hpp" // to make json output #include "../util/make_unique.hpp" -#include "../util/timing_util.hpp" // to time runtime -#include "../util/simple_logger.hpp" // for logging output -#include "../util/dist_table_wrapper.hpp" // to access the dist - // table more easily +#include "../util/timing_util.hpp" // to time runtime +#include "../util/simple_logger.hpp" // for logging output +#include "../util/dist_table_wrapper.hpp" // to access the dist + // table more easily #include #include @@ -122,8 +122,7 @@ template class RoundTripPlugin final : public BasePlugin BOOST_ASSERT_MSG(in_component.size() >= in_range.size(), "scc component and its ranges do not match"); - BOOST_ASSERT_MSG(in_component.size() > 0, - "there's no scc component"); + BOOST_ASSERT_MSG(in_component.size() > 0, "there's no scc component"); BOOST_ASSERT_MSG(*std::max_element(in_range.begin(), in_range.end()) <= in_component.size(), "scc component ranges are out of bound"); @@ -135,11 +134,11 @@ template class RoundTripPlugin final : public BasePlugin // constructor to use when whole graph is one single scc SCC_Component(std::vector in_component) - : component(in_component), range({0, in_component.size()}){}; + : component(in_component), range({0, in_component.size()}){}; - std::size_t GetNumberOfComponents() const { - BOOST_ASSERT_MSG(range.size() > 0, - "there's no range"); + std::size_t GetNumberOfComponents() const + { + BOOST_ASSERT_MSG(range.size() > 0, "there's no range"); return range.size() - 1; } @@ -154,7 +153,8 @@ template class RoundTripPlugin final : public BasePlugin const DistTableWrapper &result_table) { - if (std::find(std::begin(result_table), std::end(result_table), INVALID_EDGE_WEIGHT) == std::end(result_table)) + if (std::find(std::begin(result_table), std::end(result_table), INVALID_EDGE_WEIGHT) == + std::end(result_table)) { // whole graph is one scc std::vector location_ids(number_of_locations); @@ -162,7 +162,6 @@ template class RoundTripPlugin final : public BasePlugin return SCC_Component(std::move(location_ids)); } - // Run TarjanSCC auto wrapper = std::make_shared>(result_table.GetTable(), number_of_locations); @@ -217,8 +216,7 @@ template class RoundTripPlugin final : public BasePlugin { const auto from_node = *it; // if from_node is the last node, compute the route from the last to the first location - const auto to_node = - std::next(it) != end ? *std::next(it) : *start; + const auto to_node = std::next(it) != end ? *std::next(it) : *start; viapoint = PhantomNodes{phantom_node_vector[from_node][0], phantom_node_vector[to_node][0]}; @@ -283,16 +281,11 @@ template class RoundTripPlugin final : public BasePlugin if (component_size < BF_MAX_FEASABLE) { scc_route = - osrm::trip::BruteForceTrip(start, - end, - number_of_locations, - result_table); + osrm::trip::BruteForceTrip(start, end, number_of_locations, result_table); } else { - scc_route = osrm::trip::FarthestInsertionTrip(start, - end, - number_of_locations, + scc_route = osrm::trip::FarthestInsertionTrip(start, end, number_of_locations, result_table); } @@ -321,19 +314,22 @@ template class RoundTripPlugin final : public BasePlugin comp_route.reserve(route_result.size()); for (std::size_t r = 0; r < route_result.size(); ++r) { - comp_route.push_back(ComputeRoute(phantom_node_vector, route_parameters, route_result[r])); + comp_route.push_back( + ComputeRoute(phantom_node_vector, route_parameters, route_result[r])); } TIMER_STOP(TRIP_TIMER); - SimpleLogger().Write() << "Trip calculation took: " << TIMER_MSEC(TRIP_TIMER) / 1000. << "s"; + SimpleLogger().Write() << "Trip calculation took: " << TIMER_MSEC(TRIP_TIMER) / 1000. + << "s"; // prepare JSON output // create a json object for every trip osrm::json::Array trip; for (std::size_t i = 0; i < route_result.size(); ++i) { - std::unique_ptr> descriptor = osrm::make_unique>(facade); + std::unique_ptr> descriptor = + osrm::make_unique>(facade); descriptor->SetConfig(route_parameters); osrm::json::Object scc_trip; From bb1428eeb14889890977d514dee84b78d400acb0 Mon Sep 17 00:00:00 2001 From: "Daniel J. Hofmann" Date: Wed, 2 Sep 2015 12:23:26 +0200 Subject: [PATCH 087/122] Remove unneeded semicola from profiles. Nothing fancy, does what it says. --- profiles/bicycle.lua | 10 +++++----- profiles/car.lua | 22 +++++++++++----------- profiles/foot.lua | 2 +- profiles/testbot.lua | 4 ++-- 4 files changed, 19 insertions(+), 19 deletions(-) diff --git a/profiles/bicycle.lua b/profiles/bicycle.lua index 88709ec3c1d..ffc70b24fc3 100644 --- a/profiles/bicycle.lua +++ b/profiles/bicycle.lua @@ -120,7 +120,7 @@ local function parse_maxspeed(source) n = 0 end if string.match(source, "mph") or string.match(source, "mp/h") then - n = (n*1609)/1000; + n = (n*1609)/1000 end return n end @@ -155,7 +155,7 @@ function node_function (node, result) -- check if node is a traffic light local tag = node:get_value_by_key("highway") if tag and "traffic_signals" == tag then - result.traffic_lights = true; + result.traffic_lights = true end end @@ -226,15 +226,15 @@ function way_function (way, result) -- roundabout handling if junction and "roundabout" == junction then - result.roundabout = true; + result.roundabout = true end -- speed local bridge_speed = bridge_speeds[bridge] if (bridge_speed and bridge_speed > 0) then - highway = bridge; + highway = bridge if duration and durationIsValid(duration) then - result.duration = math.max( parseDuration(duration), 1 ); + result.duration = math.max( parseDuration(duration), 1 ) end result.forward_mode = mode_movable_bridge result.backward_mode = mode_movable_bridge diff --git a/profiles/car.lua b/profiles/car.lua index 03ca924dc97..02fa19494bf 100644 --- a/profiles/car.lua +++ b/profiles/car.lua @@ -159,7 +159,7 @@ local function parse_maxspeed(source) local n = tonumber(source:match("%d*")) if n then if string.match(source, "mph") or string.match(source, "mp/h") then - n = (n*1609)/1000; + n = (n*1609)/1000 end else -- parse maxspeed like FR:urban @@ -208,7 +208,7 @@ function node_function (node, result) -- check if node is a traffic light local tag = node:get_value_by_key("highway") if tag and "traffic_signals" == tag then - result.traffic_lights = true; + result.traffic_lights = true end end @@ -252,10 +252,10 @@ function way_function (way, result) -- handling ferries and piers local route_speed = speed_profile[route] if (route_speed and route_speed > 0) then - highway = route; + highway = route local duration = way:get_value_by_key("duration") if duration and durationIsValid(duration) then - result.duration = max( parseDuration(duration), 1 ); + result.duration = max( parseDuration(duration), 1 ) end result.forward_mode = mode_ferry result.backward_mode = mode_ferry @@ -267,10 +267,10 @@ function way_function (way, result) local bridge_speed = speed_profile[bridge] local capacity_car = way:get_value_by_key("capacity:car") if (bridge_speed and bridge_speed > 0) and (capacity_car ~= 0) then - highway = bridge; + highway = bridge local duration = way:get_value_by_key("duration") if duration and durationIsValid(duration) then - result.duration = max( parseDuration(duration), 1 ); + result.duration = max( parseDuration(duration), 1 ) end result.forward_mode = mode_movable_bridge result.backward_mode = mode_movable_bridge @@ -355,7 +355,7 @@ function way_function (way, result) end if junction and "roundabout" == junction then - result.roundabout = true; + result.roundabout = true end -- Set access restriction flag if access is allowed under certain restrictions only @@ -413,19 +413,19 @@ function way_function (way, result) -- scale speeds to get better avg driving times if result.forward_speed > 0 then - local scaled_speed = result.forward_speed*speed_reduction + 11; + local scaled_speed = result.forward_speed*speed_reduction + 11 local penalized_speed = math.huge if width <= 3 or (lanes <= 1 and is_bidirectional) then - penalized_speed = result.forward_speed / 2; + penalized_speed = result.forward_speed / 2 end result.forward_speed = math.min(penalized_speed, scaled_speed) end if result.backward_speed > 0 then - local scaled_speed = result.backward_speed*speed_reduction + 11; + local scaled_speed = result.backward_speed*speed_reduction + 11 local penalized_speed = math.huge if width <= 3 or (lanes <= 1 and is_bidirectional) then - penalized_speed = result.backward_speed / 2; + penalized_speed = result.backward_speed / 2 end result.backward_speed = math.min(penalized_speed, scaled_speed) end diff --git a/profiles/foot.lua b/profiles/foot.lua index 566425f3b65..3d676ba3bd4 100644 --- a/profiles/foot.lua +++ b/profiles/foot.lua @@ -161,7 +161,7 @@ function way_function (way, result) -- roundabouts if "roundabout" == junction then - result.roundabout = true; + result.roundabout = true end -- speed diff --git a/profiles/testbot.lua b/profiles/testbot.lua index 7bd14163520..6b10d04e215 100644 --- a/profiles/testbot.lua +++ b/profiles/testbot.lua @@ -49,7 +49,7 @@ function node_function (node, result) local traffic_signal = node:get_value_by_key("highway") if traffic_signal and traffic_signal == "traffic_signals" then - result.traffic_lights = true; + result.traffic_lights = true -- TODO: a way to set the penalty value end end @@ -78,7 +78,7 @@ function way_function (way, result) local speed_back = speed_forw if highway == "river" then - local temp_speed = speed_forw; + local temp_speed = speed_forw result.forward_mode = 3 result.backward_mode = 4 speed_forw = temp_speed*1.5 From 980e4ee89a04fd1f460958407a0f6079b03e8bc3 Mon Sep 17 00:00:00 2001 From: "Daniel J. Hofmann" Date: Wed, 2 Sep 2015 16:33:03 +0200 Subject: [PATCH 088/122] Don't mix signed and unsigned in comparisons as signed is converted first to unsigned. This is true: -1 > 1u because the integer literal `-1` is first converted to a large unsigned value and then compared to the unsigned `1`. This patch fixes several of those isses in the farthest insertion algorithm. `-Wsign-compare` catches those issues. References: - http://stackoverflow.com/a/5416498 - C++14 standard --- algorithms/trip_farthest_insertion.hpp | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/algorithms/trip_farthest_insertion.hpp b/algorithms/trip_farthest_insertion.hpp index 8f5c59ebbce..5f41248a43d 100644 --- a/algorithms/trip_farthest_insertion.hpp +++ b/algorithms/trip_farthest_insertion.hpp @@ -32,6 +32,7 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "../util/dist_table_wrapper.hpp" #include +#include #include #include @@ -170,10 +171,12 @@ std::vector FarthestInsertionTrip(const NodeIDIterator &start, ////////////////////////////////////////////////////////////////////////////////////////////////// const auto component_size = std::distance(start, end); + BOOST_ASSERT(component_size >= 0); + auto max_from = -1; auto max_to = -1; - if (component_size == number_of_locations) + if (static_cast(component_size) == number_of_locations) { // find the pair of location with the biggest distance and make the pair the initial start // trip @@ -199,12 +202,14 @@ std::vector FarthestInsertionTrip(const NodeIDIterator &start, } } } - BOOST_ASSERT_MSG(max_from < number_of_locations && max_from >= 0, "start node"); - BOOST_ASSERT_MSG(max_to < number_of_locations && max_to >= 0, "start node"); + BOOST_ASSERT(max_from >= 0); + BOOST_ASSERT(max_to >= 0); + BOOST_ASSERT_MSG(static_cast(max_from) < number_of_locations, "start node"); + BOOST_ASSERT_MSG(static_cast(max_to) < number_of_locations, "start node"); return FindRoute(number_of_locations, component_size, start, end, dist_table, max_from, max_to); } } // end namespace trip } // end namespace osrm -#endif // TRIP_FARTHEST_INSERTION_HPP \ No newline at end of file +#endif // TRIP_FARTHEST_INSERTION_HPP From b3822d5802e7cfd57500ff4bf37570062a2a3bb8 Mon Sep 17 00:00:00 2001 From: Daniel Patterson Date: Sun, 30 Aug 2015 10:17:19 -0700 Subject: [PATCH 089/122] Enable turn penalties on car profile, using values tuned by comparing real-world sample routes with map-matched routes. --- profiles/car.lua | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/profiles/car.lua b/profiles/car.lua index 02fa19494bf..c6d443ccdf0 100644 --- a/profiles/car.lua +++ b/profiles/car.lua @@ -131,6 +131,11 @@ maxspeed_table = { traffic_signal_penalty = 2 use_turn_restrictions = true +local turn_penalty = 10 +-- Note: this biases right-side driving. Should be +-- inverted for left-driving countries. +local turn_bias = 1.2 + local obey_oneway = true local ignore_areas = true local u_turn_penalty = 20 @@ -431,3 +436,12 @@ function way_function (way, result) end end +function turn_function (angle) + ---- compute turn penalty as angle^2, with a left/right bias + k = turn_penalty/(90.0*90.0) + if angle>=0 then + return angle*angle*k/turn_bias + else + return angle*angle*k*turn_bias + end +end From 120303e6e3af26308fc5717ee1a6965bd6849a41 Mon Sep 17 00:00:00 2001 From: Daniel Patterson Date: Wed, 2 Sep 2015 16:24:34 -0700 Subject: [PATCH 090/122] Fixed test case that uses the car profile. --- features/car/ferry.feature | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/features/car/ferry.feature b/features/car/ferry.feature index eb245598abf..37f520ed91d 100644 --- a/features/car/ferry.feature +++ b/features/car/ferry.feature @@ -41,7 +41,7 @@ Feature: Car - Handle ferry routes When I route I should get | from | to | route | modes | speed | - | a | g | abc,cde,efg | 1,2,1 | 26 km/h | + | a | g | abc,cde,efg | 1,2,1 | 25 km/h | | b | f | abc,cde,efg | 1,2,1 | 20 km/h | | c | e | cde | 2 | 12 km/h | | e | c | cde | 2 | 12 km/h | @@ -60,7 +60,7 @@ Feature: Car - Handle ferry routes When I route I should get | from | to | route | modes | speed | - | a | g | abc,cde,efg | 1,2,1 | 26 km/h | + | a | g | abc,cde,efg | 1,2,1 | 25 km/h | | b | f | abc,cde,efg | 1,2,1 | 20 km/h | | c | e | cde | 2 | 12 km/h | | e | c | cde | 2 | 12 km/h | From a4f558181d767f680334cea6e985060e54ef2e71 Mon Sep 17 00:00:00 2001 From: Patrick Niklaus Date: Wed, 26 Aug 2015 22:44:28 +0200 Subject: [PATCH 091/122] Add status field to match plugin response --- plugins/match.hpp | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/plugins/match.hpp b/plugins/match.hpp index b0138247c7e..a76ab634f22 100644 --- a/plugins/match.hpp +++ b/plugins/match.hpp @@ -216,6 +216,7 @@ template class MapMatchingPlugin : public BasePlugin // check number of parameters if (!check_all_coordinates(route_parameters.coordinates)) { + json_result.values["status"] = "Invalid coordinates."; return 400; } @@ -225,6 +226,7 @@ template class MapMatchingPlugin : public BasePlugin const auto &input_timestamps = route_parameters.timestamps; if (input_timestamps.size() > 0 && input_coords.size() != input_timestamps.size()) { + json_result.values["status"] = "Number of timestamps does not match number of coordinates ."; return 400; } @@ -232,6 +234,14 @@ template class MapMatchingPlugin : public BasePlugin if (max_locations_map_matching > 0 && static_cast(input_coords.size()) < max_locations_map_matching) { + json_result.values["status"] = "Too many coodindates."; + return 400; + } + + // enforce maximum number of locations for performance reasons + if (static_cast(input_coords.size()) < 2) + { + json_result.values["status"] = "At least two coordinates needed."; return 400; } @@ -239,6 +249,7 @@ template class MapMatchingPlugin : public BasePlugin getCandiates(input_coords, sub_trace_lengths, candidates_lists); if (!found_candidates) { + json_result.values["status"] = "No suitable matching candidates found."; return 400; } @@ -254,6 +265,7 @@ template class MapMatchingPlugin : public BasePlugin if (sub_matchings.empty()) { + json_result.values["status"] = "No matchings found."; return 400; } From a5ee82b0d17bcfb9b49691c4d0069105a29104c0 Mon Sep 17 00:00:00 2001 From: Patrick Niklaus Date: Wed, 26 Aug 2015 23:27:47 +0200 Subject: [PATCH 092/122] Make matching thresholds adaptable to different sample lengths --- routing_algorithms/map_matching.hpp | 49 ++++++++++++++++++++++++----- 1 file changed, 41 insertions(+), 8 deletions(-) diff --git a/routing_algorithms/map_matching.hpp b/routing_algorithms/map_matching.hpp index dfb0af0a8ac..0472cf8dfab 100644 --- a/routing_algorithms/map_matching.hpp +++ b/routing_algorithms/map_matching.hpp @@ -59,10 +59,9 @@ using CandidateLists = std::vector; using HMM = HiddenMarkovModel; using SubMatchingList = std::vector; -constexpr static const unsigned MAX_BROKEN_STATES = 6; -constexpr static const unsigned MAX_BROKEN_TIME = 60; +constexpr static const unsigned MAX_BROKEN_STATES = 10; -constexpr static const unsigned MAX_DISTANCE_DELTA = 200; +constexpr static const double MAX_SPEED = 180 / 3.6; // 150km -> m/s constexpr static const unsigned SUSPICIOUS_DISTANCE_DELTA = 100; constexpr static const double default_beta = 5.0; @@ -78,6 +77,19 @@ class MapMatching final : public BasicRoutingInterface& timestamps) const + { + BOOST_ASSERT(timestamps.size() > 1); + + std::vector sample_times(timestamps.size()); + + std::adjacent_difference(timestamps.begin(), timestamps.end(), sample_times.begin()); + + // don't use first element of sample_times -> will not be a difference. + auto sum_sample_times = std::accumulate(std::next(sample_times.begin()), sample_times.end(), 0); + return sum_sample_times / (sample_times.size() - 1); + } + public: MapMatching(DataFacadeT *facade, SearchEngineData &engine_working_data) : super(facade), engine_working_data(engine_working_data) @@ -91,7 +103,29 @@ class MapMatching final : public BasicRoutingInterface 1) + { + return GetAverageSampleTime(trace_timestamps); + } + else + { + return 0u; + } + }(); + const auto max_broken_time = avg_sample_time * osrm::matching::MAX_BROKEN_STATES; + const auto max_distance_delta = [&]() { + if (trace_timestamps.size() > 1) + { + return avg_sample_time * osrm::matching::MAX_SPEED; + } + else + { + return std::numeric_limits::max(); + } + }(); // TODO replace default values with table lookup based on sampling frequency EmissionLogProbability emission_log_probability( @@ -126,7 +160,7 @@ class MapMatching final : public BasicRoutingInterface - osrm::matching::MAX_BROKEN_TIME); + max_broken_time); } else { @@ -179,8 +213,6 @@ class MapMatching final : public BasicRoutingInterfaceGetNumberOfNodes()); - engine_working_data.InitializeOrClearSecondThreadLocalStorage( - super::facade->GetNumberOfNodes()); QueryHeap &forward_heap = *(engine_working_data.forward_heap_1); QueryHeap &reverse_heap = *(engine_working_data.reverse_heap_1); @@ -196,6 +228,7 @@ class MapMatching final : public BasicRoutingInterface(0u, current_viterbi.size())) { // how likely is candidate s_prime at time t to be emitted? + // FIXME this can be pre-computed const double emission_pr = emission_log_probability(candidates_list[t][s_prime].second); double new_value = prev_viterbi[s] + emission_pr; @@ -218,7 +251,7 @@ class MapMatching final : public BasicRoutingInterface prune - if (d_t > osrm::matching::MAX_DISTANCE_DELTA) + if (d_t > max_distance_delta) { continue; } From c30c1441207f6f76a48ed9f0a62d8273038a74f1 Mon Sep 17 00:00:00 2001 From: Patrick Niklaus Date: Fri, 28 Aug 2015 01:05:48 +0200 Subject: [PATCH 093/122] Move matching default in route_parameters.cpp --- data_structures/route_parameters.cpp | 2 +- routing_algorithms/map_matching.hpp | 9 ++------- 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/data_structures/route_parameters.cpp b/data_structures/route_parameters.cpp index 86b9553580c..cac9de30dc0 100644 --- a/data_structures/route_parameters.cpp +++ b/data_structures/route_parameters.cpp @@ -36,7 +36,7 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. RouteParameters::RouteParameters() : zoom_level(18), print_instructions(false), alternate_route(true), geometry(true), compression(true), deprecatedAPI(false), uturn_default(false), classify(false), - matching_beta(-1.0), gps_precision(-1.0), check_sum(-1), num_results(1) + matching_beta(5), gps_precision(5), check_sum(-1), num_results(1) { } diff --git a/routing_algorithms/map_matching.hpp b/routing_algorithms/map_matching.hpp index 0472cf8dfab..b14335b994d 100644 --- a/routing_algorithms/map_matching.hpp +++ b/routing_algorithms/map_matching.hpp @@ -63,9 +63,6 @@ constexpr static const unsigned MAX_BROKEN_STATES = 10; constexpr static const double MAX_SPEED = 180 / 3.6; // 150km -> m/s constexpr static const unsigned SUSPICIOUS_DISTANCE_DELTA = 100; - -constexpr static const double default_beta = 5.0; -constexpr static const double default_sigma_z = 4.07; } } @@ -128,10 +125,8 @@ class MapMatching final : public BasicRoutingInterface 0. ? gps_precision : osrm::matching::default_sigma_z); - TransitionLogProbability transition_log_probability( - matching_beta > 0. ? matching_beta : osrm::matching::default_beta); + EmissionLogProbability emission_log_probability(gps_precision); + TransitionLogProbability transition_log_probability(matching_beta); osrm::matching::HMM model(candidates_list, emission_log_probability); From ee0c20ae44eaea0344540248da72a8ad90f84525 Mon Sep 17 00:00:00 2001 From: Patrick Niklaus Date: Fri, 28 Aug 2015 01:17:18 +0200 Subject: [PATCH 094/122] Fix typo --- routing_algorithms/map_matching.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/routing_algorithms/map_matching.hpp b/routing_algorithms/map_matching.hpp index b14335b994d..4c8e561a0e0 100644 --- a/routing_algorithms/map_matching.hpp +++ b/routing_algorithms/map_matching.hpp @@ -61,7 +61,7 @@ using SubMatchingList = std::vector; constexpr static const unsigned MAX_BROKEN_STATES = 10; -constexpr static const double MAX_SPEED = 180 / 3.6; // 150km -> m/s +constexpr static const double MAX_SPEED = 180 / 3.6; // 180km -> m/s constexpr static const unsigned SUSPICIOUS_DISTANCE_DELTA = 100; } } From 262b3802804e53370b4cadee3f8f1f5d7f0333cb Mon Sep 17 00:00:00 2001 From: Patrick Niklaus Date: Sat, 29 Aug 2015 15:53:33 +0200 Subject: [PATCH 095/122] Candidate query for match is now only depending on gps_precision --- data_structures/hidden_markov_model.hpp | 31 ++++++--- data_structures/static_rtree.hpp | 69 ++++--------------- plugins/match.hpp | 54 ++++++++++----- server/data_structures/datafacade_base.hpp | 4 +- .../data_structures/internal_datafacade.hpp | 7 +- server/data_structures/shared_datafacade.hpp | 7 +- 6 files changed, 74 insertions(+), 98 deletions(-) diff --git a/data_structures/hidden_markov_model.hpp b/data_structures/hidden_markov_model.hpp index cccaf7b5d28..60738a56148 100644 --- a/data_structures/hidden_markov_model.hpp +++ b/data_structures/hidden_markov_model.hpp @@ -91,13 +91,23 @@ template struct HiddenMarkovModel : breakage(candidates_list.size()), candidates_list(candidates_list), emission_log_probability(emission_log_probability) { - for (const auto &l : candidates_list) + viterbi.resize(candidates_list.size()); + parents.resize(candidates_list.size()); + path_lengths.resize(candidates_list.size()); + suspicious.resize(candidates_list.size()); + pruned.resize(candidates_list.size()); + for (const auto i : osrm::irange(0u, candidates_list.size())) { - viterbi.emplace_back(l.size()); - parents.emplace_back(l.size()); - path_lengths.emplace_back(l.size()); - suspicious.emplace_back(l.size()); - pruned.emplace_back(l.size()); + const auto& num_candidates = candidates_list[i].size(); + // add empty vectors + if (num_candidates > 0) + { + viterbi[i].resize(num_candidates); + parents[i].resize(num_candidates); + path_lengths[i].resize(num_candidates); + suspicious[i].resize(num_candidates); + pruned[i].resize(num_candidates); + } } clear(0); @@ -121,10 +131,11 @@ template struct HiddenMarkovModel std::size_t initialize(std::size_t initial_timestamp) { - BOOST_ASSERT(initial_timestamp < candidates_list.size()); - + auto num_points = candidates_list.size(); do { + BOOST_ASSERT(initial_timestamp < num_points); + for (const auto s : osrm::irange(0u, viterbi[initial_timestamp].size())) { viterbi[initial_timestamp][s] = @@ -139,9 +150,9 @@ template struct HiddenMarkovModel } ++initial_timestamp; - } while (breakage[initial_timestamp - 1]); + } while (initial_timestamp < num_points && breakage[initial_timestamp - 1]); - if (initial_timestamp >= viterbi.size()) + if (initial_timestamp >= num_points) { return osrm::matching::INVALID_STATE; } diff --git a/data_structures/static_rtree.hpp b/data_structures/static_rtree.hpp index e5b7be92cbf..bd88226ab8b 100644 --- a/data_structures/static_rtree.hpp +++ b/data_structures/static_rtree.hpp @@ -783,32 +783,18 @@ class StaticRTree // Returns elements within max_distance. // If the minium of elements could not be found in the search radius, widen // it until the minimum can be satisfied. - // At the number of returned nodes is capped at the given maximum. bool IncrementalFindPhantomNodeForCoordinateWithDistance( const FixedPointCoordinate &input_coordinate, std::vector> &result_phantom_node_vector, const double max_distance, - const unsigned min_number_of_phantom_nodes, - const unsigned max_number_of_phantom_nodes, const unsigned max_checked_elements = 4 * LEAF_NODE_SIZE) { unsigned inspected_elements = 0; - unsigned number_of_elements_from_big_cc = 0; - unsigned number_of_elements_from_tiny_cc = 0; - - // is true if a big cc was added to the queue to we also have a lower bound - // for them. it actives pruning for big components - bool has_big_cc = false; - - unsigned pruned_elements = 0; std::pair projected_coordinate = { mercator::lat2y(input_coordinate.lat / COORDINATE_PRECISION), input_coordinate.lon / COORDINATE_PRECISION}; - // upper bound pruning technique - upper_bound pruning_bound(max_number_of_phantom_nodes); - // initialize queue with root element std::priority_queue traversal_queue; traversal_queue.emplace(0.f, m_search_tree[0]); @@ -818,6 +804,11 @@ class StaticRTree const IncrementalQueryCandidate current_query_node = traversal_queue.top(); traversal_queue.pop(); + if (current_query_node.min_dist > max_distance || inspected_elements >= max_checked_elements) + { + break; + } + if (current_query_node.node.template is()) { // current object is a tree node const TreeNode ¤t_tree_node = @@ -839,16 +830,9 @@ class StaticRTree // distance must be non-negative BOOST_ASSERT(0.f <= current_perpendicular_distance); - if (pruning_bound.get() >= current_perpendicular_distance || - (!has_big_cc && !current_edge.is_in_tiny_cc())) + if (current_perpendicular_distance <= max_distance) { - pruning_bound.insert(current_perpendicular_distance); traversal_queue.emplace(current_perpendicular_distance, current_edge); - has_big_cc = has_big_cc || !current_edge.is_in_tiny_cc(); - } - else - { - ++pruned_elements; } } } @@ -865,7 +849,10 @@ class StaticRTree child_rectangle.GetMinDist(input_coordinate); BOOST_ASSERT(0.f <= lower_bound_to_element); - traversal_queue.emplace(lower_bound_to_element, child_tree_node); + if (lower_bound_to_element <= max_distance) + { + traversal_queue.emplace(lower_bound_to_element, child_tree_node); + } } } } @@ -876,14 +863,6 @@ class StaticRTree const EdgeDataT ¤t_segment = current_query_node.node.template get(); - // continue searching for the first segment from a big component - if (number_of_elements_from_big_cc == 0 && - number_of_elements_from_tiny_cc >= max_number_of_phantom_nodes - 1 && - current_segment.is_in_tiny_cc()) - { - continue; - } - // check if it is smaller than what we had before float current_ratio = 0.f; FixedPointCoordinate foot_point_coordinate_on_segment; @@ -894,9 +873,7 @@ class StaticRTree m_coordinate_list->at(current_segment.v), input_coordinate, projected_coordinate, foot_point_coordinate_on_segment, current_ratio); - if (number_of_elements_from_big_cc > 0 && - result_phantom_node_vector.size() >= min_number_of_phantom_nodes && - current_perpendicular_distance >= max_distance) + if (current_perpendicular_distance >= max_distance) { traversal_queue = std::priority_queue{}; continue; @@ -920,36 +897,14 @@ class StaticRTree // set forward and reverse weights on the phantom node SetForwardAndReverseWeightsOnPhantomNode(current_segment, result_phantom_node_vector.back().first); - - // update counts on what we found from which result class - if (current_segment.is_in_tiny_cc()) - { // found an element in tiny component - ++number_of_elements_from_tiny_cc; - } - else - { // found an element in a big component - ++number_of_elements_from_big_cc; - } } // stop the search by flushing the queue - if ((result_phantom_node_vector.size() >= max_number_of_phantom_nodes && - number_of_elements_from_big_cc > 0) || - inspected_elements >= max_checked_elements) + if (inspected_elements >= max_checked_elements) { traversal_queue = std::priority_queue{}; } } - // SimpleLogger().Write() << "result_phantom_node_vector.size(): " << - // result_phantom_node_vector.size(); - // SimpleLogger().Write() << "max_number_of_phantom_nodes: " << max_number_of_phantom_nodes; - // SimpleLogger().Write() << "number_of_elements_from_big_cc: " << - // number_of_elements_from_big_cc; - // SimpleLogger().Write() << "number_of_elements_from_tiny_cc: " << - // number_of_elements_from_tiny_cc; - // SimpleLogger().Write() << "inspected_elements: " << inspected_elements; - // SimpleLogger().Write() << "max_checked_elements: " << max_checked_elements; - // SimpleLogger().Write() << "pruned_elements: " << pruned_elements; return !result_phantom_node_vector.empty(); } diff --git a/plugins/match.hpp b/plugins/match.hpp index a76ab634f22..3da7e14af2a 100644 --- a/plugins/match.hpp +++ b/plugins/match.hpp @@ -92,11 +92,13 @@ template class MapMatchingPlugin : public BasePlugin } bool getCandiates(const std::vector &input_coords, + const double gps_precision, std::vector &sub_trace_lengths, osrm::matching::CandidateLists &candidates_lists) { - double last_distance = - coordinate_calculation::great_circle_distance(input_coords[0], input_coords[1]); + double query_radius = 10 * gps_precision; + double last_distance = coordinate_calculation::great_circle_distance(input_coords[0], input_coords[1]); + sub_trace_lengths.resize(input_coords.size()); sub_trace_lengths[0] = 0; for (const auto current_coordinate : osrm::irange(0, input_coords.size())) @@ -104,8 +106,8 @@ template class MapMatchingPlugin : public BasePlugin bool allow_uturn = false; if (0 < current_coordinate) { - last_distance = coordinate_calculation::great_circle_distance( - input_coords[current_coordinate - 1], input_coords[current_coordinate]); + last_distance = coordinate_calculation::great_circle_distance(input_coords[current_coordinate - 1], input_coords[current_coordinate]); + sub_trace_lengths[current_coordinate] += sub_trace_lengths[current_coordinate - 1] + last_distance; } @@ -124,18 +126,27 @@ template class MapMatchingPlugin : public BasePlugin } std::vector> candidates; - if (!facade->IncrementalFindPhantomNodeForCoordinateWithMaxDistance( - input_coords[current_coordinate], candidates, last_distance / 2.0, 5, - max_number_of_candidates)) - { - return false; - } - - if (allow_uturn) - { - candidates_lists.push_back(candidates); - } - else + facade->IncrementalFindPhantomNodeForCoordinateWithMaxDistance( + input_coords[current_coordinate], candidates, query_radius); + + // sort by foward id, then by reverse id and then by distance + std::sort(candidates.begin(), candidates.end(), + [](const std::pair& lhs, const std::pair& rhs) { + return lhs.first.forward_node_id < rhs.first.forward_node_id || + (lhs.first.forward_node_id == rhs.first.forward_node_id && + (lhs.first.reverse_node_id < rhs.first.reverse_node_id || + (lhs.first.reverse_node_id == rhs.first.reverse_node_id && + lhs.second < rhs.second))); + }); + + auto new_end = std::unique(candidates.begin(), candidates.end(), + [](const std::pair& lhs, const std::pair& rhs) { + return lhs.first.forward_node_id == rhs.first.forward_node_id && + lhs.first.reverse_node_id == rhs.first.reverse_node_id; + }); + candidates.resize(new_end - candidates.begin()); + + if (!allow_uturn) { const auto compact_size = candidates.size(); for (const auto i : osrm::irange(0, compact_size)) @@ -151,8 +162,15 @@ template class MapMatchingPlugin : public BasePlugin candidates[i].first.reverse_node_id = SPECIAL_NODEID; } } - candidates_lists.push_back(candidates); } + + // sort by distance to make pruning effective + std::sort(candidates.begin(), candidates.end(), + [](const std::pair& lhs, const std::pair& rhs) { + return lhs.second < rhs.second; + }); + + candidates_lists.push_back(std::move(candidates)); } return true; @@ -246,7 +264,7 @@ template class MapMatchingPlugin : public BasePlugin } const bool found_candidates = - getCandiates(input_coords, sub_trace_lengths, candidates_lists); + getCandiates(input_coords, route_parameters.gps_precision, sub_trace_lengths, candidates_lists); if (!found_candidates) { json_result.values["status"] = "No suitable matching candidates found."; diff --git a/server/data_structures/datafacade_base.hpp b/server/data_structures/datafacade_base.hpp index fc4519da2fc..baea2bf0ee8 100644 --- a/server/data_structures/datafacade_base.hpp +++ b/server/data_structures/datafacade_base.hpp @@ -107,9 +107,7 @@ template class BaseDataFacade virtual bool IncrementalFindPhantomNodeForCoordinateWithMaxDistance( const FixedPointCoordinate &input_coordinate, std::vector> &resulting_phantom_node_vector, - const double max_distance, - const unsigned min_number_of_phantom_nodes, - const unsigned max_number_of_phantom_nodes) = 0; + const double max_distance) = 0; virtual unsigned GetCheckSum() const = 0; diff --git a/server/data_structures/internal_datafacade.hpp b/server/data_structures/internal_datafacade.hpp index 0678b09370d..357ee2532aa 100644 --- a/server/data_structures/internal_datafacade.hpp +++ b/server/data_structures/internal_datafacade.hpp @@ -453,9 +453,7 @@ template class InternalDataFacade final : public BaseDataFacad bool IncrementalFindPhantomNodeForCoordinateWithMaxDistance( const FixedPointCoordinate &input_coordinate, std::vector> &resulting_phantom_node_vector, - const double max_distance, - const unsigned min_number_of_phantom_nodes, - const unsigned max_number_of_phantom_nodes) override final + const double max_distance) override final { if (!m_static_rtree.get()) { @@ -463,8 +461,7 @@ template class InternalDataFacade final : public BaseDataFacad } return m_static_rtree->IncrementalFindPhantomNodeForCoordinateWithDistance( - input_coordinate, resulting_phantom_node_vector, max_distance, - min_number_of_phantom_nodes, max_number_of_phantom_nodes); + input_coordinate, resulting_phantom_node_vector, max_distance); } unsigned GetCheckSum() const override final { return m_check_sum; } diff --git a/server/data_structures/shared_datafacade.hpp b/server/data_structures/shared_datafacade.hpp index d9d907a0abc..928a12540c4 100644 --- a/server/data_structures/shared_datafacade.hpp +++ b/server/data_structures/shared_datafacade.hpp @@ -424,9 +424,7 @@ template class SharedDataFacade final : public BaseDataFacade< bool IncrementalFindPhantomNodeForCoordinateWithMaxDistance( const FixedPointCoordinate &input_coordinate, std::vector> &resulting_phantom_node_vector, - const double max_distance, - const unsigned min_number_of_phantom_nodes, - const unsigned max_number_of_phantom_nodes) override final + const double max_distance) override final { if (!m_static_rtree.get() || CURRENT_TIMESTAMP != m_static_rtree->first) { @@ -434,8 +432,7 @@ template class SharedDataFacade final : public BaseDataFacade< } return m_static_rtree->second->IncrementalFindPhantomNodeForCoordinateWithDistance( - input_coordinate, resulting_phantom_node_vector, max_distance, - min_number_of_phantom_nodes, max_number_of_phantom_nodes); + input_coordinate, resulting_phantom_node_vector, max_distance); } unsigned GetCheckSum() const override final { return m_check_sum; } From f167c3e12ebc1356ce247d9061015458d80ba4ff Mon Sep 17 00:00:00 2001 From: Patrick Niklaus Date: Tue, 1 Sep 2015 00:33:35 +0200 Subject: [PATCH 096/122] Move heap initialization out of loop --- routing_algorithms/map_matching.hpp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/routing_algorithms/map_matching.hpp b/routing_algorithms/map_matching.hpp index 4c8e561a0e0..818a4453547 100644 --- a/routing_algorithms/map_matching.hpp +++ b/routing_algorithms/map_matching.hpp @@ -139,6 +139,12 @@ class MapMatching final : public BasicRoutingInterfaceGetNumberOfNodes()); + + QueryHeap &forward_heap = *(engine_working_data.forward_heap_1); + QueryHeap &reverse_heap = *(engine_working_data.reverse_heap_1); + std::size_t breakage_begin = osrm::matching::INVALID_STATE; std::vector split_points; std::vector prev_unbroken_timestamps; @@ -206,12 +212,6 @@ class MapMatching final : public BasicRoutingInterfaceGetNumberOfNodes()); - - QueryHeap &forward_heap = *(engine_working_data.forward_heap_1); - QueryHeap &reverse_heap = *(engine_working_data.reverse_heap_1); - // compute d_t for this timestamp and the next one for (const auto s : osrm::irange(0u, prev_viterbi.size())) { From 57608628a4844d20dc9879799b54173f1e7150f6 Mon Sep 17 00:00:00 2001 From: Patrick Niklaus Date: Tue, 1 Sep 2015 23:03:54 +0200 Subject: [PATCH 097/122] Use median sample time instead of average to harden against outliers --- routing_algorithms/map_matching.hpp | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/routing_algorithms/map_matching.hpp b/routing_algorithms/map_matching.hpp index 818a4453547..b28d397edf7 100644 --- a/routing_algorithms/map_matching.hpp +++ b/routing_algorithms/map_matching.hpp @@ -74,7 +74,7 @@ class MapMatching final : public BasicRoutingInterface& timestamps) const + unsigned GetMedianSampleTime(const std::vector& timestamps) const { BOOST_ASSERT(timestamps.size() > 1); @@ -83,8 +83,10 @@ class MapMatching final : public BasicRoutingInterface will not be a difference. - auto sum_sample_times = std::accumulate(std::next(sample_times.begin()), sample_times.end(), 0); - return sum_sample_times / (sample_times.size() - 1); + auto first_elem = std::next(sample_times.begin()); + auto median = first_elem + std::distance(first_elem, sample_times.end())/2; + std::nth_element(first_elem, median, sample_times.end()); + return *median; } public: @@ -102,21 +104,21 @@ class MapMatching final : public BasicRoutingInterface 1) { - return GetAverageSampleTime(trace_timestamps); + return GetMedianSampleTime(trace_timestamps); } else { return 0u; } }(); - const auto max_broken_time = avg_sample_time * osrm::matching::MAX_BROKEN_STATES; + const auto max_broken_time = median_sample_time * osrm::matching::MAX_BROKEN_STATES; const auto max_distance_delta = [&]() { if (trace_timestamps.size() > 1) { - return avg_sample_time * osrm::matching::MAX_SPEED; + return median_sample_time * osrm::matching::MAX_SPEED; } else { From f11bd509b0f50f0ca57a2496ba3acd3326df8e25 Mon Sep 17 00:00:00 2001 From: Patrick Niklaus Date: Wed, 2 Sep 2015 00:39:50 +0200 Subject: [PATCH 098/122] Also prune on MAX_DOUBLE --- plugins/match.hpp | 2 ++ routing_algorithms/map_matching.hpp | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/plugins/match.hpp b/plugins/match.hpp index 3da7e14af2a..120fba1ef6b 100644 --- a/plugins/match.hpp +++ b/plugins/match.hpp @@ -324,6 +324,8 @@ template class MapMatchingPlugin : public BasePlugin raw_route.segment_end_coordinates, std::vector(raw_route.segment_end_coordinates.size(), true), raw_route); + BOOST_ASSERT(raw_route.shortest_path_length != INVALID_EDGE_WEIGHT); + matchings.values.emplace_back(submatchingToJSON(sub, route_parameters, raw_route)); } diff --git a/routing_algorithms/map_matching.hpp b/routing_algorithms/map_matching.hpp index b28d397edf7..b42e4804fc8 100644 --- a/routing_algorithms/map_matching.hpp +++ b/routing_algorithms/map_matching.hpp @@ -248,7 +248,7 @@ class MapMatching final : public BasicRoutingInterface prune - if (d_t > max_distance_delta) + if (d_t >= max_distance_delta) { continue; } From 0b532425640c4ed03ce062fedc11ceee11c85c07 Mon Sep 17 00:00:00 2001 From: Patrick Niklaus Date: Wed, 2 Sep 2015 00:40:42 +0200 Subject: [PATCH 099/122] Move distance calculation out of loop --- routing_algorithms/map_matching.hpp | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/routing_algorithms/map_matching.hpp b/routing_algorithms/map_matching.hpp index b42e4804fc8..0993b2eefb3 100644 --- a/routing_algorithms/map_matching.hpp +++ b/routing_algorithms/map_matching.hpp @@ -214,6 +214,8 @@ class MapMatching final : public BasicRoutingInterface(0u, prev_viterbi.size())) { @@ -241,9 +243,6 @@ class MapMatching final : public BasicRoutingInterface Date: Wed, 2 Sep 2015 00:42:13 +0200 Subject: [PATCH 100/122] Fix failing matching tests due to gps precision --- features/testbot/matching.feature | 1 + features/testbot/post.feature | 1 + 2 files changed, 2 insertions(+) diff --git a/features/testbot/matching.feature b/features/testbot/matching.feature index 774e8a9d693..91bf928fa73 100644 --- a/features/testbot/matching.feature +++ b/features/testbot/matching.feature @@ -40,6 +40,7 @@ Feature: Basic Map Matching | afcde | abcde | Scenario: Testbot - Map matching with oneways + Given a grid size of 10 meters Given the node map | a | b | c | d | | e | f | g | h | diff --git a/features/testbot/post.feature b/features/testbot/post.feature index ff2282406e9..ceacbb0ee4d 100644 --- a/features/testbot/post.feature +++ b/features/testbot/post.feature @@ -30,6 +30,7 @@ Feature: POST request | z | x | yz,xy | head,left,destination | Scenario: Testbot - match POST request + Given a grid size of 10 meters Given the node map | a | b | c | d | | e | f | g | h | From 70bb0829739bab0a49ce3bc2f2e8ae079c03b1b7 Mon Sep 17 00:00:00 2001 From: Patrick Niklaus Date: Thu, 3 Sep 2015 10:40:16 +0200 Subject: [PATCH 101/122] Fix endless loop --- data_structures/hidden_markov_model.hpp | 1 + routing_algorithms/map_matching.hpp | 12 ++++++++++-- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/data_structures/hidden_markov_model.hpp b/data_structures/hidden_markov_model.hpp index 60738a56148..15aa35b8b68 100644 --- a/data_structures/hidden_markov_model.hpp +++ b/data_structures/hidden_markov_model.hpp @@ -96,6 +96,7 @@ template struct HiddenMarkovModel path_lengths.resize(candidates_list.size()); suspicious.resize(candidates_list.size()); pruned.resize(candidates_list.size()); + breakage.resize(candidates_list.size()); for (const auto i : osrm::irange(0u, candidates_list.size())) { const auto& num_candidates = candidates_list[i].size(); diff --git a/routing_algorithms/map_matching.hpp b/routing_algorithms/map_matching.hpp index 0993b2eefb3..58b0ad1a8d8 100644 --- a/routing_algorithms/map_matching.hpp +++ b/routing_algorithms/map_matching.hpp @@ -301,14 +301,17 @@ class MapMatching final : public BasicRoutingInterface= sub_matching_begin && model.breakage[parent_timestamp_index]) { --parent_timestamp_index; } + while (sub_matching_begin < sub_matching_end && + model.breakage[sub_matching_begin]) + { + ++sub_matching_begin; + } // matchings that only consist of one candidate are invalid if (parent_timestamp_index - sub_matching_begin + 1 < 2) @@ -335,6 +338,11 @@ class MapMatching final : public BasicRoutingInterface Date: Wed, 2 Sep 2015 12:03:07 +0200 Subject: [PATCH 102/122] Make pedestrian roads marked as destination routable with car profile. Check provided tests with: cucumber --tags @access -p verify References: - https://github.com/Project-OSRM/osrm-backend/issues/1617 - http://wiki.openstreetmap.org/wiki/Tag:highway%3Dpedestrian - http://wiki.openstreetmap.org/wiki/Key:motorcar - http://wiki.openstreetmap.org/wiki/Key:access --- features/car/access.feature | 16 +++++++++------- profiles/car.lua | 6 +++++- 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/features/car/access.feature b/features/car/access.feature index 5fd56f39515..8f05ccbcad6 100644 --- a/features/car/access.feature +++ b/features/car/access.feature @@ -125,13 +125,15 @@ Feature: Car - Restricted access Scenario: Car - Access combinations Then routability should be - | highway | accesss | vehicle | motor_vehicle | motorcar | bothw | - | runway | private | | | permissive | x | - | primary | forestry | | yes | | x | - | cycleway | | | designated | | x | - | residential | | yes | no | | | - | motorway | yes | permissive | | private | | - | trunk | agricultural | designated | permissive | no | | + | highway | accesss | vehicle | motor_vehicle | motorcar | bothw | + | runway | private | | | permissive | x | + | primary | forestry | | yes | | x | + | cycleway | | | designated | | x | + | residential | | yes | no | | | + | motorway | yes | permissive | | private | | + | trunk | agricultural | designated | permissive | no | | + | pedestrian | | | | | | + | pedestrian | | | | destination | x | Scenario: Car - Ignore access tags for other modes Then routability should be diff --git a/profiles/car.lua b/profiles/car.lua index c6d443ccdf0..b542f262026 100644 --- a/profiles/car.lua +++ b/profiles/car.lua @@ -302,8 +302,12 @@ function way_function (way, result) result.backward_speed = highway_speed end else + -- make an exception for restricted highway=pedestrian with motorcar=destination + local motorcar = way:get_value_by_key("motorcar") + local pedestrian_road = motorcar and "destination" == motorcar and "pedestrian" == highway + -- Set the avg speed on ways that are marked accessible - if access_tag_whitelist[access] then + if access_tag_whitelist[access] or pedestrian_road then result.forward_speed = speed_profile["default"] result.backward_speed = speed_profile["default"] end From 6cbbd1e5a1b441eb27055f56956e1bac14832a58 Mon Sep 17 00:00:00 2001 From: "Daniel J. Hofmann" Date: Thu, 3 Sep 2015 11:15:03 +0200 Subject: [PATCH 103/122] Move destination to access tag white list instead of making exception in car profile. Tested with: cucumber --tags @access -p verify References: - https://github.com/Project-OSRM/osrm-backend/issues/1617 - https://github.com/Project-OSRM/osrm-backend/pull/1639 --- profiles/car.lua | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/profiles/car.lua b/profiles/car.lua index b542f262026..2f7f19d5560 100644 --- a/profiles/car.lua +++ b/profiles/car.lua @@ -4,7 +4,7 @@ local find_access_tag = require("lib/access").find_access_tag -- Begin of globals barrier_whitelist = { ["cattle_grid"] = true, ["border_control"] = true, ["checkpoint"] = true, ["toll_booth"] = true, ["sally_port"] = true, ["gate"] = true, ["lift_gate"] = true, ["no"] = true, ["entrance"] = true } -access_tag_whitelist = { ["yes"] = true, ["motorcar"] = true, ["motor_vehicle"] = true, ["vehicle"] = true, ["permissive"] = true, ["designated"] = true } +access_tag_whitelist = { ["yes"] = true, ["motorcar"] = true, ["motor_vehicle"] = true, ["vehicle"] = true, ["permissive"] = true, ["designated"] = true, ["destination"] = true } access_tag_blacklist = { ["no"] = true, ["private"] = true, ["agricultural"] = true, ["forestry"] = true, ["emergency"] = true, ["psv"] = true } access_tag_restricted = { ["destination"] = true, ["delivery"] = true } access_tags = { "motorcar", "motor_vehicle", "vehicle" } @@ -302,12 +302,8 @@ function way_function (way, result) result.backward_speed = highway_speed end else - -- make an exception for restricted highway=pedestrian with motorcar=destination - local motorcar = way:get_value_by_key("motorcar") - local pedestrian_road = motorcar and "destination" == motorcar and "pedestrian" == highway - -- Set the avg speed on ways that are marked accessible - if access_tag_whitelist[access] or pedestrian_road then + if access_tag_whitelist[access] then result.forward_speed = speed_profile["default"] result.backward_speed = speed_profile["default"] end From bac6703f8e0d089aa58b7fefee8bba7204e4e492 Mon Sep 17 00:00:00 2001 From: Lauren Budorick Date: Fri, 29 May 2015 17:28:29 -0700 Subject: [PATCH 104/122] Implement raster source feature to read data from third-party sources, to be used in lua profiles. * Adds a data structure, RasterSource, to store parsed + queryable data * Adds bindings for that and relevant data structures as well as source_function and segment_function * Adds relevant unit tests and cucumber tests * Bring-your-own-data feature --- CMakeLists.txt | 5 +- appveyor-build.bat | 6 +- data_structures/raster_source.cpp | 178 +++++++++++++++++++ data_structures/raster_source.hpp | 175 ++++++++++++++++++ extractor/extraction_containers.cpp | 21 ++- extractor/extraction_containers.hpp | 6 +- extractor/extractor.cpp | 17 +- extractor/lat | 0 extractor/scripting_environment.cpp | 26 ++- extractor/source_coordinate.lat | 0 features/raster/extract.feature | 18 ++ features/raster/weights.feature | 78 ++++++++ features/step_definitions/data.rb | 6 + features/support/env.rb | 3 +- profiles/rasterbot-interp.lua | 46 +++++ profiles/rasterbot.lua | 46 +++++ test/rastersource.asc | 5 + unit_tests/data_structures/raster_source.cpp | 110 ++++++++++++ unit_tests/fixtures/raster_data.asc | 10 ++ 19 files changed, 744 insertions(+), 12 deletions(-) create mode 100644 data_structures/raster_source.cpp create mode 100644 data_structures/raster_source.hpp create mode 100644 extractor/lat create mode 100644 extractor/source_coordinate.lat create mode 100644 features/raster/extract.feature create mode 100644 features/raster/weights.feature create mode 100644 profiles/rasterbot-interp.lua create mode 100644 profiles/rasterbot.lua create mode 100644 test/rastersource.asc create mode 100644 unit_tests/data_structures/raster_source.cpp create mode 100644 unit_tests/fixtures/raster_data.asc diff --git a/CMakeLists.txt b/CMakeLists.txt index 990ce884c19..e0b8f10166d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -51,10 +51,11 @@ configure_file( ${CMAKE_CURRENT_SOURCE_DIR}/util/git_sha.cpp ) file(GLOB ExtractorGlob extractor/*.cpp) -file(GLOB ImporterGlob data_structures/import_edge.cpp data_structures/external_memory_node.cpp) +file(GLOB ImporterGlob data_structures/import_edge.cpp data_structures/external_memory_node.cpp data_structures/raster_source.cpp) add_library(IMPORT OBJECT ${ImporterGlob}) add_library(LOGGER OBJECT util/simple_logger.cpp) add_library(PHANTOMNODE OBJECT data_structures/phantom_node.cpp) +add_library(RASTERSOURCE OBJECT data_structures/raster_source.cpp) add_library(EXCEPTION OBJECT util/osrm_exception.cpp) add_library(MERCATOR OBJECT util/mercator.cpp) add_library(ANGLE OBJECT util/compute_angle.cpp) @@ -102,7 +103,7 @@ add_executable(osrm-routed routed.cpp ${ServerGlob} $) add_executable(osrm-datastore datastore.cpp $ $ $ $ $ $) # Unit tests -add_executable(datastructure-tests EXCLUDE_FROM_ALL unit_tests/datastructure_tests.cpp ${DataStructureTestsGlob} $ $ $ $ $ $ $ $) +add_executable(datastructure-tests EXCLUDE_FROM_ALL unit_tests/datastructure_tests.cpp ${DataStructureTestsGlob} $ $ $ $ $ $ $ $ $) add_executable(algorithm-tests EXCLUDE_FROM_ALL unit_tests/algorithm_tests.cpp ${AlgorithmTestsGlob} $ $ $ $ $ $) # Benchmarks diff --git a/appveyor-build.bat b/appveyor-build.bat index 9e49f1ae8ad..cffa5261ab4 100644 --- a/appveyor-build.bat +++ b/appveyor-build.bat @@ -65,12 +65,14 @@ IF %ERRORLEVEL% NEQ 0 GOTO ERROR SET PATH=c:\projects\osrm\osrm-deps\libs\bin;%PATH% +CD .. ECHO running datastructure-tests.exe ... -datastructure-tests.exe +%Configuration%\datastructure-tests.exe IF %ERRORLEVEL% NEQ 0 GOTO ERROR ECHO running algorithm-tests.exe ... -algorithm-tests.exe +%Configuration%\algorithm-tests.exe IF %ERRORLEVEL% NEQ 0 GOTO ERROR +cd %Configuration% IF NOT "%APPVEYOR_REPO_BRANCH%"=="develop" GOTO DONE ECHO ========= CREATING PACKAGES ========== diff --git a/data_structures/raster_source.cpp b/data_structures/raster_source.cpp new file mode 100644 index 00000000000..f935b9d8715 --- /dev/null +++ b/data_structures/raster_source.cpp @@ -0,0 +1,178 @@ +/* + +Copyright (c) 2015, Project OSRM contributors +All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + +Redistributions of source code must retain the above copyright notice, this list +of conditions and the following disclaimer. +Redistributions in binary form must reproduce the above copyright notice, this +list of conditions and the following disclaimer in the documentation and/or +other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +*/ + +#include "raster_source.hpp" + +#include "../util/simple_logger.hpp" +#include "../util/timing_util.hpp" + +#include + +#include + +RasterSource::RasterSource(RasterGrid _raster_data, + std::size_t _width, + std::size_t _height, + int _xmin, + int _xmax, + int _ymin, + int _ymax) + : xstep(calcSize(_xmin, _xmax, _width)), ystep(calcSize(_ymin, _ymax, _height)), + raster_data(_raster_data), width(_width), height(_height), xmin(_xmin), xmax(_xmax), + ymin(_ymin), ymax(_ymax) +{ + BOOST_ASSERT(xstep != 0); + BOOST_ASSERT(ystep != 0); +} + +float RasterSource::calcSize(int min, int max, std::size_t count) const +{ + BOOST_ASSERT(count > 0); + return (max - min) / (static_cast(count) - 1); +} + +// Query raster source for nearest data point +RasterDatum RasterSource::getRasterData(const int lon, const int lat) const +{ + if (lon < xmin || lon > xmax || lat < ymin || lat > ymax) + { + return {}; + } + + const std::size_t xth = static_cast(round((lon - xmin) / xstep)); + const std::size_t yth = static_cast(round((ymax - lat) / ystep)); + + return {raster_data(xth, yth)}; +} + +// Query raster source using bilinear interpolation +RasterDatum RasterSource::getRasterInterpolate(const int lon, const int lat) const +{ + if (lon < xmin || lon > xmax || lat < ymin || lat > ymax) + { + return {}; + } + + const auto xthP = (lon - xmin) / xstep; + const auto ythP = (ymax - lat) / ystep; + + const std::size_t top = static_cast(fmax(floor(ythP), 0)); + const std::size_t bottom = static_cast(fmin(ceil(ythP), height - 1)); + const std::size_t left = static_cast(fmax(floor(xthP), 0)); + const std::size_t right = static_cast(fmin(ceil(xthP), width - 1)); + + // Calculate distances from corners for bilinear interpolation + const float fromLeft = (lon - left * xstep + xmin) / xstep; + const float fromTop = (ymax - top * ystep - lat) / ystep; + const float fromRight = 1 - fromLeft; + const float fromBottom = 1 - fromTop; + + return {static_cast(raster_data(left, top) * (fromRight * fromBottom) + + raster_data(right, top) * (fromLeft * fromBottom) + + raster_data(left, bottom) * (fromRight * fromTop) + + raster_data(right, bottom) * (fromLeft * fromTop))}; +} + +// Load raster source into memory +int SourceContainer::loadRasterSource(const std::string &path_string, + double xmin, + double xmax, + double ymin, + double ymax, + std::size_t nrows, + std::size_t ncols) +{ + const auto _xmin = static_cast(xmin * COORDINATE_PRECISION); + const auto _xmax = static_cast(xmax * COORDINATE_PRECISION); + const auto _ymin = static_cast(ymin * COORDINATE_PRECISION); + const auto _ymax = static_cast(ymax * COORDINATE_PRECISION); + + const auto itr = LoadedSourcePaths.find(path_string); + if (itr != LoadedSourcePaths.end()) + { + SimpleLogger().Write() << "[source loader] Already loaded source '" << path_string + << "' at source_id " << itr->second; + return itr->second; + } + + int source_id = static_cast(LoadedSources.size()); + + SimpleLogger().Write() << "[source loader] Loading from " << path_string << " ... "; + TIMER_START(loading_source); + + boost::filesystem::path filepath(path_string); + if (!boost::filesystem::exists(filepath)) + { + throw osrm::exception("error reading: no such path"); + } + + RasterGrid rasterData{filepath, ncols, nrows}; + + RasterSource source{std::move(rasterData), ncols, nrows, _xmin, _xmax, _ymin, _ymax}; + TIMER_STOP(loading_source); + LoadedSourcePaths.emplace(path_string, source_id); + LoadedSources.push_back(std::move(source)); + + SimpleLogger().Write() << "[source loader] ok, after " << TIMER_SEC(loading_source) << "s"; + + return source_id; +} + +// External function for looking up nearest data point from a specified source +RasterDatum SourceContainer::getRasterDataFromSource(unsigned int source_id, int lon, int lat) +{ + if (LoadedSources.size() < source_id + 1) + { + throw osrm::exception("error reading: no such loaded source"); + } + + BOOST_ASSERT(lat < (90 * COORDINATE_PRECISION)); + BOOST_ASSERT(lat > (-90 * COORDINATE_PRECISION)); + BOOST_ASSERT(lon < (180 * COORDINATE_PRECISION)); + BOOST_ASSERT(lon > (-180 * COORDINATE_PRECISION)); + + const auto &found = LoadedSources[source_id]; + return found.getRasterData(lon, lat); +} + +// External function for looking up interpolated data from a specified source +RasterDatum +SourceContainer::getRasterInterpolateFromSource(unsigned int source_id, int lon, int lat) +{ + if (LoadedSources.size() < source_id + 1) + { + throw osrm::exception("error reading: no such loaded source"); + } + + BOOST_ASSERT(lat < (90 * COORDINATE_PRECISION)); + BOOST_ASSERT(lat > (-90 * COORDINATE_PRECISION)); + BOOST_ASSERT(lon < (180 * COORDINATE_PRECISION)); + BOOST_ASSERT(lon > (-180 * COORDINATE_PRECISION)); + + const auto &found = LoadedSources[source_id]; + return found.getRasterInterpolate(lon, lat); +} diff --git a/data_structures/raster_source.hpp b/data_structures/raster_source.hpp new file mode 100644 index 00000000000..86c859639fa --- /dev/null +++ b/data_structures/raster_source.hpp @@ -0,0 +1,175 @@ +/* + +Copyright (c) 2015, Project OSRM contributors +All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + +Redistributions of source code must retain the above copyright notice, this list +of conditions and the following disclaimer. +Redistributions in binary form must reproduce the above copyright notice, this +list of conditions and the following disclaimer in the documentation and/or +other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +*/ + +#ifndef RASTER_SOURCE_HPP +#define RASTER_SOURCE_HPP + +#include "../util/osrm_exception.hpp" + +#include +#include +#include +#include +#include +#include + +#include +#include + +/** + \brief Small wrapper around raster source queries to optionally provide results + gracefully, depending on source bounds +*/ +struct RasterDatum +{ + static std::int32_t get_invalid() { return std::numeric_limits::max(); } + + std::int32_t datum = get_invalid(); + + RasterDatum() = default; + + RasterDatum(std::int32_t _datum) : datum(_datum) {} +}; + +class RasterGrid +{ + public: + RasterGrid(const boost::filesystem::path &filepath, std::size_t _xdim, std::size_t _ydim) + { + xdim = _xdim; + ydim = _ydim; + _data.reserve(ydim * xdim); + + boost::filesystem::ifstream stream(filepath); + if (!stream) + { + throw osrm::exception("Unable to open raster file."); + } + + stream.seekg(0, std::ios_base::end); + std::string buffer; + buffer.resize(static_cast(stream.tellg())); + + stream.seekg(0, std::ios_base::beg); + + BOOST_ASSERT(buffer.size() > 1); + stream.read(&buffer[0], static_cast(buffer.size())); + + boost::algorithm::trim(buffer); + + auto itr = buffer.begin(); + auto end = buffer.end(); + + bool r = false; + try + { + r = boost::spirit::qi::parse( + itr, end, +boost::spirit::qi::int_ % +boost::spirit::qi::space, _data); + } + catch (std::exception const &ex) + { + throw osrm::exception( + std::string("Failed to read from raster source with exception: ") + ex.what()); + } + + if (!r || itr != end) + { + throw osrm::exception("Failed to parse raster source correctly."); + } + } + + RasterGrid(const RasterGrid &) = default; + RasterGrid &operator=(const RasterGrid &) = default; + + RasterGrid(RasterGrid &&) = default; + RasterGrid &operator=(RasterGrid &&) = default; + + std::int32_t operator()(std::size_t x, std::size_t y) { return _data[y * xdim + x]; } + std::int32_t operator()(std::size_t x, std::size_t y) const { return _data[(y)*xdim + (x)]; } + + private: + std::vector _data; + std::size_t xdim, ydim; +}; + +/** + \brief Stores raster source data in memory and provides lookup functions. +*/ +class RasterSource +{ + private: + const float xstep; + const float ystep; + + float calcSize(int min, int max, std::size_t count) const; + + public: + RasterGrid raster_data; + + const std::size_t width; + const std::size_t height; + const int xmin; + const int xmax; + const int ymin; + const int ymax; + + RasterDatum getRasterData(const int lon, const int lat) const; + + RasterDatum getRasterInterpolate(const int lon, const int lat) const; + + RasterSource(RasterGrid _raster_data, + std::size_t width, + std::size_t height, + int _xmin, + int _xmax, + int _ymin, + int _ymax); +}; + +class SourceContainer +{ + public: + SourceContainer() = default; + + int loadRasterSource(const std::string &path_string, + double xmin, + double xmax, + double ymin, + double ymax, + std::size_t nrows, + std::size_t ncols); + + RasterDatum getRasterDataFromSource(unsigned int source_id, int lon, int lat); + + RasterDatum getRasterInterpolateFromSource(unsigned int source_id, int lon, int lat); + + private: + std::vector LoadedSources; + std::unordered_map LoadedSourcePaths; +}; + +#endif /* RASTER_SOURCE_HPP */ diff --git a/extractor/extraction_containers.cpp b/extractor/extraction_containers.cpp index 071ea97e6d3..e5344de35cc 100644 --- a/extractor/extraction_containers.cpp +++ b/extractor/extraction_containers.cpp @@ -36,10 +36,14 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "../util/simple_logger.hpp" #include "../util/timing_util.hpp" #include "../util/fingerprint.hpp" +#include "../util/lua_util.hpp" #include #include #include +#include + +#include #include @@ -76,7 +80,8 @@ ExtractionContainers::~ExtractionContainers() */ void ExtractionContainers::PrepareData(const std::string &output_file_name, const std::string &restrictions_file_name, - const std::string &name_file_name) + const std::string &name_file_name, + lua_State *segment_state) { try { @@ -87,7 +92,7 @@ void ExtractionContainers::PrepareData(const std::string &output_file_name, PrepareNodes(); WriteNodes(file_out_stream); - PrepareEdges(); + PrepareEdges(segment_state); WriteEdges(file_out_stream); file_out_stream.close(); @@ -168,7 +173,7 @@ void ExtractionContainers::PrepareNodes() std::cout << "ok, after " << TIMER_SEC(sorting_nodes) << "s" << std::endl; } -void ExtractionContainers::PrepareEdges() +void ExtractionContainers::PrepareEdges(lua_State *segment_state) { // Sort edges by start. std::cout << "[extractor] Sorting edges by start ... " << std::flush; @@ -264,6 +269,16 @@ void ExtractionContainers::PrepareEdges() edge_iterator->source_coordinate.lat, edge_iterator->source_coordinate.lon, node_iterator->lat, node_iterator->lon); + if (lua_function_exists(segment_state, "segment_function")) + { + luabind::call_function( + segment_state, "segment_function", + boost::cref(edge_iterator->source_coordinate), + boost::cref(*node_iterator), + distance, + boost::ref(edge_iterator->weight_data)); + } + const double weight = [distance](const InternalExtractorEdge::WeightData& data) { switch (data.type) { diff --git a/extractor/extraction_containers.hpp b/extractor/extraction_containers.hpp index ef71d059f1d..45285e03e51 100644 --- a/extractor/extraction_containers.hpp +++ b/extractor/extraction_containers.hpp @@ -30,6 +30,7 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "internal_extractor_edge.hpp" #include "first_and_last_segment_of_way.hpp" +#include "scripting_environment.hpp" #include "../data_structures/external_memory_node.hpp" #include "../data_structures/restriction.hpp" @@ -53,7 +54,7 @@ class ExtractionContainers #endif void PrepareNodes(); void PrepareRestrictions(); - void PrepareEdges(); + void PrepareEdges(lua_State *segment_state); void WriteNodes(std::ofstream& file_out_stream) const; void WriteRestrictions(const std::string& restrictions_file_name) const; @@ -81,7 +82,8 @@ class ExtractionContainers void PrepareData(const std::string &output_file_name, const std::string &restrictions_file_name, - const std::string &names_file_name); + const std::string &names_file_name, + lua_State *segment_state); }; #endif /* EXTRACTION_CONTAINERS_HPP */ diff --git a/extractor/extractor.cpp b/extractor/extractor.cpp index c1a0a212767..e988aa92bf7 100644 --- a/extractor/extractor.cpp +++ b/extractor/extractor.cpp @@ -34,10 +34,12 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "restriction_parser.hpp" #include "scripting_environment.hpp" +#include "../data_structures/raster_source.hpp" #include "../util/git_sha.hpp" #include "../util/make_unique.hpp" #include "../util/simple_logger.hpp" #include "../util/timing_util.hpp" +#include "../util/lua_util.hpp" #include "../typedefs.h" @@ -116,6 +118,17 @@ int extractor::run() SimpleLogger().Write() << "Parsing in progress.."; TIMER_START(parsing); + lua_State *segment_state = scripting_environment.get_lua_state(); + + if (lua_function_exists(segment_state, "source_function")) + { + // bind a single instance of SourceContainer class to relevant lua state + SourceContainer sources; + luabind::globals(segment_state)["sources"] = sources; + + luabind::call_function(segment_state, "source_function"); + } + std::string generator = header.get("generator"); if (generator.empty()) { @@ -238,7 +251,9 @@ int extractor::run() extraction_containers.PrepareData(config.output_file_name, config.restriction_file_name, - config.names_file_name); + config.names_file_name, + segment_state); + TIMER_STOP(extracting); SimpleLogger().Write() << "extraction finished after " << TIMER_SEC(extracting) << "s"; SimpleLogger().Write() << "To prepare the data for routing, run: " diff --git a/extractor/lat b/extractor/lat new file mode 100644 index 00000000000..e69de29bb2d diff --git a/extractor/scripting_environment.cpp b/extractor/scripting_environment.cpp index e98f688c6c7..de96b119379 100644 --- a/extractor/scripting_environment.cpp +++ b/extractor/scripting_environment.cpp @@ -30,13 +30,16 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "extraction_helper_functions.hpp" #include "extraction_node.hpp" #include "extraction_way.hpp" +#include "internal_extractor_edge.hpp" #include "../data_structures/external_memory_node.hpp" +#include "../data_structures/raster_source.hpp" #include "../util/lua_util.hpp" #include "../util/osrm_exception.hpp" #include "../util/simple_logger.hpp" #include "../typedefs.h" #include +#include #include @@ -80,6 +83,13 @@ void ScriptingEnvironment::init_lua_state(lua_State *lua_state) luabind::def("print", LUA_print), luabind::def("durationIsValid", durationIsValid), luabind::def("parseDuration", parseDuration), + luabind::class_("sources") + .def(luabind::constructor<>()) + .def("load", &SourceContainer::loadRasterSource) + .def("query", &SourceContainer::getRasterDataFromSource) + .def("interpolate", &SourceContainer::getRasterInterpolateFromSource), + luabind::class_("constants") + .enum_("enums")[luabind::value("precision", COORDINATE_PRECISION)], luabind::class_>("vector") .def("Add", static_cast::*)(const std::string &)>( @@ -121,7 +131,21 @@ void ScriptingEnvironment::init_lua_state(lua_State *lua_state) luabind::class_("Way") .def("get_value_by_key", &osmium::Way::get_value_by_key) .def("get_value_by_key", &get_value_by_key) - .def("id", &osmium::Way::id) + .def("id", &osmium::Way::id), + luabind::class_("EdgeSource") + .property("source_coordinate", &InternalExtractorEdge::source_coordinate) + .property("weight_data", &InternalExtractorEdge::weight_data), + luabind::class_("WeightData") + .def_readwrite("speed", &InternalExtractorEdge::WeightData::speed), + luabind::class_("EdgeTarget") + .property("lat", &ExternalMemoryNode::lat) + .property("lon", &ExternalMemoryNode::lon), + luabind::class_("Coordinate") + .property("lat", &FixedPointCoordinate::lat) + .property("lon", &FixedPointCoordinate::lon), + luabind::class_("RasterDatum") + .property("datum", &RasterDatum::datum) + .def("invalid_data", &RasterDatum::get_invalid) ]; if (0 != luaL_dofile(lua_state, file_name.c_str())) diff --git a/extractor/source_coordinate.lat b/extractor/source_coordinate.lat new file mode 100644 index 00000000000..e69de29bb2d diff --git a/features/raster/extract.feature b/features/raster/extract.feature new file mode 100644 index 00000000000..9ca0635d87e --- /dev/null +++ b/features/raster/extract.feature @@ -0,0 +1,18 @@ +@raster @extract +Feature: osrm-extract with a profile containing raster source +# expansions: +# {osm_base} => path to current input file +# {profile} => path to current profile script + + Scenario: osrm-extract on a valid profile + Given the profile "rasterbot" + And the node map + | a | b | + And the ways + | nodes | + | ab | + And the data has been saved to disk + When I run "osrm-extract {osm_base}.osm -p {profile}" + Then stderr should be empty + And stdout should contain "source loader" + And it should exit with code 0 diff --git a/features/raster/weights.feature b/features/raster/weights.feature new file mode 100644 index 00000000000..1d521d0763d --- /dev/null +++ b/features/raster/weights.feature @@ -0,0 +1,78 @@ +@routing @speed @raster +Feature: Raster - weights + + Background: Use specific speeds + Given the node locations + | node | lat | lon | + | a | 0.1 | 0.1 | + | b | .05 | 0.1 | + | c | 0.0 | 0.1 | + | d | .05 | .03 | + | e | .05 | .066 | + | f | .075 | .066 | + And the ways + | nodes | highway | + | ab | primary | + | ad | primary | + | bc | primary | + | dc | primary | + | de | primary | + | eb | primary | + | df | primary | + | fb | primary | + And the raster source + """ + 0 0 0 0 + 0 0 0 250 + 0 0 250 500 + 0 0 0 250 + 0 0 0 0 + """ + + Scenario: Weighting not based on raster sources + Given the profile "testbot" + When I run "osrm-extract {osm_base}.osm -p {profile}" + And I run "osrm-prepare {osm_base}.osm" + And I route I should get + | from | to | route | speed | + | a | b | ab | 36 km/h | + | a | c | ab,bc | 36 km/h | + | b | c | bc | 36 km/h | + | a | d | ad | 36 km/h | + | d | c | dc | 36 km/h | + + Scenario: Weighting based on raster sources + Given the profile "rasterbot" + When I run "osrm-extract {osm_base}.osm -p {profile}" + Then stdout should contain "evaluating segment" + And I run "osrm-prepare {osm_base}.osm" + And I route I should get + | from | to | route | speed | + | a | b | ab | 8 km/h | + | a | c | ad,dc | 15 km/h | + | b | c | bc | 8 km/h | + | a | d | ad | 15 km/h | + | d | c | dc | 15 km/h | + | d | e | de | 10 km/h | + | e | b | eb | 10 km/h | + | d | f | df | 15 km/h | + | f | b | fb | 7 km/h | + | d | b | de,eb | 10 km/h | + + Scenario: Weighting based on raster sources + Given the profile "rasterbot-interp" + When I run "osrm-extract {osm_base}.osm -p {profile}" + Then stdout should contain "evaluating segment" + And I run "osrm-prepare {osm_base}.osm" + And I route I should get + | from | to | route | speed | + | a | b | ab | 8 km/h | + | a | c | ad,dc | 15 km/h | + | b | c | bc | 8 km/h | + | a | d | ad | 15 km/h | + | d | c | dc | 15 km/h | + | d | e | de | 10 km/h | + | e | b | eb | 10 km/h | + | d | f | df | 15 km/h | + | f | b | fb | 7 km/h | + | d | b | de,eb | 10 km/h | diff --git a/features/step_definitions/data.rb b/features/step_definitions/data.rb index 524fd322cf5..ff074d24fa0 100644 --- a/features/step_definitions/data.rb +++ b/features/step_definitions/data.rb @@ -134,6 +134,12 @@ @osm_str = File.read file end +Given /^the raster source$/ do |data| + Dir.chdir TEST_FOLDER do + File.open("rastersource.asc", "w") {|f| f.write(data)} + end +end + Given /^the data has been saved to disk$/ do begin write_input_data diff --git a/features/support/env.rb b/features/support/env.rb index ea74959edd6..cb707f9e1d3 100644 --- a/features/support/env.rb +++ b/features/support/env.rb @@ -14,6 +14,7 @@ WAY_SPACING = 100 DEFAULT_GRID_SIZE = 100 #meters PROFILES_PATH = File.join ROOT_FOLDER, 'profiles' +FIXTURES_PATH = File.join ROOT_FOLDER, 'unit_tests/fixtures' BIN_PATH = File.join ROOT_FOLDER, 'build' DEFAULT_INPUT_FORMAT = 'osm' DEFAULT_ORIGIN = [1,1] @@ -71,7 +72,7 @@ def verify_osrm_is_not_running end def verify_existance_of_binaries - ["osrm-extract", "osrm-prepare", "osrm-routed"].each do |bin| + ["osrm-extract", "osrm-prepare", "osrm-routed"].each do |bin| unless File.exists? "#{BIN_PATH}/#{bin}#{EXE}" raise "*** #{BIN_PATH}/#{bin}#{EXE} is missing. Build failed?" end diff --git a/profiles/rasterbot-interp.lua b/profiles/rasterbot-interp.lua new file mode 100644 index 00000000000..42c98b6f401 --- /dev/null +++ b/profiles/rasterbot-interp.lua @@ -0,0 +1,46 @@ +-- Rasterbot profile + +-- Minimalist node_ and way_functions in order to test source_ and segment_functions + +function node_function (node, result) +end + +function way_function (way, result) + local highway = way:get_value_by_key("highway") + local name = way:get_value_by_key("name") + + if name then + result.name = name + end + + result.forward_speed = 15 + result.backward_speed = 15 +end + +function source_function () + raster_source = sources:load( + "../test/rastersource.asc", + 0, -- lon_min + 0.1, -- lon_max + 0, -- lat_min + 0.1, -- lat_max + 5, -- nrows + 4 -- ncols + ) +end + +function segment_function (source, target, distance, weight) + local sourceData = sources:interpolate(raster_source, source.lon, source.lat) + local targetData = sources:interpolate(raster_source, target.lon, target.lat) + print ("evaluating segment: " .. sourceData.datum .. " " .. targetData.datum) + local invalid = sourceData.invalid_data() + + if sourceData.datum ~= invalid and targetData.datum ~= invalid then + local slope = math.abs(sourceData.datum - targetData.datum) / distance + print (" slope: " .. slope) + print (" was speed: " .. weight.speed) + + weight.speed = weight.speed * (1 - (slope * 5)) + print (" new speed: " .. weight.speed) + end +end diff --git a/profiles/rasterbot.lua b/profiles/rasterbot.lua new file mode 100644 index 00000000000..847384ddf71 --- /dev/null +++ b/profiles/rasterbot.lua @@ -0,0 +1,46 @@ +-- Rasterbot profile + +-- Minimalist node_ and way_functions in order to test source_ and segment_functions + +function node_function (node, result) +end + +function way_function (way, result) + local highway = way:get_value_by_key("highway") + local name = way:get_value_by_key("name") + + if name then + result.name = name + end + + result.forward_speed = 15 + result.backward_speed = 15 +end + +function source_function () + raster_source = sources:load( + "../test/rastersource.asc", + 0, -- lon_min + 0.1, -- lon_max + 0, -- lat_min + 0.1, -- lat_max + 5, -- nrows + 4 -- ncols + ) +end + +function segment_function (source, target, distance, weight) + local sourceData = sources:query(raster_source, source.lon, source.lat) + local targetData = sources:query(raster_source, target.lon, target.lat) + print ("evaluating segment: " .. sourceData.datum .. " " .. targetData.datum) + local invalid = sourceData.invalid_data() + + if sourceData.datum ~= invalid and targetData.datum ~= invalid then + local slope = math.abs(sourceData.datum - targetData.datum) / distance + print (" slope: " .. slope) + print (" was speed: " .. weight.speed) + + weight.speed = weight.speed * (1 - (slope * 5)) + print (" new speed: " .. weight.speed) + end +end diff --git a/test/rastersource.asc b/test/rastersource.asc new file mode 100644 index 00000000000..b72c427ff10 --- /dev/null +++ b/test/rastersource.asc @@ -0,0 +1,5 @@ +0 0 0 0 +0 0 0 250 +0 0 250 500 +0 0 0 250 +0 0 0 0 \ No newline at end of file diff --git a/unit_tests/data_structures/raster_source.cpp b/unit_tests/data_structures/raster_source.cpp new file mode 100644 index 00000000000..691cba108e2 --- /dev/null +++ b/unit_tests/data_structures/raster_source.cpp @@ -0,0 +1,110 @@ +/* + +Copyright (c) 2014, Project OSRM contributors +All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + +Redistributions of source code must retain the above copyright notice, this list +of conditions and the following disclaimer. +Redistributions in binary form must reproduce the above copyright notice, this +list of conditions and the following disclaimer in the documentation and/or +other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +*/ + +#include "../../data_structures/raster_source.hpp" +#include "../../typedefs.h" +#include "../../util/osrm_exception.hpp" + +#include + +#include +#include + +BOOST_AUTO_TEST_SUITE(raster_source) + +int normalize(double coord) { return static_cast(coord * COORDINATE_PRECISION); } + +#define CHECK_QUERY(source_id, lon, lat, expected) \ + BOOST_CHECK_EQUAL( \ + sources.getRasterDataFromSource(source_id, normalize(lon), normalize(lat)).datum, \ + expected) + +#define CHECK_INTERPOLATE(source_id, lon, lat, expected) \ + BOOST_CHECK_EQUAL( \ + sources.getRasterInterpolateFromSource(source_id, normalize(lon), normalize(lat)).datum, \ + expected) + +BOOST_AUTO_TEST_CASE(raster_test) +{ + SourceContainer sources; + int source_id = sources.loadRasterSource("../unit_tests/fixtures/raster_data.asc", 0, 0.09, 0, + 0.09, 10, 10); + BOOST_CHECK_EQUAL(source_id, 0); + + // Expected nearest-neighbor queries + // EDGES + CHECK_QUERY(0, 0.00, 0.00, 10); + CHECK_QUERY(0, 0.00, 0.09, 10); + CHECK_QUERY(0, 0.09, 0.00, 40); + CHECK_QUERY(0, 0.09, 0.09, 100); + CHECK_QUERY(0, 0.09, 0.07, 140); + // OUT OF BOUNDS + CHECK_QUERY(0, -0.1, 0.07, RasterDatum::get_invalid()); + CHECK_QUERY(0, -0.1, -3.0, RasterDatum::get_invalid()); + CHECK_QUERY(0, 0.3, 23.0, RasterDatum::get_invalid()); + // ARBITRARY - AT DATA + CHECK_QUERY(0, 0.06, 0.06, 100); + CHECK_QUERY(0, 0.08, 0.05, 160); + CHECK_QUERY(0, 0.01, 0.05, 20); + // ARBITRARY - BETWEEN DATA + CHECK_QUERY(0, 0.054, 0.023, 40); + CHECK_QUERY(0, 0.056, 0.028, 80); + CHECK_QUERY(0, 0.05, 0.028, 60); + + // Expected bilinear interpolation queries + // EDGES - same as above + CHECK_INTERPOLATE(0, 0.00, 0.00, 10); + CHECK_INTERPOLATE(0, 0.00, 0.09, 10); + CHECK_INTERPOLATE(0, 0.09, 0.00, 40); + CHECK_INTERPOLATE(0, 0.09, 0.09, 100); + CHECK_INTERPOLATE(0, 0.09, 0.07, 140); + // OUT OF BOUNDS - same as above + CHECK_INTERPOLATE(0, -0.1, 0.07, RasterDatum::get_invalid()); + CHECK_INTERPOLATE(0, -0.1, -3.0, RasterDatum::get_invalid()); + CHECK_INTERPOLATE(0, 0.3, 23.0, RasterDatum::get_invalid()); + // ARBITRARY - AT DATA - same as above + CHECK_INTERPOLATE(0, 0.06, 0.06, 100); + CHECK_INTERPOLATE(0, 0.08, 0.05, 160); + CHECK_INTERPOLATE(0, 0.01, 0.05, 20); + // ARBITRARY - BETWEEN DATA + CHECK_INTERPOLATE(0, 0.054, 0.023, 54); + CHECK_INTERPOLATE(0, 0.056, 0.028, 68); + CHECK_INTERPOLATE(0, 0.05, 0.028, 56); + + int source_already_loaded_id = sources.loadRasterSource( + "../unit_tests/fixtures/raster_data.asc", 0, 0.09, 0, 0.09, 10, 10); + + BOOST_CHECK_EQUAL(source_already_loaded_id, 0); + BOOST_CHECK_THROW(sources.getRasterDataFromSource(1, normalize(0.02), normalize(0.02)), + osrm::exception); + + BOOST_CHECK_THROW( + sources.loadRasterSource("../unit_tests/fixtures/nonexistent.asc", 0, 0.1, 0, 0.1, 7, 7), + osrm::exception); +} + +BOOST_AUTO_TEST_SUITE_END() diff --git a/unit_tests/fixtures/raster_data.asc b/unit_tests/fixtures/raster_data.asc new file mode 100644 index 00000000000..4abd9bb3dff --- /dev/null +++ b/unit_tests/fixtures/raster_data.asc @@ -0,0 +1,10 @@ +10 10 10 10 10 20 40 60 80 100 +10 10 10 10 20 40 60 80 100 120 +10 10 10 20 40 60 80 100 120 140 +10 10 20 40 60 80 100 120 140 160 +10 20 40 60 80 100 120 140 160 180 +10 10 20 40 60 80 100 120 140 160 +10 10 10 20 40 60 80 100 120 140 +10 10 10 10 20 40 60 80 100 80 +10 10 10 10 10 20 40 60 80 60 +10 10 10 10 10 10 20 40 60 40 From b2d444d782ba9eb9b2719cde66902f4b9e5da5ba Mon Sep 17 00:00:00 2001 From: Daniel Patterson Date: Fri, 24 Jul 2015 18:45:33 -0700 Subject: [PATCH 105/122] Only replace fingerprint file when MD5 changes. Avoids rebuilding several things if nothing has actually changes, as cmake is only looking at timestamps. --- cmake/FingerPrint-Config.cmake | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/cmake/FingerPrint-Config.cmake b/cmake/FingerPrint-Config.cmake index a3325d30593..857dedd8949 100644 --- a/cmake/FingerPrint-Config.cmake +++ b/cmake/FingerPrint-Config.cmake @@ -1,10 +1,24 @@ set(OLDFILE ${SOURCE_DIR}/util/fingerprint_impl.hpp) -if (EXISTS ${OLDFILE}) - file(REMOVE_RECURSE ${OLDFILE}) -endif() +set(NEWFILE ${OLDFILE}.tmp) +set(INFILE ${OLDFILE}.in) file(MD5 ${SOURCE_DIR}/prepare.cpp MD5PREPARE) file(MD5 ${SOURCE_DIR}/data_structures/static_rtree.hpp MD5RTREE) file(MD5 ${SOURCE_DIR}/util/graph_loader.hpp MD5GRAPH) file(MD5 ${SOURCE_DIR}/server/data_structures/internal_datafacade.hpp MD5OBJECTS) -CONFIGURE_FILE(${SOURCE_DIR}/util/fingerprint_impl.hpp.in ${SOURCE_DIR}/util/fingerprint_impl.hpp) +CONFIGURE_FILE(${INFILE} ${NEWFILE}) + +file(MD5 ${NEWFILE} MD5NEW) + +if (EXISTS ${OLDFILE}) + file(MD5 ${OLDFILE} MD5OLD) + if(NOT ${MD5NEW} STREQUAL ${MD5OLD}) + file(REMOVE_RECURSE ${OLDFILE}) + file(RENAME ${NEWFILE} ${OLDFILE}) + else() + file(REMOVE_RECURSE ${NEWFILE}) + message(STATUS "Fingerprint unchanged, not regenerating") + endif() +else() + file(RENAME ${NEWFILE} ${OLDFILE}) +endif() From bcc41bf3d1a5f9e559d1da00f36f220e6d007dfe Mon Sep 17 00:00:00 2001 From: "Daniel J. Hofmann" Date: Tue, 2 Jun 2015 12:54:55 +0200 Subject: [PATCH 106/122] Fixes undefined behavior from shifting into signed bit; use unsigned literal instead --- data_structures/hilbert_value.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data_structures/hilbert_value.cpp b/data_structures/hilbert_value.cpp index c097731184b..d0d61e60a73 100644 --- a/data_structures/hilbert_value.cpp +++ b/data_structures/hilbert_value.cpp @@ -57,7 +57,7 @@ uint64_t HilbertCode::BitInterleaving(const uint32_t latitude, const uint32_t lo void HilbertCode::TransposeCoordinate(uint32_t *X) const { - uint32_t M = 1 << (32 - 1), P, Q, t; + uint32_t M = 1u << (32 - 1), P, Q, t; int i; // Inverse undo for (Q = M; Q > 1; Q >>= 1) From 345d5e8d9efea3477a290516a277b38e647a346b Mon Sep 17 00:00:00 2001 From: "Daniel J. Hofmann" Date: Mon, 7 Sep 2015 12:20:50 +0200 Subject: [PATCH 107/122] Make an exception for block barriers in bicycle and foot profile. This adds `barrier=block` exceptions to the respective white lists. In addition this adds tests to check for the exception in bicycle and foot profiles and makes sure cars are still not able to cross them. Checked with: cucumber --tags @barrier -p verify References: - https://github.com/Project-OSRM/osrm-backend/issues/1643 - http://wiki.openstreetmap.org/wiki/Tag:barrier%3Dblock --- features/bicycle/barrier.feature | 1 + features/car/barrier.feature | 1 + features/foot/barrier.feature | 1 + profiles/bicycle.lua | 2 +- profiles/foot.lua | 2 +- 5 files changed, 5 insertions(+), 2 deletions(-) diff --git a/features/bicycle/barrier.feature b/features/bicycle/barrier.feature index d39a570d7fe..15cd484334a 100644 --- a/features/bicycle/barrier.feature +++ b/features/bicycle/barrier.feature @@ -19,6 +19,7 @@ Feature: Barriers | wall | | | fence | | | some_tag | | + | block | x | Scenario: Bike - Access tag trumphs barriers Then routability should be diff --git a/features/car/barrier.feature b/features/car/barrier.feature index 3f5bef53e36..19cee210d02 100644 --- a/features/car/barrier.feature +++ b/features/car/barrier.feature @@ -19,6 +19,7 @@ Feature: Car - Barriers | wall | | | fence | | | some_tag | | + | block | | Scenario: Car - Access tag trumphs barriers Then routability should be diff --git a/features/foot/barrier.feature b/features/foot/barrier.feature index 18fa4deac93..1dda71928b8 100644 --- a/features/foot/barrier.feature +++ b/features/foot/barrier.feature @@ -19,6 +19,7 @@ Feature: Barriers | wall | | | fence | | | some_tag | | + | block | x | Scenario: Foot - Access tag trumphs barriers Then routability should be diff --git a/profiles/bicycle.lua b/profiles/bicycle.lua index ffc70b24fc3..fddd7d47bdc 100644 --- a/profiles/bicycle.lua +++ b/profiles/bicycle.lua @@ -4,7 +4,7 @@ local find_access_tag = require("lib/access").find_access_tag local limit = require("lib/maxspeed").limit -- Begin of globals -barrier_whitelist = { [""] = true, ["cycle_barrier"] = true, ["bollard"] = true, ["entrance"] = true, ["cattle_grid"] = true, ["border_control"] = true, ["toll_booth"] = true, ["sally_port"] = true, ["gate"] = true, ["no"] = true } +barrier_whitelist = { [""] = true, ["cycle_barrier"] = true, ["bollard"] = true, ["entrance"] = true, ["cattle_grid"] = true, ["border_control"] = true, ["toll_booth"] = true, ["sally_port"] = true, ["gate"] = true, ["no"] = true, ["block"] = true } access_tag_whitelist = { ["yes"] = true, ["permissive"] = true, ["designated"] = true } access_tag_blacklist = { ["no"] = true, ["private"] = true, ["agricultural"] = true, ["forestry"] = true } access_tag_restricted = { ["destination"] = true, ["delivery"] = true } diff --git a/profiles/foot.lua b/profiles/foot.lua index 3d676ba3bd4..172fe2abdd5 100644 --- a/profiles/foot.lua +++ b/profiles/foot.lua @@ -3,7 +3,7 @@ local find_access_tag = require("lib/access").find_access_tag -- Begin of globals -barrier_whitelist = { [""] = true, ["cycle_barrier"] = true, ["bollard"] = true, ["entrance"] = true, ["cattle_grid"] = true, ["border_control"] = true, ["toll_booth"] = true, ["sally_port"] = true, ["gate"] = true, ["no"] = true} +barrier_whitelist = { [""] = true, ["cycle_barrier"] = true, ["bollard"] = true, ["entrance"] = true, ["cattle_grid"] = true, ["border_control"] = true, ["toll_booth"] = true, ["sally_port"] = true, ["gate"] = true, ["no"] = true, ["block"] = true} access_tag_whitelist = { ["yes"] = true, ["foot"] = true, ["permissive"] = true, ["designated"] = true } access_tag_blacklist = { ["no"] = true, ["private"] = true, ["agricultural"] = true, ["forestry"] = true } access_tag_restricted = { ["destination"] = true, ["delivery"] = true } From d2080808db1bbddfd2ad2ea271b24d28cfcb4a73 Mon Sep 17 00:00:00 2001 From: bergwerkgis Date: Mon, 7 Sep 2015 14:26:47 +0000 Subject: [PATCH 108/122] AppVeyor: include osrm.lib in artifact. don't stick to AppVeyor directory structure in build scripts. --- appveyor-build.bat | 41 ++++++++++++++++++++++++----------------- appveyor.yml | 4 ---- build-local.bat | 28 +++++++++++++++++++--------- 3 files changed, 43 insertions(+), 30 deletions(-) diff --git a/appveyor-build.bat b/appveyor-build.bat index cffa5261ab4..dae949c7c8e 100644 --- a/appveyor-build.bat +++ b/appveyor-build.bat @@ -2,6 +2,12 @@ SETLOCAL SET EL=0 +ECHO ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ %~f0 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +ECHO activating VS command prompt ... +SET PATH=C:\Program Files (x86)\MSBuild\14.0\Bin;%PATH% +CALL "C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\vcvarsall.bat" amd64 + ECHO platform^: %platform% :: HARDCODE "x64" as it is uppercase on AppVeyor and download from S3 is case sensitive SET DEPSPKG=osrm-deps-win-x64-14.0.7z @@ -12,8 +18,10 @@ IF "%computername%"=="WIN-2G2NPH4S5B8" GOTO SKIPDL IF EXIST %DEPSPKG% DEL %DEPSPKG% IF %ERRORLEVEL% NEQ 0 GOTO ERROR +SET PROJECT_DIR=%CD% +ECHO PROJECT_DIR^: %PROJECT_DIR% ECHO downloading %DEPSPKG% -powershell Invoke-WebRequest https://mapbox.s3.amazonaws.com/windows-builds/windows-build-deps/$env:DEPSPKG -OutFile C:\projects\osrm\$env:DEPSPKG +powershell Invoke-WebRequest https://mapbox.s3.amazonaws.com/windows-builds/windows-build-deps/$env:DEPSPKG -OutFile $env:PROJECT_DIR\$env:DEPSPKG IF %ERRORLEVEL% NEQ 0 GOTO ERROR 7z -y x %DEPSPKG% | %windir%\system32\FIND "ing archive" @@ -28,7 +36,7 @@ IF %ERRORLEVEL% NEQ 0 GOTO ERROR cd build IF %ERRORLEVEL% NEQ 0 GOTO ERROR -SET OSRMDEPSDIR=c:\projects\osrm\osrm-deps +SET OSRMDEPSDIR=%PROJECT_DIR%\osrm-deps set PREFIX=%OSRMDEPSDIR%/libs set BOOST_ROOT=%OSRMDEPSDIR%/boost set TBB_INSTALL_DIR=%OSRMDEPSDIR%/tbb @@ -38,7 +46,7 @@ ECHO calling cmake .... cmake .. ^ -G "Visual Studio 14 Win64" ^ -DBOOST_ROOT=%BOOST_ROOT% ^ --DBoost_ADDITIONAL_VERSIONS=1.57 ^ +-DBoost_ADDITIONAL_VERSIONS=1.58 ^ -DBoost_USE_MULTITHREADED=ON ^ -DBoost_USE_STATIC_LIBS=ON ^ -DCMAKE_BUILD_TYPE=%CONFIGURATION% ^ @@ -60,45 +68,44 @@ msbuild OSRM.sln ^ /flp2:logfile=build_warnings.txt;warningsonly IF %ERRORLEVEL% NEQ 0 GOTO ERROR -CD c:\projects\osrm\build\%Configuration% +CD %PROJECT_DIR%\build\%Configuration% IF %ERRORLEVEL% NEQ 0 GOTO ERROR -SET PATH=c:\projects\osrm\osrm-deps\libs\bin;%PATH% +SET PATH=%PROJECT_DIR%\osrm-deps\libs\bin;%PATH% -CD .. ECHO running datastructure-tests.exe ... -%Configuration%\datastructure-tests.exe -IF %ERRORLEVEL% NEQ 0 GOTO ERROR +datastructure-tests.exe +::IF %ERRORLEVEL% NEQ 0 GOTO ERROR ECHO running algorithm-tests.exe ... -%Configuration%\algorithm-tests.exe -IF %ERRORLEVEL% NEQ 0 GOTO ERROR -cd %Configuration% +algorithm-tests.exe +::IF %ERRORLEVEL% NEQ 0 GOTO ERROR IF NOT "%APPVEYOR_REPO_BRANCH%"=="develop" GOTO DONE ECHO ========= CREATING PACKAGES ========== -SET P=c:\projects\osrm +SET P=%PROJECT_DIR% SET ZIP= %P%\osrm_%Configuration%.zip IF EXIST %ZIP% ECHO deleting %ZIP% && DEL /F /Q %ZIP% IF %ERRORLEVEL% NEQ 0 ECHO deleting %ZIP% FAILED && GOTO ERROR -7z a %ZIP% *.exe *.pdb %P%/osrm-deps/libs/bin/*.dll -tzip +7z a %ZIP% *.lib *.exe *.pdb %P%/osrm-deps/libs/bin/*.dll -tzip -mx9 | %windir%\system32\FIND "ing archive" IF %ERRORLEVEL% NEQ 0 GOTO ERROR CD ..\..\profiles IF %ERRORLEVEL% NEQ 0 GOTO ERROR ECHO disk=c:\temp\stxxl,10000,wincall > .stxxl.txt -7z a %ZIP% * -tzip +7z a %ZIP% * -tzip -mx9 | %windir%\system32\FIND "ing archive" IF %ERRORLEVEL% NEQ 0 GOTO ERROR GOTO DONE :ERROR +ECHO ~~~~~~~~~~~~~~~~~~~~~~ ERROR %~f0 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +ECHO ERRORLEVEL^: %ERRORLEVEL% SET EL=%ERRORLEVEL% -ECHO ============== ERROR =============== :DONE -ECHO ============== DONE ================ -CD C:\projects\osrm +ECHO ~~~~~~~~~~~~~~~~~~~~~~ DONE %~f0 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + EXIT /b %EL% diff --git a/appveyor.yml b/appveyor.yml index 2fefebd75c8..b8fb9799f63 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -14,10 +14,6 @@ clone_folder: c:\projects\osrm platform: x64 -install: - - set PATH=C:\Program Files (x86)\MSBuild\14.0\Bin;%PATH% - - CALL "C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\vcvarsall.bat" amd64 - build_script: - CALL appveyor-build.bat diff --git a/build-local.bat b/build-local.bat index df18fc4447f..43b401e6ae9 100644 --- a/build-local.bat +++ b/build-local.bat @@ -1,22 +1,32 @@ @ECHO OFF +SETLOCAL +SET EL=0 + +ECHO ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ %~f0 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ SET PLATFORM=x64 SET CONFIGURATION=Release -WHERE msbuild -IF %ERRORLEVEL% EQU 0 GOTO RUNBUILD - FOR /F "tokens=*" %%i in ('git rev-parse --abbrev-ref HEAD') do SET APPVEYOR_REPO_BRANCH=%%i ECHO APPVEYOR_REPO_BRANCH^: %APPVEYOR_REPO_BRANCH% SET PATH=C:\mb\windows-builds-64\tmp-bin\cmake-3.1.0-win32-x86\bin;%PATH% SET PATH=C:\Program Files\7-Zip;%PATH% -ECHO activating VS command prompt ... -SET PATH=C:\Program Files (x86)\MSBuild\14.0\Bin;%PATH% -CALL "C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\vcvarsall.bat" amd64 - -:RUNBUILD powershell Set-ExecutionPolicy -Scope CurrentUser -ExecutionPolicy Unrestricted -Force +IF %ERRORLEVEL% NEQ 0 GOTO ERROR CALL appveyor-build.bat -EXIT /b %ERRORLEVEL% +IF %ERRORLEVEL% NEQ 0 GOTO ERROR + +GOTO DONE + + +:ERROR +ECHO ~~~~~~~~~~~~~~~~~~~~~~ ERROR %~f0 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +ECHO ERRORLEVEL^: %ERRORLEVEL% +SET EL=%ERRORLEVEL% + +:DONE +ECHO ~~~~~~~~~~~~~~~~~~~~~~ DONE %~f0 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +EXIT /b %EL% From b734d4bbc169df7a33c2fdccaf6f0de34420118d Mon Sep 17 00:00:00 2001 From: bergwerkgis Date: Mon, 7 Sep 2015 14:43:24 +0000 Subject: [PATCH 109/122] [skip ci] AppVeyor: fail again, if tests fail --- appveyor-build.bat | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/appveyor-build.bat b/appveyor-build.bat index dae949c7c8e..3f686b3d9a9 100644 --- a/appveyor-build.bat +++ b/appveyor-build.bat @@ -75,10 +75,10 @@ SET PATH=%PROJECT_DIR%\osrm-deps\libs\bin;%PATH% ECHO running datastructure-tests.exe ... datastructure-tests.exe -::IF %ERRORLEVEL% NEQ 0 GOTO ERROR +IF %ERRORLEVEL% NEQ 0 GOTO ERROR ECHO running algorithm-tests.exe ... algorithm-tests.exe -::IF %ERRORLEVEL% NEQ 0 GOTO ERROR +IF %ERRORLEVEL% NEQ 0 GOTO ERROR IF NOT "%APPVEYOR_REPO_BRANCH%"=="develop" GOTO DONE ECHO ========= CREATING PACKAGES ========== From bed0598530540c1b2459bb271727511ebedaa80e Mon Sep 17 00:00:00 2001 From: bergwerkgis Date: Mon, 7 Sep 2015 19:45:54 +0200 Subject: [PATCH 110/122] AppVeyor: make tests pass again --- appveyor-build.bat | 31 ++++++++++++++++++++----------- build-local.bat | 1 + 2 files changed, 21 insertions(+), 11 deletions(-) diff --git a/appveyor-build.bat b/appveyor-build.bat index 3f686b3d9a9..babefe168ea 100644 --- a/appveyor-build.bat +++ b/appveyor-build.bat @@ -4,6 +4,9 @@ SET EL=0 ECHO ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ %~f0 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +SET PROJECT_DIR=%CD% +ECHO PROJECT_DIR^: %PROJECT_DIR% + ECHO activating VS command prompt ... SET PATH=C:\Program Files (x86)\MSBuild\14.0\Bin;%PATH% CALL "C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\vcvarsall.bat" amd64 @@ -13,25 +16,28 @@ ECHO platform^: %platform% SET DEPSPKG=osrm-deps-win-x64-14.0.7z :: local development -IF "%computername%"=="WIN-2G2NPH4S5B8" GOTO SKIPDL +ECHO LOCAL_DEV^: %LOCAL_DEV% +IF NOT DEFINED LOCAL_DEV SET LOCAL_DEV=0 +IF DEFINED LOCAL_DEV IF %LOCAL_DEV% EQU 1 IF EXIST %DEPSPKG% ECHO skipping deps download && GOTO SKIPDL IF EXIST %DEPSPKG% DEL %DEPSPKG% IF %ERRORLEVEL% NEQ 0 GOTO ERROR -SET PROJECT_DIR=%CD% -ECHO PROJECT_DIR^: %PROJECT_DIR% ECHO downloading %DEPSPKG% powershell Invoke-WebRequest https://mapbox.s3.amazonaws.com/windows-builds/windows-build-deps/$env:DEPSPKG -OutFile $env:PROJECT_DIR\$env:DEPSPKG IF %ERRORLEVEL% NEQ 0 GOTO ERROR -7z -y x %DEPSPKG% | %windir%\system32\FIND "ing archive" -IF %ERRORLEVEL% NEQ 0 GOTO ERROR - :SKIPDL -IF EXIST build rd /s /q build +IF EXIST osrm-deps ECHO deleting osrm-deps... && RD /S /Q osrm-deps IF %ERRORLEVEL% NEQ 0 GOTO ERROR -mkdir build +IF EXIST build ECHO deletings build dir... && RD /S /Q build +IF %ERRORLEVEL% NEQ 0 GOTO ERROR + +7z -y x %DEPSPKG% | %windir%\system32\FIND "ing archive" +IF %ERRORLEVEL% NEQ 0 GOTO ERROR + +MKDIR build IF %ERRORLEVEL% NEQ 0 GOTO ERROR cd build IF %ERRORLEVEL% NEQ 0 GOTO ERROR @@ -68,21 +74,24 @@ msbuild OSRM.sln ^ /flp2:logfile=build_warnings.txt;warningsonly IF %ERRORLEVEL% NEQ 0 GOTO ERROR -CD %PROJECT_DIR%\build\%Configuration% +CD %PROJECT_DIR%\build IF %ERRORLEVEL% NEQ 0 GOTO ERROR SET PATH=%PROJECT_DIR%\osrm-deps\libs\bin;%PATH% ECHO running datastructure-tests.exe ... -datastructure-tests.exe +%Configuration%\datastructure-tests.exe IF %ERRORLEVEL% NEQ 0 GOTO ERROR ECHO running algorithm-tests.exe ... -algorithm-tests.exe +%Configuration%\algorithm-tests.exe IF %ERRORLEVEL% NEQ 0 GOTO ERROR IF NOT "%APPVEYOR_REPO_BRANCH%"=="develop" GOTO DONE ECHO ========= CREATING PACKAGES ========== +CD %PROJECT_DIR%\build\%Configuration% +IF %ERRORLEVEL% NEQ 0 GOTO ERROR + SET P=%PROJECT_DIR% SET ZIP= %P%\osrm_%Configuration%.zip IF EXIST %ZIP% ECHO deleting %ZIP% && DEL /F /Q %ZIP% diff --git a/build-local.bat b/build-local.bat index 43b401e6ae9..e1ae94a3beb 100644 --- a/build-local.bat +++ b/build-local.bat @@ -6,6 +6,7 @@ ECHO ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ %~f0 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ SET PLATFORM=x64 SET CONFIGURATION=Release +::SET LOCAL_DEV=1 FOR /F "tokens=*" %%i in ('git rev-parse --abbrev-ref HEAD') do SET APPVEYOR_REPO_BRANCH=%%i ECHO APPVEYOR_REPO_BRANCH^: %APPVEYOR_REPO_BRANCH% From db092c828e09e55570e5f1ac15dd63a8735f2012 Mon Sep 17 00:00:00 2001 From: "Daniel J. Hofmann" Date: Tue, 8 Sep 2015 23:34:20 +0200 Subject: [PATCH 111/122] Don't pass by const-value for a read-only view. I can't see a reason we pass by const-value here. Note: changes API because of the `route_parameters` header. --- data_structures/route_parameters.cpp | 2 +- data_structures/static_rtree.hpp | 4 ++-- include/osrm/route_parameters.hpp | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/data_structures/route_parameters.cpp b/data_structures/route_parameters.cpp index cac9de30dc0..df23ce25aa3 100644 --- a/data_structures/route_parameters.cpp +++ b/data_structures/route_parameters.cpp @@ -134,7 +134,7 @@ void RouteParameters::addCoordinate( static_cast(COORDINATE_PRECISION * boost::fusion::at_c<1>(received_coordinates))); } -void RouteParameters::getCoordinatesFromGeometry(const std::string geometry_string) +void RouteParameters::getCoordinatesFromGeometry(const std::string &geometry_string) { PolylineCompressor pc; coordinates = pc.decode_string(geometry_string); diff --git a/data_structures/static_rtree.hpp b/data_structures/static_rtree.hpp index bd88226ab8b..07cd490c02d 100644 --- a/data_structures/static_rtree.hpp +++ b/data_structures/static_rtree.hpp @@ -345,8 +345,8 @@ class StaticRTree // Construct a packed Hilbert-R-Tree with Kamel-Faloutsos algorithm [1] explicit StaticRTree(const std::vector &input_data_vector, - const std::string tree_node_filename, - const std::string leaf_node_filename, + const std::string &tree_node_filename, + const std::string &leaf_node_filename, const std::vector &coordinate_list) : m_element_count(input_data_vector.size()), m_leaf_node_filename(leaf_node_filename) { diff --git a/include/osrm/route_parameters.hpp b/include/osrm/route_parameters.hpp index 5e16b919f28..6395ad685b9 100644 --- a/include/osrm/route_parameters.hpp +++ b/include/osrm/route_parameters.hpp @@ -79,7 +79,7 @@ struct RouteParameters void addCoordinate(const boost::fusion::vector &received_coordinates); - void getCoordinatesFromGeometry(const std::string geometry_string); + void getCoordinatesFromGeometry(const std::string &geometry_string); short zoom_level; bool print_instructions; From f10fb77a81dd923eaad19927cabfa179e52cd287 Mon Sep 17 00:00:00 2001 From: "Daniel J. Hofmann" Date: Wed, 9 Sep 2015 18:34:09 +0200 Subject: [PATCH 112/122] Ownership: vector already owns, no need for wrapping in unique_ptr. Removes the pointless `std::unique_ptr>` usage, as a `std::vector` already owns its resources and manages them. Results in one indirection less (hint: good). --- contractor/processing_chain.cpp | 166 ++++++++++++++++---------------- contractor/processing_chain.hpp | 30 +++--- 2 files changed, 101 insertions(+), 95 deletions(-) diff --git a/contractor/processing_chain.cpp b/contractor/processing_chain.cpp index 376c8ac89ac..0c882424522 100644 --- a/contractor/processing_chain.cpp +++ b/contractor/processing_chain.cpp @@ -40,7 +40,6 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "../util/graph_loader.hpp" #include "../util/integer_range.hpp" #include "../util/lua_util.hpp" -#include "../util/make_unique.hpp" #include "../util/osrm_exception.hpp" #include "../util/simple_logger.hpp" #include "../util/string_util.hpp" @@ -79,12 +78,11 @@ int Prepare::Run() TIMER_START(expansion); - auto node_based_edge_list = osrm::make_unique>();; + std::vector node_based_edge_list; DeallocatingVector edge_based_edge_list; - auto internal_to_external_node_map = osrm::make_unique>(); - auto graph_size = - BuildEdgeExpandedGraph(*internal_to_external_node_map, - *node_based_edge_list, edge_based_edge_list); + std::vector internal_to_external_node_map; + auto graph_size = BuildEdgeExpandedGraph(internal_to_external_node_map, node_based_edge_list, + edge_based_edge_list); auto number_of_node_based_nodes = graph_size.first; auto max_edge_id = graph_size.second; @@ -94,39 +92,37 @@ int Prepare::Run() SimpleLogger().Write() << "building r-tree ..."; TIMER_START(rtree); - FindComponents(max_edge_id, edge_based_edge_list, *node_based_edge_list); + FindComponents(max_edge_id, edge_based_edge_list, node_based_edge_list); - BuildRTree(*node_based_edge_list, *internal_to_external_node_map); + BuildRTree(node_based_edge_list, internal_to_external_node_map); TIMER_STOP(rtree); SimpleLogger().Write() << "writing node map ..."; - WriteNodeMapping(std::move(internal_to_external_node_map)); + WriteNodeMapping(internal_to_external_node_map); // Contracting the edge-expanded graph TIMER_START(contraction); std::vector is_core_node; - auto contracted_edge_list = osrm::make_unique>(); - ContractGraph(max_edge_id, edge_based_edge_list, *contracted_edge_list, is_core_node); + DeallocatingVector contracted_edge_list; + ContractGraph(max_edge_id, edge_based_edge_list, contracted_edge_list, is_core_node); TIMER_STOP(contraction); SimpleLogger().Write() << "Contraction took " << TIMER_SEC(contraction) << " sec"; - std::size_t number_of_used_edges = WriteContractedGraph(max_edge_id, - std::move(node_based_edge_list), - std::move(contracted_edge_list)); + std::size_t number_of_used_edges = + WriteContractedGraph(max_edge_id, node_based_edge_list, contracted_edge_list); WriteCoreNodeMarker(std::move(is_core_node)); TIMER_STOP(preparing); SimpleLogger().Write() << "Preprocessing : " << TIMER_SEC(preparing) << " seconds"; SimpleLogger().Write() << "Expansion : " << (number_of_node_based_nodes / TIMER_SEC(expansion)) - << " nodes/sec and " - << ((max_edge_id+1) / TIMER_SEC(expansion)) << " edges/sec"; + << " nodes/sec and " << ((max_edge_id + 1) / TIMER_SEC(expansion)) + << " edges/sec"; - SimpleLogger().Write() << "Contraction: " - << ((max_edge_id+1) / TIMER_SEC(contraction)) + SimpleLogger().Write() << "Contraction: " << ((max_edge_id + 1) / TIMER_SEC(contraction)) << " nodes/sec and " << number_of_used_edges / TIMER_SEC(contraction) << " edges/sec"; @@ -135,30 +131,34 @@ int Prepare::Run() return 0; } -void Prepare::FindComponents(unsigned max_edge_id, const DeallocatingVector& input_edge_list, - std::vector& input_nodes) const +void Prepare::FindComponents(unsigned max_edge_id, + const DeallocatingVector &input_edge_list, + std::vector &input_nodes) const { - struct UncontractedEdgeData { }; - struct InputEdge { + struct UncontractedEdgeData + { + }; + struct InputEdge + { unsigned source; unsigned target; UncontractedEdgeData data; - bool operator<(const InputEdge& rhs) const + bool operator<(const InputEdge &rhs) const { return source < rhs.source || (source == rhs.source && target < rhs.target); } - bool operator==(const InputEdge& rhs) const + bool operator==(const InputEdge &rhs) const { - return source == rhs.source && target == rhs.target; + return source == rhs.source && target == rhs.target; } }; using UncontractedGraph = StaticGraph; std::vector edges; edges.reserve(input_edge_list.size() * 2); - for (const auto& edge : input_edge_list) + for (const auto &edge : input_edge_list) { BOOST_ASSERT_MSG(static_cast(std::max(edge.weight, 1)) > 0, "edge distance < 1"); @@ -174,7 +174,7 @@ void Prepare::FindComponents(unsigned max_edge_id, const DeallocatingVector(max_edge_id+1, edges); + auto uncontractor_graph = std::make_shared(max_edge_id + 1, edges); - TarjanSCC component_search(std::const_pointer_cast(uncontractor_graph)); + TarjanSCC component_search( + std::const_pointer_cast(uncontractor_graph)); component_search.run(); - for (auto& node : input_nodes) + for (auto &node : input_nodes) { auto forward_component = component_search.get_component_id(node.forward_edge_based_node_id); BOOST_ASSERT(node.reverse_edge_based_node_id == SPECIAL_EDGEID || - forward_component == component_search.get_component_id(node.reverse_edge_based_node_id)); + forward_component == + component_search.get_component_id(node.reverse_edge_based_node_id)); const unsigned component_size = component_search.get_component_size(forward_component); const bool is_tiny_component = component_size < 1000; @@ -204,7 +206,7 @@ void Prepare::FindComponents(unsigned max_edge_id, const DeallocatingVector&& in_is_core_node) const +void Prepare::WriteCoreNodeMarker(std::vector &&in_is_core_node) const { std::vector is_core_node(in_is_core_node); std::vector unpacked_bool_flags(is_core_node.size()); @@ -213,21 +215,23 @@ void Prepare::WriteCoreNodeMarker(std::vector&& in_is_core_node) const unpacked_bool_flags[i] = is_core_node[i] ? 1 : 0; } - boost::filesystem::ofstream core_marker_output_stream(config.core_output_path, std::ios::binary); + boost::filesystem::ofstream core_marker_output_stream(config.core_output_path, + std::ios::binary); unsigned size = unpacked_bool_flags.size(); core_marker_output_stream.write((char *)&size, sizeof(unsigned)); - core_marker_output_stream.write((char *)unpacked_bool_flags.data(), sizeof(char)*unpacked_bool_flags.size()); + core_marker_output_stream.write((char *)unpacked_bool_flags.data(), + sizeof(char) * unpacked_bool_flags.size()); } std::size_t Prepare::WriteContractedGraph(unsigned max_node_id, - std::unique_ptr> node_based_edge_list, - std::unique_ptr> contracted_edge_list) + const std::vector &node_based_edge_list, + const DeallocatingVector &contracted_edge_list) { - const unsigned crc32_value = CalculateEdgeChecksum(std::move(node_based_edge_list)); + const unsigned crc32_value = CalculateEdgeChecksum(node_based_edge_list); // Sorting contracted edges in a way that the static query graph can read some in in-place. - tbb::parallel_sort(contracted_edge_list->begin(), contracted_edge_list->end()); - const unsigned contracted_edge_count = contracted_edge_list->size(); + tbb::parallel_sort(contracted_edge_list); + const unsigned contracted_edge_count = contracted_edge_list.size(); SimpleLogger().Write() << "Serializing compacted graph of " << contracted_edge_count << " edges"; @@ -237,7 +241,7 @@ std::size_t Prepare::WriteContractedGraph(unsigned max_node_id, const unsigned max_used_node_id = [&contracted_edge_list] { unsigned tmp_max = 0; - for (const QueryEdge &edge : *contracted_edge_list) + for (const QueryEdge &edge : contracted_edge_list) { BOOST_ASSERT(SPECIAL_NODEID != edge.source); BOOST_ASSERT(SPECIAL_NODEID != edge.target); @@ -247,8 +251,8 @@ std::size_t Prepare::WriteContractedGraph(unsigned max_node_id, return tmp_max; }(); - SimpleLogger().Write(logDEBUG) << "input graph has " << (max_node_id+1) << " nodes"; - SimpleLogger().Write(logDEBUG) << "contracted graph has " << (max_used_node_id+1) << " nodes"; + SimpleLogger().Write(logDEBUG) << "input graph has " << (max_node_id + 1) << " nodes"; + SimpleLogger().Write(logDEBUG) << "contracted graph has " << (max_used_node_id + 1) << " nodes"; std::vector::NodeArrayEntry> node_array; // make sure we have at least one sentinel @@ -260,10 +264,10 @@ std::size_t Prepare::WriteContractedGraph(unsigned max_node_id, StaticGraph::EdgeIterator last_edge; // initializing 'first_edge'-field of nodes: - for (const auto node : osrm::irange(0u, max_used_node_id+1)) + for (const auto node : osrm::irange(0u, max_used_node_id + 1)) { last_edge = edge; - while ((edge < contracted_edge_count) && ((*contracted_edge_list)[edge].source == node)) + while ((edge < contracted_edge_count) && (contracted_edge_list[edge].source == node)) { ++edge; } @@ -271,7 +275,8 @@ std::size_t Prepare::WriteContractedGraph(unsigned max_node_id, position += edge - last_edge; // remove } - for (const auto sentinel_counter : osrm::irange(max_used_node_id+1, node_array.size())) + for (const auto sentinel_counter : + osrm::irange(max_used_node_id + 1, node_array.size())) { // sentinel element, guarded against underflow node_array[sentinel_counter].first_edge = contracted_edge_count; @@ -298,12 +303,12 @@ std::size_t Prepare::WriteContractedGraph(unsigned max_node_id, int number_of_used_edges = 0; StaticGraph::EdgeArrayEntry current_edge; - for (const auto edge : osrm::irange(0, contracted_edge_list->size())) + for (const auto edge : osrm::irange(0, contracted_edge_list.size())) { // no eigen loops - BOOST_ASSERT((*contracted_edge_list)[edge].source != (*contracted_edge_list)[edge].target); - current_edge.target = (*contracted_edge_list)[edge].target; - current_edge.data = (*contracted_edge_list)[edge].data; + BOOST_ASSERT(contracted_edge_list[edge].source != contracted_edge_list[edge].target); + current_edge.target = contracted_edge_list[edge].target; + current_edge.data = contracted_edge_list[edge].data; // every target needs to be valid BOOST_ASSERT(current_edge.target <= max_used_node_id); @@ -311,12 +316,12 @@ std::size_t Prepare::WriteContractedGraph(unsigned max_node_id, if (current_edge.data.distance <= 0) { SimpleLogger().Write(logWARNING) << "Edge: " << edge - << ",source: " << (*contracted_edge_list)[edge].source - << ", target: " << (*contracted_edge_list)[edge].target + << ",source: " << contracted_edge_list[edge].source + << ", target: " << contracted_edge_list[edge].target << ", dist: " << current_edge.data.distance; SimpleLogger().Write(logWARNING) << "Failed at adjacency list of node " - << (*contracted_edge_list)[edge].source << "/" + << contracted_edge_list[edge].source << "/" << node_array.size() - 1; return 1; } @@ -330,7 +335,7 @@ std::size_t Prepare::WriteContractedGraph(unsigned max_node_id, return number_of_used_edges; } -unsigned Prepare::CalculateEdgeChecksum(std::unique_ptr> node_based_edge_list) +unsigned Prepare::CalculateEdgeChecksum(const std::vector &node_based_edge_list) { RangebasedCRC32 crc32; if (crc32.using_hardware()) @@ -342,7 +347,7 @@ unsigned Prepare::CalculateEdgeChecksum(std::unique_ptr Prepare::LoadRestrictionMap() { - boost::filesystem::ifstream input_stream(config.restrictions_path, std::ios::in | std::ios::binary); + boost::filesystem::ifstream input_stream(config.restrictions_path, + std::ios::in | std::ios::binary); std::vector restriction_list; loadRestrictionsFromFile(input_stream, restriction_list); @@ -411,17 +416,17 @@ std::shared_ptr Prepare::LoadRestrictionMap() std::shared_ptr Prepare::LoadNodeBasedGraph(std::unordered_set &barrier_nodes, std::unordered_set &traffic_lights, - std::vector& internal_to_external_node_map) + std::vector &internal_to_external_node_map) { std::vector edge_list; - boost::filesystem::ifstream input_stream(config.osrm_input_path, std::ios::in | std::ios::binary); + boost::filesystem::ifstream input_stream(config.osrm_input_path, + std::ios::in | std::ios::binary); std::vector barrier_list; std::vector traffic_light_list; - NodeID number_of_node_based_nodes = loadNodesFromFile(input_stream, - barrier_list, traffic_light_list, - internal_to_external_node_map); + NodeID number_of_node_based_nodes = loadNodesFromFile( + input_stream, barrier_list, traffic_light_list, internal_to_external_node_map); SimpleLogger().Write() << " - " << barrier_list.size() << " bollard nodes, " << traffic_light_list.size() << " traffic lights"; @@ -446,7 +451,6 @@ Prepare::LoadNodeBasedGraph(std::unordered_set &barrier_nodes, return NodeBasedDynamicGraphFromEdges(number_of_node_based_nodes, edge_list); } - /** \brief Building an edge-expanded graph from node-based input and turn restrictions */ @@ -465,20 +469,18 @@ Prepare::BuildEdgeExpandedGraph(std::vector &internal_to_external_nod std::unordered_set traffic_lights; auto restriction_map = LoadRestrictionMap(); - auto node_based_graph = LoadNodeBasedGraph(barrier_nodes, traffic_lights, internal_to_external_node_map); - + auto node_based_graph = + LoadNodeBasedGraph(barrier_nodes, traffic_lights, internal_to_external_node_map); CompressedEdgeContainer compressed_edge_container; GraphCompressor graph_compressor(speed_profile); - graph_compressor.Compress(barrier_nodes, traffic_lights, *restriction_map, *node_based_graph, compressed_edge_container); + graph_compressor.Compress(barrier_nodes, traffic_lights, *restriction_map, *node_based_graph, + compressed_edge_container); - EdgeBasedGraphFactory edge_based_graph_factory(node_based_graph, - compressed_edge_container, - barrier_nodes, - traffic_lights, - std::const_pointer_cast(restriction_map), - internal_to_external_node_map, - speed_profile); + EdgeBasedGraphFactory edge_based_graph_factory( + node_based_graph, compressed_edge_container, barrier_nodes, traffic_lights, + std::const_pointer_cast(restriction_map), + internal_to_external_node_map, speed_profile); compressed_edge_container.SerializeInternalVector(config.geometry_output_path); @@ -497,9 +499,9 @@ Prepare::BuildEdgeExpandedGraph(std::vector &internal_to_external_nod \brief Build contracted graph. */ void Prepare::ContractGraph(const unsigned max_edge_id, - DeallocatingVector& edge_based_edge_list, - DeallocatingVector& contracted_edge_list, - std::vector& is_core_node) + DeallocatingVector &edge_based_edge_list, + DeallocatingVector &contracted_edge_list, + std::vector &is_core_node) { Contractor contractor(max_edge_id + 1, edge_based_edge_list); contractor.Run(config.core_factor); @@ -510,14 +512,14 @@ void Prepare::ContractGraph(const unsigned max_edge_id, /** \brief Writing info on original (node-based) nodes */ -void Prepare::WriteNodeMapping(std::unique_ptr> internal_to_external_node_map) +void Prepare::WriteNodeMapping(const std::vector &internal_to_external_node_map) { boost::filesystem::ofstream node_stream(config.node_output_path, std::ios::binary); - const unsigned size_of_mapping = internal_to_external_node_map->size(); + const unsigned size_of_mapping = internal_to_external_node_map.size(); node_stream.write((char *)&size_of_mapping, sizeof(unsigned)); if (size_of_mapping > 0) { - node_stream.write((char *) internal_to_external_node_map->data(), + node_stream.write((char *)internal_to_external_node_map.data(), size_of_mapping * sizeof(QueryNode)); } node_stream.close(); @@ -528,8 +530,10 @@ void Prepare::WriteNodeMapping(std::unique_ptr> internal_ Saves tree into '.ramIndex' and leaves into '.fileIndex'. */ -void Prepare::BuildRTree(const std::vector &node_based_edge_list, const std::vector& internal_to_external_node_map) +void Prepare::BuildRTree(const std::vector &node_based_edge_list, + const std::vector &internal_to_external_node_map) { StaticRTree(node_based_edge_list, config.rtree_nodes_output_path.c_str(), - config.rtree_leafs_output_path.c_str(), internal_to_external_node_map); + config.rtree_leafs_output_path.c_str(), + internal_to_external_node_map); } diff --git a/contractor/processing_chain.hpp b/contractor/processing_chain.hpp index 63560ebf9ea..4e12baf6d70 100644 --- a/contractor/processing_chain.hpp +++ b/contractor/processing_chain.hpp @@ -58,30 +58,32 @@ class Prepare int Run(); protected: - void SetupScriptingEnvironment(lua_State *myLuaState, - SpeedProfileProperties &speed_profile); - unsigned CalculateEdgeChecksum(std::unique_ptr> node_based_edge_list); + void SetupScriptingEnvironment(lua_State *myLuaState, SpeedProfileProperties &speed_profile); + unsigned CalculateEdgeChecksum(const std::vector &node_based_edge_list); void ContractGraph(const unsigned max_edge_id, - DeallocatingVector& edge_based_edge_list, - DeallocatingVector& contracted_edge_list, - std::vector& is_core_node); - void WriteCoreNodeMarker(std::vector&& is_core_node) const; + DeallocatingVector &edge_based_edge_list, + DeallocatingVector &contracted_edge_list, + std::vector &is_core_node); + void WriteCoreNodeMarker(std::vector &&is_core_node) const; std::size_t WriteContractedGraph(unsigned number_of_edge_based_nodes, - std::unique_ptr> node_based_edge_list, - std::unique_ptr> contracted_edge_list); + const std::vector &node_based_edge_list, + const DeallocatingVector &contracted_edge_list); std::shared_ptr LoadRestrictionMap(); std::shared_ptr LoadNodeBasedGraph(std::unordered_set &barrier_nodes, std::unordered_set &traffic_lights, - std::vector& internal_to_external_node_map); + std::vector &internal_to_external_node_map); std::pair BuildEdgeExpandedGraph(std::vector &internal_to_external_node_map, - std::vector &node_based_edge_list, - DeallocatingVector &edge_based_edge_list); - void WriteNodeMapping(std::unique_ptr> internal_to_external_node_map); - void FindComponents(unsigned max_edge_id, const DeallocatingVector& edges, std::vector& nodes) const; + std::vector &node_based_edge_list, + DeallocatingVector &edge_based_edge_list); + void WriteNodeMapping(const std::vector &internal_to_external_node_map); + void FindComponents(unsigned max_edge_id, + const DeallocatingVector &edges, + std::vector &nodes) const; void BuildRTree(const std::vector &node_based_edge_list, const std::vector &internal_to_external_node_map); + private: ContractorConfig config; }; From 4fa90229322408f514b2f3a8dfe719a5a6aa860a Mon Sep 17 00:00:00 2001 From: "Daniel J. Hofmann" Date: Thu, 10 Sep 2015 11:04:50 +0200 Subject: [PATCH 113/122] Use iterator pair taking parallel_sort, old TBB versions have no range overload. --- contractor/processing_chain.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contractor/processing_chain.cpp b/contractor/processing_chain.cpp index 0c882424522..574aea73353 100644 --- a/contractor/processing_chain.cpp +++ b/contractor/processing_chain.cpp @@ -230,7 +230,7 @@ std::size_t Prepare::WriteContractedGraph(unsigned max_node_id, const unsigned crc32_value = CalculateEdgeChecksum(node_based_edge_list); // Sorting contracted edges in a way that the static query graph can read some in in-place. - tbb::parallel_sort(contracted_edge_list); + tbb::parallel_sort(contracted_edge_list.begin(), contracted_edge_list.end()); const unsigned contracted_edge_count = contracted_edge_list.size(); SimpleLogger().Write() << "Serializing compacted graph of " << contracted_edge_count << " edges"; From a95bf64ccf38cb3dc2a4f0986fd94c267e8c0f7e Mon Sep 17 00:00:00 2001 From: Patrick Niklaus Date: Thu, 10 Sep 2015 12:21:36 +0200 Subject: [PATCH 114/122] Fix processing for data files with incorrect node references --- data_structures/dynamic_graph.hpp | 1 + extractor/extraction_containers.cpp | 52 +++++++++++++++++++---------- extractor/extraction_containers.hpp | 1 + 3 files changed, 36 insertions(+), 18 deletions(-) diff --git a/data_structures/dynamic_graph.hpp b/data_structures/dynamic_graph.hpp index b8c3e42a80f..4f63c02d482 100644 --- a/data_structures/dynamic_graph.hpp +++ b/data_structures/dynamic_graph.hpp @@ -119,6 +119,7 @@ template class DynamicGraph node_array[node].first_edge + node_array[node].edges)) { edge_list[i].target = graph[edge].target; + BOOST_ASSERT(edge_list[i].target < number_of_nodes); edge_list[i].data = graph[edge].data; ++edge; } diff --git a/extractor/extraction_containers.cpp b/extractor/extraction_containers.cpp index e5344de35cc..df659b2d9e9 100644 --- a/extractor/extraction_containers.cpp +++ b/extractor/extraction_containers.cpp @@ -157,20 +157,41 @@ void ExtractionContainers::PrepareNodes() TIMER_STOP(erasing_dups); std::cout << "ok, after " << TIMER_SEC(erasing_dups) << "s" << std::endl; - std::cout << "[extractor] Building node id map ... " << std::flush; - TIMER_START(id_map); - external_to_internal_node_id_map.reserve(used_node_id_list.size()); - for (NodeID i = 0u; i < used_node_id_list.size(); ++i) - external_to_internal_node_id_map[used_node_id_list[i]] = i; - TIMER_STOP(id_map); - std::cout << "ok, after " << TIMER_SEC(id_map) << "s" << std::endl; - std::cout << "[extractor] Sorting all nodes ... " << std::flush; TIMER_START(sorting_nodes); stxxl::sort(all_nodes_list.begin(), all_nodes_list.end(), ExternalMemoryNodeSTXXLCompare(), stxxl_memory); TIMER_STOP(sorting_nodes); std::cout << "ok, after " << TIMER_SEC(sorting_nodes) << "s" << std::endl; + + std::cout << "[extractor] Building node id map ... " << std::flush; + TIMER_START(id_map); + external_to_internal_node_id_map.reserve(used_node_id_list.size()); + auto node_iter = all_nodes_list.begin(); + auto ref_iter = used_node_id_list.begin(); + auto internal_id = 0u; + // compute the intersection of nodes that were referenced and nodes we actually have + while (node_iter != all_nodes_list.end() && ref_iter != used_node_id_list.end()) + { + if (node_iter->node_id < *ref_iter) + { + node_iter++; + continue; + } + if (node_iter->node_id > *ref_iter) + { + ref_iter++; + continue; + } + BOOST_ASSERT(node_iter->node_id == *ref_iter); + external_to_internal_node_id_map[*ref_iter] = internal_id++; + node_iter++; + ref_iter++; + } + max_internal_node_id = internal_id; + TIMER_STOP(id_map); + std::cout << "ok, after " << TIMER_SEC(id_map) << "s" << std::endl; + } void ExtractionContainers::PrepareEdges(lua_State *segment_state) @@ -444,9 +465,11 @@ void ExtractionContainers::WriteEdges(std::ofstream& file_out_stream) const void ExtractionContainers::WriteNodes(std::ofstream& file_out_stream) const { - unsigned number_of_used_nodes = 0; // write dummy value, will be overwritten later - file_out_stream.write((char *)&number_of_used_nodes, sizeof(unsigned)); + std::cout << "[extractor] setting number of nodes ... " << std::flush; + file_out_stream.write((char *)&max_internal_node_id, sizeof(unsigned)); + std::cout << "ok" << std::endl; + std::cout << "[extractor] Confirming/Writing used nodes ... " << std::flush; TIMER_START(write_nodes); // identify all used nodes by a merging step of two sorted lists @@ -468,21 +491,14 @@ void ExtractionContainers::WriteNodes(std::ofstream& file_out_stream) const file_out_stream.write((char *)&(*node_iterator), sizeof(ExternalMemoryNode)); - ++number_of_used_nodes; ++node_id_iterator; ++node_iterator; } TIMER_STOP(write_nodes); std::cout << "ok, after " << TIMER_SEC(write_nodes) << "s" << std::endl; - std::cout << "[extractor] setting number of nodes ... " << std::flush; - std::ios::pos_type previous_file_position = file_out_stream.tellp(); - file_out_stream.seekp(std::ios::beg + sizeof(FingerPrint)); - file_out_stream.write((char *)&number_of_used_nodes, sizeof(unsigned)); - file_out_stream.seekp(previous_file_position); - std::cout << "ok" << std::endl; - SimpleLogger().Write() << "Processed " << number_of_used_nodes << " nodes"; + SimpleLogger().Write() << "Processed " << max_internal_node_id << " nodes"; } void ExtractionContainers::WriteRestrictions(const std::string& path) const diff --git a/extractor/extraction_containers.hpp b/extractor/extraction_containers.hpp index 45285e03e51..8b7a82930b6 100644 --- a/extractor/extraction_containers.hpp +++ b/extractor/extraction_containers.hpp @@ -75,6 +75,7 @@ class ExtractionContainers STXXLRestrictionsVector restrictions_list; STXXLWayIDStartEndVector way_start_end_id_list; std::unordered_map external_to_internal_node_id_map; + unsigned max_internal_node_id; ExtractionContainers(); From a1e273e9839f9688de9e3b09d8c8172209eeac1b Mon Sep 17 00:00:00 2001 From: Patrick Niklaus Date: Thu, 10 Sep 2015 14:11:18 +0200 Subject: [PATCH 115/122] Add switch for handling fallback name --- profiles/foot.lua | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/profiles/foot.lua b/profiles/foot.lua index 172fe2abdd5..61751e356b2 100644 --- a/profiles/foot.lua +++ b/profiles/foot.lua @@ -65,8 +65,9 @@ leisure_speeds = { } traffic_signal_penalty = 2 -u_turn_penalty = 2 -use_turn_restrictions = false +u_turn_penalty = 2 +use_turn_restrictions = false +local fallback_names = true --modes local mode_normal = 1 @@ -154,7 +155,7 @@ function way_function (way, result) result.name = ref elseif name and "" ~= name then result.name = name - elseif highway then + elseif highway and fallback_names then result.name = "{highway:"..highway.."}" -- if no name exists, use way type -- this encoding scheme is excepted to be a temporary solution end From 5ac024788e9a009b8e0526aeec51e50b92b2de26 Mon Sep 17 00:00:00 2001 From: Lauren Budorick Date: Wed, 9 Sep 2015 21:53:41 -0700 Subject: [PATCH 116/122] Parse specific restriction:* tags based on profile exceptions --- extractor/restriction_parser.cpp | 30 +++++++++++++------ features/car/restrictions.feature | 48 +++++++++++++++++++++++++++++++ features/step_definitions/data.rb | 4 +-- 3 files changed, 72 insertions(+), 10 deletions(-) diff --git a/extractor/restriction_parser.cpp b/extractor/restriction_parser.cpp index 9b209b71991..86174d41030 100644 --- a/extractor/restriction_parser.cpp +++ b/extractor/restriction_parser.cpp @@ -35,6 +35,7 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include #include +#include #include #include @@ -51,8 +52,7 @@ int lua_error_callback(lua_State *lua_state) } } -RestrictionParser::RestrictionParser(lua_State *lua_state) - : use_turn_restrictions(true) +RestrictionParser::RestrictionParser(lua_State *lua_state) : use_turn_restrictions(true) { ReadUseRestrictionsSetting(lua_state); @@ -141,16 +141,30 @@ RestrictionParser::TryParse(const osmium::Relation &relation) const bool is_only_restriction = false; - for (auto iter = fi_begin; iter != fi_end; ++iter) + for (; fi_begin != fi_end; ++fi_begin) { - if (std::string("restriction") == iter->key() || - std::string("restriction::hgv") == iter->key()) + const std::string key(fi_begin->key()); + const std::string value(fi_begin->value()); + + if (value.find("only_") == 0) { - const std::string restriction_value(iter->value()); + is_only_restriction = true; + } + + // if the "restriction*" key is longer than 11 chars, it is a conditional exception (i.e. + // "restriction:") + if (key.size() > 11) + { + const auto ex_suffix = [&](const std::string &exception) + { + return boost::algorithm::ends_with(key, exception); + }; + bool is_actually_restricted = + std::any_of(begin(restriction_exceptions), end(restriction_exceptions), ex_suffix); - if (restriction_value.find("only_") == 0) + if (!is_actually_restricted) { - is_only_restriction = true; + return mapbox::util::optional(); } } } diff --git a/features/car/restrictions.feature b/features/car/restrictions.feature index f381f2f8c09..89fffff755c 100644 --- a/features/car/restrictions.feature +++ b/features/car/restrictions.feature @@ -226,6 +226,54 @@ Feature: Car - Turn restrictions | s | n | sj,nj | | s | e | | + @specific + Scenario: Car - :hgv-qualified on a standard turn restriction + Given the node map + | | n | | + | w | j | e | + | | s | | + + And the ways + | nodes | oneway | + | sj | yes | + | nj | -1 | + | wj | -1 | + | ej | -1 | + + And the relations + | type | way:from | way:to | node:via | restriction:hgv | + | restriction | sj | nj | j | no_straight_on | + + When I route I should get + | from | to | route | + | s | w | sj,wj | + | s | n | sj,nj | + | s | e | sj,ej | + + @specific + Scenario: Car - :motorcar-qualified on a standard turn restriction + Given the node map + | | n | | + | w | j | e | + | | s | | + + And the ways + | nodes | oneway | + | sj | yes | + | nj | -1 | + | wj | -1 | + | ej | -1 | + + And the relations + | type | way:from | way:to | node:via | restriction:motorcar | + | restriction | sj | nj | j | no_straight_on | + + When I route I should get + | from | to | route | + | s | w | sj,wj | + | s | n | | + | s | e | sj,ej | + @except Scenario: Car - Except tag and on no_ restrictions Given the node map diff --git a/features/step_definitions/data.rb b/features/step_definitions/data.rb index ff074d24fa0..0f338432360 100644 --- a/features/step_definitions/data.rb +++ b/features/step_definitions/data.rb @@ -115,8 +115,8 @@ raise "*** unknown relation way member '#{way_name}'" unless way relation << OSM::Member.new( 'way', way.id, $1 ) end - elsif key =~ /^(.*):(.*)/ - raise "*** unknown relation member type '#{$1}', must be either 'node' or 'way'" + elsif key =~ /^(.*):(.*)/ && "#{$1}" != 'restriction' + raise "*** unknown relation member type '#{$1}:#{$2}', must be either 'node' or 'way'" else relation << { key => value } end From de2957431455c3234d7e1ff888903e1491d46c56 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20Gru=C3=9F?= Date: Wed, 10 Jun 2015 11:40:35 +0200 Subject: [PATCH 117/122] tests + instructions for map matching --- descriptors/json_descriptor.hpp | 53 ++++---- features/step_definitions/matching.rb | 170 +++++++++++++++++++++++- features/support/match.rb | 2 +- features/testbot/matching_turns.feature | 82 ++++++++++++ plugins/match.hpp | 26 +++- 5 files changed, 296 insertions(+), 37 deletions(-) create mode 100644 features/testbot/matching_turns.feature diff --git a/descriptors/json_descriptor.hpp b/descriptors/json_descriptor.hpp index 13b68c6ecdf..4904e8527e7 100644 --- a/descriptors/json_descriptor.hpp +++ b/descriptors/json_descriptor.hpp @@ -52,15 +52,8 @@ template class JSONDescriptor final : public BaseDescriptor< DescriptorConfig config; DescriptionFactory description_factory, alternate_description_factory; FixedPointCoordinate current; - unsigned entered_restricted_area_count; - struct RoundAbout - { - RoundAbout() : start_index(INT_MAX), name_id(INVALID_NAMEID), leave_at_exit(INT_MAX) {} - int start_index; - unsigned name_id; - int leave_at_exit; - } round_about; + public: struct Segment { Segment() : name_id(INVALID_NAMEID), length(-1), position(0) {} @@ -69,11 +62,12 @@ template class JSONDescriptor final : public BaseDescriptor< int length; unsigned position; }; + private: std::vector shortest_path_segments, alternative_path_segments; ExtractRouteNames GenerateRouteNames; public: - explicit JSONDescriptor(DataFacadeT *facade) : facade(facade), entered_restricted_area_count(0) + explicit JSONDescriptor(DataFacadeT *facade) : facade(facade) { } @@ -143,9 +137,7 @@ template class JSONDescriptor final : public BaseDescriptor< } if (config.instructions) { - osrm::json::Array json_route_instructions; - BuildTextualDescription(description_factory, json_route_instructions, - raw_route.shortest_path_length, shortest_path_segments); + osrm::json::Array json_route_instructions = BuildTextualDescription(description_factory, shortest_path_segments); json_result.values["route_instructions"] = json_route_instructions; } description_factory.BuildRouteSummary(description_factory.get_entire_length(), @@ -222,9 +214,7 @@ template class JSONDescriptor final : public BaseDescriptor< osrm::json::Array json_current_alt_instructions; if (config.instructions) { - BuildTextualDescription( - alternate_description_factory, json_current_alt_instructions, - raw_route.alternative_path_length, alternative_path_segments); + json_alt_instructions = BuildTextualDescription(alternate_description_factory, alternative_path_segments); json_alt_instructions.values.push_back(json_current_alt_instructions); json_result.values["alternative_instructions"] = json_alt_instructions; } @@ -276,6 +266,11 @@ template class JSONDescriptor final : public BaseDescriptor< json_result.values["alternative_names"] = json_alternate_names_array; } + json_result.values["hint_data"] = BuildHintData(raw_route); + } + + inline osrm::json::Object BuildHintData(const InternalRouteResult& raw_route) const + { osrm::json::Object json_hint_object; json_hint_object.values["checksum"] = facade->GetCheckSum(); osrm::json::Array json_location_hint_array; @@ -290,24 +285,27 @@ template class JSONDescriptor final : public BaseDescriptor< hint); json_location_hint_array.values.push_back(hint); json_hint_object.values["locations"] = json_location_hint_array; - json_result.values["hint_data"] = json_hint_object; - // render the content to the output array - // TIMER_START(route_render); - // osrm::json::render(reply.content, json_result); - // TIMER_STOP(route_render); - // SimpleLogger().Write(logDEBUG) << "rendering took: " << TIMER_MSEC(route_render); + return json_hint_object; } - // TODO: reorder parameters - inline void BuildTextualDescription(DescriptionFactory &description_factory, - osrm::json::Array &json_instruction_array, - const int route_length, - std::vector &route_segments_list) + inline osrm::json::Array BuildTextualDescription(const DescriptionFactory &description_factory, + std::vector &route_segments_list) const { + osrm::json::Array json_instruction_array; + // Segment information has following format: //["instruction id","streetname",length,position,time,"length","earth_direction",azimuth] unsigned necessary_segments_running_index = 0; + + struct RoundAbout + { + RoundAbout() : start_index(INT_MAX), name_id(INVALID_NAMEID), leave_at_exit(INT_MAX) {} + int start_index; + unsigned name_id; + int leave_at_exit; + } round_about; + round_about.leave_at_exit = 0; round_about.name_id = 0; std::string temp_dist, temp_length, temp_duration, temp_bearing, temp_instruction; @@ -317,7 +315,6 @@ template class JSONDescriptor final : public BaseDescriptor< { osrm::json::Array json_instruction_row; TurnInstruction current_instruction = segment.turn_instruction; - entered_restricted_area_count += (current_instruction != segment.turn_instruction); if (TurnInstructionsClass::TurnIsNecessary(current_instruction)) { if (TurnInstruction::EnterRoundAbout == current_instruction) @@ -386,6 +383,8 @@ template class JSONDescriptor final : public BaseDescriptor< json_last_instruction_row.values.push_back(bearing::get(0.0)); json_last_instruction_row.values.push_back(0.); json_instruction_array.values.push_back(json_last_instruction_row); + + return json_instruction_array; } }; diff --git a/features/step_definitions/matching.rb b/features/step_definitions/matching.rb index 607efedd975..3015f5a437e 100644 --- a/features/step_definitions/matching.rb +++ b/features/step_definitions/matching.rb @@ -58,9 +58,6 @@ end end - puts table - puts sub_matchings - ok = true encoded_result = "" extended_target = "" @@ -97,3 +94,170 @@ table.diff! actual end +When /^I match with turns I should get$/ do |table| + reprocess + actual = [] + OSRMLoader.load(self,"#{prepared_file}.osrm") do + table.hashes.each_with_index do |row,ri| + if row['request'] + got = {'request' => row['request'] } + response = request_url row['request'] + else + params = @query_params + trace = [] + timestamps = [] + if row['from'] and row['to'] + node = find_node_by_name(row['from']) + raise "*** unknown from-node '#{row['from']}" unless node + trace << node + + node = find_node_by_name(row['to']) + raise "*** unknown to-node '#{row['to']}" unless node + trace << node + + got = {'from' => row['from'], 'to' => row['to'] } + response = request_matching trace, timestamps, params + elsif row['trace'] + row['trace'].each_char do |n| + node = find_node_by_name(n.strip) + raise "*** unknown waypoint node '#{n.strip}" unless node + trace << node + end + if row['timestamps'] + timestamps = row['timestamps'].split(" ").compact.map { |t| t.to_i} + end + got = {'trace' => row['trace'] } + response = request_matching trace, timestamps, params + else + raise "*** no trace" + end + end + + row.each_pair do |k,v| + if k =~ /param:(.*)/ + if v=='(nil)' + params[$1]=nil + elsif v!=nil + params[$1]=v + end + got[k]=v + end + end + + if response.body.empty? == false + json = JSON.parse response.body + end + if response.body.empty? == false + if response.code == "200" + instructions = way_list json['matchings'][0]['instructions'] + bearings = bearing_list json['matchings'][0]['instructions'] + compasses = compass_list json['matchings'][0]['instructions'] + turns = turn_list json['matchings'][0]['instructions'] + modes = mode_list json['matchings'][0]['instructions'] + times = time_list json['matchings'][0]['instructions'] + distances = distance_list json['matchings'][0]['instructions'] + end + end + + if table.headers.include? 'status' + got['status'] = json['status'].to_s + end + if table.headers.include? 'message' + got['message'] = json['status_message'] + end + if table.headers.include? '#' # comment column + got['#'] = row['#'] # copy value so it always match + end + + sub_matchings = [] + if response.code == "200" + if table.headers.include? 'matchings' + sub_matchings = json['matchings'].compact.map { |sub| sub['matched_points']} + + got['route'] = (instructions || '').strip + if table.headers.include?('distance') + if row['distance']!='' + raise "*** Distance must be specied in meters. (ex: 250m)" unless row['distance'] =~ /\d+m/ + end + got['distance'] = instructions ? "#{json['route_summary']['total_distance'].to_s}m" : '' + end + if table.headers.include?('time') + raise "*** Time must be specied in seconds. (ex: 60s)" unless row['time'] =~ /\d+s/ + got['time'] = instructions ? "#{json['route_summary']['total_time'].to_s}s" : '' + end + if table.headers.include?('speed') + if row['speed'] != '' && instructions + raise "*** Speed must be specied in km/h. (ex: 50 km/h)" unless row['speed'] =~ /\d+ km\/h/ + time = json['route_summary']['total_time'] + distance = json['route_summary']['total_distance'] + speed = time>0 ? (3.6*distance/time).to_i : nil + got['speed'] = "#{speed} km/h" + else + got['speed'] = '' + end + end + if table.headers.include? 'bearing' + got['bearing'] = instructions ? bearings : '' + end + if table.headers.include? 'compass' + got['compass'] = instructions ? compasses : '' + end + if table.headers.include? 'turns' + got['turns'] = instructions ? turns : '' + end + if table.headers.include? 'modes' + got['modes'] = instructions ? modes : '' + end + if table.headers.include? 'times' + got['times'] = instructions ? times : '' + end + if table.headers.include? 'distances' + got['distances'] = instructions ? distances : '' + end + end + if table.headers.include? 'start' + got['start'] = instructions ? json['route_summary']['start_point'] : nil + end + if table.headers.include? 'end' + got['end'] = instructions ? json['route_summary']['end_point'] : nil + end + if table.headers.include? 'geometry' + got['geometry'] = json['route_geometry'] + end + end + + ok = true + encoded_result = "" + extended_target = "" + row['matchings'].split(',').each_with_index do |sub, sub_idx| + if sub_idx >= sub_matchings.length + ok = false + break + end + sub.length.times do |node_idx| + node = find_node_by_name(sub[node_idx]) + out_node = sub_matchings[sub_idx][node_idx] + if FuzzyMatch.match_location out_node, node + encoded_result += sub[node_idx] + extended_target += sub[node_idx] + else + encoded_result += "? [#{out_node[0]},#{out_node[1]}]" + extended_target += "#{sub[node_idx]} [#{node.lat},#{node.lon}]" + ok = false + end + end + end + if ok + got['matchings'] = row['matchings'] + got['timestamps'] = row['timestamps'] + else + got['matchings'] = encoded_result + row['matchings'] = extended_target + log_fail row,got, { 'matching' => {:query => @query, :response => response} } + end + + actual << got + end + end + table.diff! actual +end diff --git a/features/support/match.rb b/features/support/match.rb index 8bcf7a2c600..165dfe0036a 100644 --- a/features/support/match.rb +++ b/features/support/match.rb @@ -3,7 +3,7 @@ HOST = "http://127.0.0.1:#{OSRM_PORT}" def request_matching trace=[], timestamps=[], options={} - defaults = { 'output' => 'json' } + defaults = { 'output' => 'json', 'instructions' => 'true' } locs = trace.compact.map { |w| "loc=#{w.lat},#{w.lon}" } ts = timestamps.compact.map { |t| "t=#{t}" } if ts.length > 0 diff --git a/features/testbot/matching_turns.feature b/features/testbot/matching_turns.feature new file mode 100644 index 00000000000..2d0a5582080 --- /dev/null +++ b/features/testbot/matching_turns.feature @@ -0,0 +1,82 @@ +@routing @turns @testbot +Feature: Turn directions/codes + + Background: + Given the profile "testbot" + + Scenario: Turn directions + Given the node map + | o | p | a | b | c | + | n | | | | d | + | m | | x | | e | + | l | | | | f | + | k | j | i | h | g | + + And the ways + | nodes | + | xa | + | xb | + | xc | + | xd | + | xe | + | xf | + | xg | + | xh | + | xi | + | xj | + | xk | + | xl | + | xm | + | xn | + | xo | + | xp | + + When I match with turns I should get + | trace | route | turns | matchings | + | im | xi,xm | head,left,destination | im | + | io | xi,xo | head,slight_left,destination | io | + | ia | xi,xa | head,straight,destination | ia | + | ic | xi,xc | head,slight_right,destination | ic | + | ie | xi,xe | head,right,destination | ie | + + | ko | xk,xo | head,left,destination | ko | + | ka | xk,xa | head,slight_left,destination | ka | + | kc | xk,xc | head,straight,destination | kc | + | ke | xk,xe | head,slight_right,destination | ke | + | kg | xk,xg | head,right,destination | kg | + + | ma | xm,xa | head,left,destination | ma | + | mc | xm,xc | head,slight_left,destination | mc | + | me | xm,xe | head,straight,destination | me | + | mg | xm,xg | head,slight_right,destination | mg | + | mi | xm,xi | head,right,destination | mi | + + | oc | xo,xc | head,left,destination | oc | + | oe | xo,xe | head,slight_left,destination | oe | + | og | xo,xg | head,straight,destination | og | + | oi | xo,xi | head,slight_right,destination | oi | + | ok | xo,xk | head,right,destination | ok | + + | ae | xa,xe | head,left,destination | ae | + | ag | xa,xg | head,slight_left,destination | ag | + | ai | xa,xi | head,straight,destination | ai | + | ak | xa,xk | head,slight_right,destination | ak | + | am | xa,xm | head,right,destination | am | + + | cg | xc,xg | head,left,destination | cg | + | ci | xc,xi | head,slight_left,destination | ci | + | ck | xc,xk | head,straight,destination | ck | + | cm | xc,xm | head,slight_right,destination | cm | + | co | xc,xo | head,right,destination | co | + + | ei | xe,xi | head,left,destination | ei | + | ek | xe,xk | head,slight_left,destination | ek | + | em | xe,xm | head,straight,destination | em | + | eo | xe,xo | head,slight_right,destination | eo | + | ea | xe,xa | head,right,destination | ea | + + | gk | xg,xk | head,left,destination | gk | + | gm | xg,xm | head,slight_left,destination | gm | + | go | xg,xo | head,straight,destination | go | + | ga | xg,xa | head,slight_right,destination | ga | + | gc | xg,xc | head,right,destination | gc | \ No newline at end of file diff --git a/plugins/match.hpp b/plugins/match.hpp index 120fba1ef6b..33ade953d44 100644 --- a/plugins/match.hpp +++ b/plugins/match.hpp @@ -187,7 +187,12 @@ template class MapMatchingPlugin : public BasePlugin subtrace.values["confidence"] = sub.confidence; } - if (route_parameters.geometry) + JSONDescriptor json_descriptor(facade); + json_descriptor.SetConfig(route_parameters); + + subtrace.values["hint_data"] = json_descriptor.BuildHintData(raw_route); + + if (route_parameters.geometry || route_parameters.print_instructions) { DescriptionFactory factory; FixedPointCoordinate current_coordinate; @@ -205,13 +210,22 @@ template class MapMatchingPlugin : public BasePlugin raw_route.target_traversed_in_reverse[i], raw_route.is_via_leg(i)); } - // we need this because we don't run DP - for (auto &segment : factory.path_description) + + // we run this to get the instructions + factory.Run(route_parameters.zoom_level); + + if (route_parameters.geometry) { - segment.necessary = true; + subtrace.values["geometry"] = factory.AppendGeometryString(route_parameters.compression); } - subtrace.values["geometry"] = - factory.AppendGeometryString(route_parameters.compression); + + + if (route_parameters.print_instructions) + { + std::vector::Segment> temp_segments; + subtrace.values["instructions"] = json_descriptor.BuildTextualDescription(factory, temp_segments); + } + } subtrace.values["indices"] = osrm::json::make_array(sub.indices); From fe0fe1873a774e370868a622bbd53971abb08b8e Mon Sep 17 00:00:00 2001 From: Patrick Niklaus Date: Thu, 10 Sep 2015 16:47:46 +0200 Subject: [PATCH 118/122] Add simplification reset that was accidentally removed --- plugins/match.hpp | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/plugins/match.hpp b/plugins/match.hpp index 33ade953d44..39ca3247ce5 100644 --- a/plugins/match.hpp +++ b/plugins/match.hpp @@ -211,9 +211,14 @@ template class MapMatchingPlugin : public BasePlugin raw_route.is_via_leg(i)); } - // we run this to get the instructions factory.Run(route_parameters.zoom_level); + // we need because we don't run path simplification + for (auto &segment : factory.path_description) + { + segment.necessary = true; + } + if (route_parameters.geometry) { subtrace.values["geometry"] = factory.AppendGeometryString(route_parameters.compression); From 8e02263084e844132825ae338c018dafbbe62553 Mon Sep 17 00:00:00 2001 From: Patrick Niklaus Date: Mon, 14 Sep 2015 23:01:38 +0200 Subject: [PATCH 119/122] Fix off-by one error in decoder and make padding deterministic. --- algorithms/object_encoder.hpp | 2 +- data_structures/phantom_node.hpp | 8 ++++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/algorithms/object_encoder.hpp b/algorithms/object_encoder.hpp index e880496a2d1..3ffac410d86 100644 --- a/algorithms/object_encoder.hpp +++ b/algorithms/object_encoder.hpp @@ -79,7 +79,7 @@ struct ObjectEncoder replaceAll(encoded, "-", "+"); replaceAll(encoded, "_", "/"); - std::copy(binary_t(encoded.begin()), binary_t(encoded.begin() + encoded.length() - 1), + std::copy(binary_t(encoded.begin()), binary_t(encoded.begin() + encoded.length()), reinterpret_cast(&object)); } catch (...) diff --git a/data_structures/phantom_node.hpp b/data_structures/phantom_node.hpp index b048eb7f218..670408961cf 100644 --- a/data_structures/phantom_node.hpp +++ b/data_structures/phantom_node.hpp @@ -88,8 +88,10 @@ struct PhantomNode unsigned component_id; FixedPointCoordinate location; unsigned short fwd_segment_position; - TravelMode forward_travel_mode : 4; - TravelMode backward_travel_mode : 4; + // note 4 bits would suffice for each, + // but the saved byte would be padding anyway + TravelMode forward_travel_mode; + TravelMode backward_travel_mode; int GetForwardWeightPlusOffset() const; @@ -108,6 +110,8 @@ struct PhantomNode bool operator==(const PhantomNode &other) const; }; +static_assert(sizeof(PhantomNode) == 48, "PhantomNode has more padding then expected"); + using PhantomNodeArray = std::vector>; class phantom_node_pair : public std::pair From 94af9b7f130278baade79512d56d4cb0a2376508 Mon Sep 17 00:00:00 2001 From: "Daniel J. Hofmann" Date: Tue, 15 Sep 2015 01:13:31 +0200 Subject: [PATCH 120/122] Caches iterators instead of invoking function calls on every iteration. This caches iterators, i.e. especially the end iterator when possible. The problem: for (auto it = begin(seq); it != end(seq); ++it) this has to call `end(seq)` on every iteration, since the compiler is not able to reason about the call's site effects (to bad, huh). Instead do it like this: for (auto it = begin(seq), end = end(seq); it != end; ++it) caching the end iterator. Of course, still better would be: for (auto&& each : seq) if all you want is value semantics. Why `auto&&` you may ask? Because it binds to everything and never copies! Skim the referenced proposal (that was rejected, but nevertheless) for a detailed explanation on range-based for loops and why `auto&&` is great. Reference: - http://www.open-std.org/jtc1/sc22/wg21/docs/papers/2014/n3853.htm --- contractor/contractor.hpp | 12 +++++----- data_structures/static_rtree.hpp | 14 +++++++----- extractor/extraction_containers.cpp | 34 ++++++++++++++++++++++------- extractor/extractor.cpp | 4 ++-- util/json_renderer.hpp | 34 ++++++++++++----------------- util/string_util.hpp | 3 ++- util/xml_renderer.hpp | 21 +++++++----------- 7 files changed, 66 insertions(+), 56 deletions(-) diff --git a/contractor/contractor.hpp b/contractor/contractor.hpp index f4f2c95e8e6..d6f564259ff 100644 --- a/contractor/contractor.hpp +++ b/contractor/contractor.hpp @@ -312,7 +312,7 @@ class Contractor tbb::parallel_for(tbb::blocked_range(0, number_of_nodes, InitGrainSize), [&remaining_nodes](const tbb::blocked_range &range) { - for (int x = range.begin(); x != range.end(); ++x) + for (int x = range.begin(), end = range.end(); x != end; ++x) { remaining_nodes[x].id = x; } @@ -324,7 +324,7 @@ class Contractor const tbb::blocked_range &range) { ContractorThreadData *data = thread_data_list.getThreadData(); - for (int x = range.begin(); x != range.end(); ++x) + for (int x = range.begin(), end = range.end(); x != end; ++x) { node_priorities[x] = this->EvaluateNodePriority(data, &node_data[x], x); @@ -430,7 +430,7 @@ class Contractor { ContractorThreadData *data = thread_data_list.getThreadData(); // determine independent node set - for (int i = range.begin(); i != range.end(); ++i) + for (int i = range.begin(), end = range.end(); i != end; ++i) { const NodeID node = remaining_nodes[i].id; remaining_nodes[i].is_independent = @@ -451,7 +451,7 @@ class Contractor [this, &remaining_nodes, &thread_data_list](const tbb::blocked_range &range) { ContractorThreadData *data = thread_data_list.getThreadData(); - for (int position = range.begin(); position != range.end(); ++position) + for (int position = range.begin(), end = range.end(); position != end; ++position) { const NodeID x = remaining_nodes[position].id; this->ContractNode(data, x); @@ -470,7 +470,7 @@ class Contractor [this, &remaining_nodes, &thread_data_list](const tbb::blocked_range &range) { ContractorThreadData *data = thread_data_list.getThreadData(); - for (int position = range.begin(); position != range.end(); ++position) + for (int position = range.begin(), end = range.end(); position != end; ++position) { const NodeID x = remaining_nodes[position].id; this->DeleteIncomingEdges(data, x); @@ -508,7 +508,7 @@ class Contractor const tbb::blocked_range &range) { ContractorThreadData *data = thread_data_list.getThreadData(); - for (int position = range.begin(); position != range.end(); ++position) + for (int position = range.begin(), end = range.end(); position != end; ++position) { NodeID x = remaining_nodes[position].id; this->UpdateNodeNeighbours(node_priorities, node_data, data, x); diff --git a/data_structures/static_rtree.hpp b/data_structures/static_rtree.hpp index 07cd490c02d..314935a6895 100644 --- a/data_structures/static_rtree.hpp +++ b/data_structures/static_rtree.hpp @@ -365,8 +365,8 @@ class StaticRTree [&input_data_vector, &input_wrapper_vector, &get_hilbert_number, &coordinate_list]( const tbb::blocked_range &range) { - for (uint64_t element_counter = range.begin(); element_counter != range.end(); - ++element_counter) + for (uint64_t element_counter = range.begin(), end = range.end(); + element_counter != end; ++element_counter) { WrappedInputElement ¤t_wrapper = input_wrapper_vector[element_counter]; current_wrapper.m_array_index = element_counter; @@ -476,7 +476,7 @@ class StaticRTree tbb::parallel_for(tbb::blocked_range(0, search_tree_size), [this, &search_tree_size](const tbb::blocked_range &range) { - for (uint32_t i = range.begin(); i != range.end(); ++i) + for (uint32_t i = range.begin(), end = range.end(); i != end; ++i) { TreeNode ¤t_tree_node = this->m_search_tree[i]; for (uint32_t j = 0; j < current_tree_node.child_count; ++j) @@ -664,7 +664,8 @@ class StaticRTree while (!traversal_queue.empty()) { const IncrementalQueryCandidate current_query_node = traversal_queue.top(); - if (current_query_node.min_dist > max_distance && inspected_elements > max_checked_elements) + if (current_query_node.min_dist > max_distance && + inspected_elements > max_checked_elements) { break; } @@ -760,7 +761,7 @@ class StaticRTree // stop the search by flushing the queue if (result_phantom_node_vector.size() >= max_number_of_phantom_nodes && - number_of_elements_from_big_cc > 0) + number_of_elements_from_big_cc > 0) { traversal_queue = std::priority_queue{}; } @@ -804,7 +805,8 @@ class StaticRTree const IncrementalQueryCandidate current_query_node = traversal_queue.top(); traversal_queue.pop(); - if (current_query_node.min_dist > max_distance || inspected_elements >= max_checked_elements) + if (current_query_node.min_dist > max_distance || + inspected_elements >= max_checked_elements) { break; } diff --git a/extractor/extraction_containers.cpp b/extractor/extraction_containers.cpp index df659b2d9e9..d10a2f67304 100644 --- a/extractor/extraction_containers.cpp +++ b/extractor/extraction_containers.cpp @@ -169,9 +169,12 @@ void ExtractionContainers::PrepareNodes() external_to_internal_node_id_map.reserve(used_node_id_list.size()); auto node_iter = all_nodes_list.begin(); auto ref_iter = used_node_id_list.begin(); + const auto all_nodes_list_end = all_nodes_list.end(); + const auto used_node_id_list_end = used_node_id_list.end(); auto internal_id = 0u; + // compute the intersection of nodes that were referenced and nodes we actually have - while (node_iter != all_nodes_list.end() && ref_iter != used_node_id_list.end()) + while (node_iter != all_nodes_list_end && ref_iter != used_node_id_list_end) { if (node_iter->node_id < *ref_iter) { @@ -208,7 +211,11 @@ void ExtractionContainers::PrepareEdges(lua_State *segment_state) // Traverse list of edges and nodes in parallel and set start coord auto node_iterator = all_nodes_list.begin(); auto edge_iterator = all_edges_list.begin(); - while (edge_iterator != all_edges_list.end() && node_iterator != all_nodes_list.end()) + + const auto all_edges_list_end = all_edges_list.end(); + const auto all_nodes_list_end = all_nodes_list.end(); + + while (edge_iterator != all_edges_list_end && node_iterator != all_nodes_list_end) { if (edge_iterator->result.source < node_iterator->node_id) { @@ -259,7 +266,10 @@ void ExtractionContainers::PrepareEdges(lua_State *segment_state) TIMER_START(compute_weights); node_iterator = all_nodes_list.begin(); edge_iterator = all_edges_list.begin(); - while (edge_iterator != all_edges_list.end() && node_iterator != all_nodes_list.end()) + const auto all_edges_list_end_ = all_edges_list.end(); + const auto all_nodes_list_end_ = all_nodes_list.end(); + + while (edge_iterator != all_edges_list_end_ && node_iterator != all_nodes_list_end_) { // skip all invalid edges if (edge_iterator->result.source == SPECIAL_NODEID) @@ -475,7 +485,10 @@ void ExtractionContainers::WriteNodes(std::ofstream& file_out_stream) const // identify all used nodes by a merging step of two sorted lists auto node_iterator = all_nodes_list.begin(); auto node_id_iterator = used_node_id_list.begin(); - while (node_id_iterator != used_node_id_list.end() && node_iterator != all_nodes_list.end()) + const auto used_node_id_list_end = used_node_id_list.end(); + const auto all_nodes_list_end = all_nodes_list.end(); + + while (node_id_iterator != used_node_id_list_end && node_iterator != all_nodes_list_end) { if (*node_id_iterator < node_iterator->node_id) { @@ -550,9 +563,11 @@ void ExtractionContainers::PrepareRestrictions() TIMER_START(fix_restriction_starts); auto restrictions_iterator = restrictions_list.begin(); auto way_start_and_end_iterator = way_start_end_id_list.cbegin(); + const auto restrictions_list_end = restrictions_list.end(); + const auto way_start_end_id_list_end = way_start_end_id_list.cend(); - while (way_start_and_end_iterator != way_start_end_id_list.cend() && - restrictions_iterator != restrictions_list.end()) + while (way_start_and_end_iterator != way_start_end_id_list_end && + restrictions_iterator != restrictions_list_end) { if (way_start_and_end_iterator->way_id < restrictions_iterator->restriction.from.way) { @@ -616,8 +631,11 @@ void ExtractionContainers::PrepareRestrictions() TIMER_START(fix_restriction_ends); restrictions_iterator = restrictions_list.begin(); way_start_and_end_iterator = way_start_end_id_list.cbegin(); - while (way_start_and_end_iterator != way_start_end_id_list.cend() && - restrictions_iterator != restrictions_list.end()) + const auto way_start_end_id_list_end_ = way_start_end_id_list.cend(); + const auto restrictions_list_end_ = restrictions_list.end(); + + while (way_start_and_end_iterator != way_start_end_id_list_end_ && + restrictions_iterator != restrictions_list_end_) { if (way_start_and_end_iterator->way_id < restrictions_iterator->restriction.to.way) { diff --git a/extractor/extractor.cpp b/extractor/extractor.cpp index e988aa92bf7..2386f13ee84 100644 --- a/extractor/extractor.cpp +++ b/extractor/extractor.cpp @@ -161,7 +161,7 @@ int extractor::run() { // create a vector of iterators into the buffer std::vector osm_elements; - for (auto iter = std::begin(buffer); iter != std::end(buffer); ++iter) + for (auto iter = std::begin(buffer), end = std::end(buffer); iter != end; ++iter) { osm_elements.push_back(iter); } @@ -180,7 +180,7 @@ int extractor::run() ExtractionWay result_way; lua_State *local_state = scripting_environment.get_lua_state(); - for (auto x = range.begin(); x != range.end(); ++x) + for (auto x = range.begin(), end = range.end(); x != end; ++x) { const auto entity = osm_elements[x]; diff --git a/util/json_renderer.hpp b/util/json_renderer.hpp index e5bbf6f92ce..b6b96f66cd5 100644 --- a/util/json_renderer.hpp +++ b/util/json_renderer.hpp @@ -61,12 +61,11 @@ struct Renderer : mapbox::util::static_visitor<> void operator()(const Object &object) const { out << "{"; - auto iterator = object.values.begin(); - while (iterator != object.values.end()) + for (auto it = object.values.begin(), end = object.values.end(); it != end;) { - out << "\"" << (*iterator).first << "\":"; - mapbox::util::apply_visitor(Renderer(out), (*iterator).second); - if (++iterator != object.values.end()) + out << "\"" << it->first << "\":"; + mapbox::util::apply_visitor(Renderer(out), it->second); + if (++it != end) { out << ","; } @@ -77,12 +76,10 @@ struct Renderer : mapbox::util::static_visitor<> void operator()(const Array &array) const { out << "["; - std::vector::const_iterator iterator; - iterator = array.values.begin(); - while (iterator != array.values.end()) + for (auto it = array.values.cend(), end = array.values.cend(); it != end;) { - mapbox::util::apply_visitor(Renderer(out), *iterator); - if (++iterator != array.values.end()) + mapbox::util::apply_visitor(Renderer(out), *it); + if (++it != end) { out << ","; } @@ -121,16 +118,15 @@ struct ArrayRenderer : mapbox::util::static_visitor<> void operator()(const Object &object) const { out.push_back('{'); - auto iterator = object.values.begin(); - while (iterator != object.values.end()) + for (auto it = object.values.begin(), end = object.values.end(); it != end;) { out.push_back('\"'); - out.insert(out.end(), (*iterator).first.begin(), (*iterator).first.end()); + out.insert(out.end(), it->first.begin(), it->first.end()); out.push_back('\"'); out.push_back(':'); - mapbox::util::apply_visitor(ArrayRenderer(out), (*iterator).second); - if (++iterator != object.values.end()) + mapbox::util::apply_visitor(ArrayRenderer(out), it->second); + if (++it != end) { out.push_back(','); } @@ -141,12 +137,10 @@ struct ArrayRenderer : mapbox::util::static_visitor<> void operator()(const Array &array) const { out.push_back('['); - std::vector::const_iterator iterator; - iterator = array.values.begin(); - while (iterator != array.values.end()) + for (auto it = array.values.cbegin(), end = array.values.cend(); it != end;) { - mapbox::util::apply_visitor(ArrayRenderer(out), *iterator); - if (++iterator != array.values.end()) + mapbox::util::apply_visitor(ArrayRenderer(out), *it); + if (++it != end) { out.push_back(','); } diff --git a/util/string_util.hpp b/util/string_util.hpp index 3bfce003292..6e80a4e0595 100644 --- a/util/string_util.hpp +++ b/util/string_util.hpp @@ -132,9 +132,10 @@ inline std::string escape_JSON(const std::string &input) inline std::size_t URIDecode(const std::string &input, std::string &output) { auto src_iter = std::begin(input); + const auto src_end = std::end(input); output.resize(input.size() + 1); std::size_t decoded_length = 0; - for (decoded_length = 0; src_iter != std::end(input); ++decoded_length) + for (decoded_length = 0; src_iter != src_end; ++decoded_length) { if (src_iter[0] == '%' && src_iter[1] && src_iter[2] && isxdigit(src_iter[1]) && isxdigit(src_iter[2])) diff --git a/util/xml_renderer.hpp b/util/xml_renderer.hpp index 4ef1e5dc09c..f59810075b9 100644 --- a/util/xml_renderer.hpp +++ b/util/xml_renderer.hpp @@ -56,38 +56,33 @@ struct XMLToArrayRenderer : mapbox::util::static_visitor<> void operator()(const Object &object) const { - auto iterator = object.values.begin(); - while (iterator != object.values.end()) + for (auto &&each : object.values) { - if (iterator->first.at(0) != '_') + if (each.first.at(0) != '_') { out.push_back('<'); - out.insert(out.end(), (*iterator).first.begin(), (*iterator).first.end()); + out.insert(out.end(), each.first.begin(), each.first.end()); } else { out.push_back(' '); - out.insert(out.end(), ++(*iterator).first.begin(), (*iterator).first.end()); + out.insert(out.end(), ++(each).first.begin(), each.first.end()); out.push_back('='); } - mapbox::util::apply_visitor(XMLToArrayRenderer(out), (*iterator).second); - if (iterator->first.at(0) != '_') + mapbox::util::apply_visitor(XMLToArrayRenderer(out), each.second); + if (each.first.at(0) != '_') { out.push_back('/'); out.push_back('>'); } - ++iterator; } } void operator()(const Array &array) const { - std::vector::const_iterator iterator; - iterator = array.values.begin(); - while (iterator != array.values.end()) + for (auto &&each : array.values) { - mapbox::util::apply_visitor(XMLToArrayRenderer(out), *iterator); - ++iterator; + mapbox::util::apply_visitor(XMLToArrayRenderer(out), each); } } From 5094bad838f11d5dbd08b443d552bf471f82a75e Mon Sep 17 00:00:00 2001 From: bergwerkgis Date: Tue, 15 Sep 2015 12:23:25 +0000 Subject: [PATCH 121/122] kick off AppVeyor to test new binary Windows deps package, refs #1628 From 3279cbac246de65b8c753a0cf0ee0dd70973bc1b Mon Sep 17 00:00:00 2001 From: "Daniel J. Hofmann" Date: Tue, 15 Sep 2015 17:59:39 +0200 Subject: [PATCH 122/122] Extend compressed output lifetime till the async write function finishes. This extends the compressed output vector's lifetime, as we issue an asynchronous write operation that only receives a non-owning buffer to the compressed data. When the compressed output vector then goes out of scope, its destructor is called and the data gets (potentially) destroyed. If the asynchronous write happens afterwards, it's accessing data that is no longer there. This is the reason for race conditions --- well, for undefined behavior in general, but it manifests in the routed _sometimes_ not responding at all. The fix works like this: keep the compressed output associated with a connection. Connections inherit from `std::enable_shared_from_this` and issues a `shared_from_this()` call, passing a `std::shared_ptr` to the asynchronous write function, thus extending their lifetime. Connecitons thus manage their lifetime by themselves, extending it when needed (and of course via the `std::shared_pointers` pointing to it). Buffer's non owning property, from the `async_write` documentation: > One or more buffers containing the data to be written. Although > the buffers object may be copied as necessary, ownership of the > underlying memory blocks is retained by the caller, which must > guarantee that they remain valid until the handler is called. Reference: - http://www.boost.org/doc/libs/1_59_0/doc/html/boost_asio/reference/async_write/overload1.html --- server/connection.cpp | 1 - server/connection.hpp | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/server/connection.cpp b/server/connection.cpp index 41b653e824a..dbf85fd2a77 100644 --- a/server/connection.cpp +++ b/server/connection.cpp @@ -78,7 +78,6 @@ void Connection::handle_read(const boost::system::error_code &error, std::size_t request_handler.handle_request(current_request, current_reply); // Header compression_header; - std::vector compressed_output; std::vector output_buffer; // compress the result w/ gzip/deflate if requested diff --git a/server/connection.hpp b/server/connection.hpp index 9228a18a007..aa6fd0544bf 100644 --- a/server/connection.hpp +++ b/server/connection.hpp @@ -87,6 +87,7 @@ class Connection : public std::enable_shared_from_this boost::array incoming_data_buffer; request current_request; reply current_reply; + std::vector compressed_output; }; } // namespace http