diff --git a/data_structures/restriction.hpp b/data_structures/restriction.hpp index b808d3070b0..a59fb1e26aa 100644 --- a/data_structures/restriction.hpp +++ b/data_structures/restriction.hpp @@ -77,11 +77,15 @@ struct TurnRestriction } }; +/** + * This is just a wrapper around TurnRestriction used in the extractor. + * + * Could be merged with TurnRestriction. For now the type-destiction makes sense + * as the format in which the restriction is presented in the extractor and in the + * preprocessing is different. (see restriction_parser.cpp) + */ struct InputRestrictionContainer { - // EdgeID fromWay; - // EdgeID toWay; - // NodeID via_node; TurnRestriction restriction; InputRestrictionContainer(EdgeID fromWay, EdgeID toWay, EdgeID vw) diff --git a/extractor/extraction_containers.cpp b/extractor/extraction_containers.cpp index 8c484eb8146..7c912f33f3e 100644 --- a/extractor/extraction_containers.cpp +++ b/extractor/extraction_containers.cpp @@ -62,6 +62,17 @@ ExtractionContainers::~ExtractionContainers() way_start_end_id_list.clear(); } +/** + * Processes the collected data and serializes it. + * At this point nodes are still referenced by their OSM id. + * + * - map start-end nodes of ways to ways used int restrictions to compute compressed + * trippe representation + * - filter nodes list to nodes that are referenced by ways + * - merge edges with nodes to include location of start/end points and serialize + * + * FIXME: Each of this step should be an own function for readability. + */ void ExtractionContainers::PrepareData(const std::string &output_file_name, const std::string &restrictions_file_name) { @@ -301,6 +312,7 @@ void ExtractionContainers::PrepareData(const std::string &output_file_name, node_iterator = all_nodes_list.begin(); edge_iterator = all_edges_list.begin(); + // Also serializes the edges while (edge_iterator != all_edges_list.end() && node_iterator != all_nodes_list.end()) { if (edge_iterator->target < node_iterator->node_id) @@ -329,6 +341,8 @@ void ExtractionContainers::PrepareData(const std::string &output_file_name, int integer_weight = std::max( 1, (int)std::floor( (edge_iterator->is_duration_set ? edge_iterator->speed : weight) + .5)); + // FIXME: This means we have a _minimum_ edge length of 1m + // maybe use dm as base unit? const int integer_distance = std::max(1, (int)distance); const short zero = 0; const short one = 1; diff --git a/extractor/extraction_containers.hpp b/extractor/extraction_containers.hpp index 12d88a26c89..d04f595496e 100644 --- a/extractor/extraction_containers.hpp +++ b/extractor/extraction_containers.hpp @@ -36,6 +36,10 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include +/** + * Uses external memory containers from stxxl to store all the data that + * is collected by the extractor callbacks. + */ class ExtractionContainers { #ifndef _MSC_VER diff --git a/extractor/extraction_way.hpp b/extractor/extraction_way.hpp index d47de20b086..d344e366529 100644 --- a/extractor/extraction_way.hpp +++ b/extractor/extraction_way.hpp @@ -34,6 +34,12 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include #include +/** + * This struct is the direct result of the call to ```way_function``` + * in the lua based profile. + * + * It is split into multiple edge segments in the ExtractorCallback. + */ struct ExtractionWay { ExtractionWay() { clear(); } diff --git a/extractor/extractor.cpp b/extractor/extractor.cpp index 0581e4e5cd1..fce0f889844 100644 --- a/extractor/extractor.cpp +++ b/extractor/extractor.cpp @@ -64,6 +64,24 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include #include +/** + * TODO: Refactor this function into smaller functions for better readability. + * + * This function is the entry point for the whole extraction process. The goal of the extraction + * step is to filter and convert the OSM geometry to something more fitting for routing. + * That includes: + * - extracting turn restrictions + * - splitting ways into (directional!) edge segments + * - checking if nodes are barriers or traffic signal + * - discarding all tag information: All relevant type information for nodes/ways + * is extracted at this point. + * + * The result of this process are the following files: + * .names : Names of all streets, stored as long consecutive string with prefix sum based index + * .osrm : Nodes and edges in a intermediate format that easy to digest for osrm-prepare + * .restrictions : Turn restrictions that are used my osrm-prepare to construct the edge-expanded graph + * + */ int extractor::run(const ExtractorConfig &extractor_config) { try @@ -83,12 +101,8 @@ int extractor::run(const ExtractorConfig &extractor_config) // setup scripting environment ScriptingEnvironment scripting_environment(extractor_config.profile_path.string().c_str()); - std::unordered_map string_map; - string_map[""] = 0; - ExtractionContainers extraction_containers; - auto extractor_callbacks = - osrm::make_unique(extraction_containers, string_map); + auto extractor_callbacks = osrm::make_unique(extraction_containers); const osmium::io::File input_file(extractor_config.input_path.string()); osmium::io::Reader reader(input_file); diff --git a/extractor/extractor_callbacks.cpp b/extractor/extractor_callbacks.cpp index 224468b060e..deef831c3e9 100644 --- a/extractor/extractor_callbacks.cpp +++ b/extractor/extractor_callbacks.cpp @@ -41,13 +41,18 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include #include -ExtractorCallbacks::ExtractorCallbacks(ExtractionContainers &extraction_containers, - std::unordered_map &string_map) - : string_map(string_map), external_memory(extraction_containers) +ExtractorCallbacks::ExtractorCallbacks(ExtractionContainers &extraction_containers) + : external_memory(extraction_containers) { + string_map[""] = 0; } -/** warning: caller needs to take care of synchronization! */ +/** + * Takes the node position from osmium and the filtered properties from the lua + * profile and saves them to external memory. + * + * warning: caller needs to take care of synchronization! + */ void ExtractorCallbacks::ProcessNode(const osmium::Node &input_node, const ExtractionNode &result_node) { @@ -72,7 +77,15 @@ void ExtractorCallbacks::ProcessRestriction( // "y" : "n"); } } -/** warning: caller needs to take care of synchronization! */ +/** + * Takes the geometry contained in the ```input_way``` and the tags computed + * by the lua profile inside ```parsed_way``` and computes all edge segments. + * + * Depending on the forward/backwards weights the edges are split into forward + * and backward edges. + * + * warning: caller needs to take care of synchronization! + */ void ExtractorCallbacks::ProcessWay(const osmium::Way &input_way, const ExtractionWay &parsed_way) { if (((0 >= parsed_way.forward_speed) || diff --git a/extractor/extractor_callbacks.hpp b/extractor/extractor_callbacks.hpp index 8eab0182b1a..25bbbaba14f 100644 --- a/extractor/extractor_callbacks.hpp +++ b/extractor/extractor_callbacks.hpp @@ -43,17 +43,24 @@ class ExtractionContainers; struct InputRestrictionContainer; struct ExtractionNode; +/** + * This class is uses by the extractor with the results of the + * osmium based parsing and the customization through the lua profile. + * + * It mediates between the multi-threaded extraction process and the external memory containers. + * Thus the synchronization is handled inside of the extractor. + */ class ExtractorCallbacks { private: - std::unordered_map &string_map; + // used to deduplicate street names: actually maps to name ids + std::unordered_map string_map; ExtractionContainers &external_memory; public: ExtractorCallbacks() = delete; ExtractorCallbacks(const ExtractorCallbacks &) = delete; - explicit ExtractorCallbacks(ExtractionContainers &extraction_containers, - std::unordered_map &string_map); + explicit ExtractorCallbacks(ExtractionContainers &extraction_containers); // warning: caller needs to take care of synchronization! void ProcessNode(const osmium::Node ¤t_node, const ExtractionNode &result_node); diff --git a/extractor/restriction_parser.cpp b/extractor/restriction_parser.cpp index ea9cad27a23..77dae9ce3a9 100644 --- a/extractor/restriction_parser.cpp +++ b/extractor/restriction_parser.cpp @@ -27,7 +27,6 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "restriction_parser.hpp" #include "extraction_way.hpp" -#include "scripting_environment.hpp" #include "../data_structures/external_memory_node.hpp" #include "../util/lua_util.hpp" @@ -53,7 +52,7 @@ int lua_error_callback(lua_State *lua_state) } RestrictionParser::RestrictionParser(lua_State *lua_state) - : /*lua_state(scripting_environment.getLuaState()),*/ use_turn_restrictions(true) + : use_turn_restrictions(true) { ReadUseRestrictionsSetting(lua_state); @@ -103,6 +102,13 @@ void RestrictionParser::ReadRestrictionExceptions(lua_State *lua_state) } } +/** + * Tries to parse an relation as turn restriction. This can fail for a number of + * reasons, this the return type is a mapbox::util::optional<>. + * + * Some restrictions can also be ignored: See the ```get_exceptions``` function + * in the corresponding profile. + */ mapbox::util::optional RestrictionParser::TryParse(const osmium::Relation &relation) const { diff --git a/extractor/restriction_parser.hpp b/extractor/restriction_parser.hpp index 0a632d83ec6..f99335d2f48 100644 --- a/extractor/restriction_parser.hpp +++ b/extractor/restriction_parser.hpp @@ -41,10 +41,27 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. struct lua_State; class ScriptingEnvironment; +/** + * Parses the relations that represents turn restrictions. + * + * Currently only restrictions where the via objects is a node are supported. + * from via to + * ------->(x)--------> + * + * While this class does not directly invoke any lua code _per relation_ it does + * load configuration values from the profile, that are saved in variables. + * Namely ```use_turn_restrictions``` and ```get_exceptions```. + * + * The restriction is represented by the osm id of the from way, the osm id of the + * to way and the osm id of the via node. This representation must be post-processed + * in the extractor to work with the edge-based data-model of OSRM: + * Since the from and to way share the via-node a turn will have the following form: + * ...----(a)-----(via)------(b)----... + * So it can be represented by the tripe (a, via, b). + */ class RestrictionParser { public: - // RestrictionParser(ScriptingEnvironment &scripting_environment); RestrictionParser(lua_State *lua_state); mapbox::util::optional TryParse(const osmium::Relation &relation) const; @@ -54,7 +71,6 @@ class RestrictionParser void ReadRestrictionExceptions(lua_State *lua_state); bool ShouldIgnoreRestriction(const std::string &except_tag_string) const; - // lua_State *lua_state; std::vector restriction_exceptions; bool use_turn_restrictions; }; diff --git a/extractor/scripting_environment.hpp b/extractor/scripting_environment.hpp index be05103c8ca..8722aee8fa5 100644 --- a/extractor/scripting_environment.hpp +++ b/extractor/scripting_environment.hpp @@ -35,6 +35,13 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. struct lua_State; +/** + * Creates a lua context and binds osmium way, node and relation objects and + * ExtractionWay and ExtractionNode to lua objects. + * + * Each thread has its own lua state which is implemented with thread specific + * storage from TBB. + */ class ScriptingEnvironment { public: diff --git a/include/osrm/libosrm_config.hpp b/include/osrm/libosrm_config.hpp index 500abf5bd66..67caa399737 100644 --- a/include/osrm/libosrm_config.hpp +++ b/include/osrm/libosrm_config.hpp @@ -35,13 +35,13 @@ struct libosrm_config libosrm_config(const libosrm_config &) = delete; libosrm_config() : max_locations_distance_table(100), max_locations_map_matching(-1), - use_shared_memory(false) + use_shared_memory(true) { } - libosrm_config(const ServerPaths &paths, const bool flag, const int max_table, const int max_matching) + libosrm_config(const ServerPaths &paths, const bool sharedmemory_flag, const int max_table, const int max_matching) : server_paths(paths), max_locations_distance_table(max_table), - max_locations_map_matching(max_matching), use_shared_memory(flag) + max_locations_map_matching(max_matching), use_shared_memory(sharedmemory_flag) { } diff --git a/routed.cpp b/routed.cpp index 8b0e2e88bfe..294efa8f180 100644 --- a/routed.cpp +++ b/routed.cpp @@ -74,6 +74,8 @@ int main(int argc, const char *argv[]) int ip_port, requested_thread_num; libosrm_config lib_config; + // make the behaviour of routed backward compatible + lib_config.use_shared_memory = false; const unsigned init_result = GenerateServerProgramOptions( argc, argv, lib_config.server_paths, ip_address, ip_port, requested_thread_num,