From 7b39ca8fa2c33cb829c5d90885be9106f0387b25 Mon Sep 17 00:00:00 2001 From: Sarah Hoffmann Date: Fri, 13 Jan 2017 17:50:21 +0100 Subject: [PATCH 01/30] improve constness of parsed osmium objects Relations and nodes can be const, as their buffer is read only. Ways need to be writable, so we can add node locations directly in the object. Hand them around as pointers to make it clear they are mutable. --- osmdata.cpp | 13 ++++++------- osmdata.hpp | 4 ++-- output-gazetteer.cpp | 10 +++++----- output-gazetteer.hpp | 6 +++--- output-multi.cpp | 18 +++++++++--------- output-multi.hpp | 6 +++--- output-null.cpp | 4 ++-- output-null.hpp | 4 ++-- output-pgsql.cpp | 14 +++++++------- output-pgsql.hpp | 4 ++-- output.hpp | 4 ++-- parse-osmium.cpp | 8 ++++---- parse-osmium.hpp | 4 ++-- tests/test-parse-diff.cpp | 6 +++--- tests/test-parse-xml2.cpp | 10 +++++----- 15 files changed, 57 insertions(+), 58 deletions(-) diff --git a/osmdata.cpp b/osmdata.cpp index b713e2c8c..d9e03f2f6 100644 --- a/osmdata.cpp +++ b/osmdata.cpp @@ -45,9 +45,9 @@ int osmdata_t::node_add(osmium::Node const &node) return status; } -int osmdata_t::way_add(osmium::Way const &way) +int osmdata_t::way_add(osmium::Way *way) { - mid->ways_set(way); + mid->ways_set(*way); int status = 0; for (auto& out: outs) { @@ -84,20 +84,19 @@ int osmdata_t::node_modify(osmium::Node const &node) return status; } -int osmdata_t::way_modify(osmium::Way const &way) +int osmdata_t::way_modify(osmium::Way *way) { - idlist_t nodes(way.nodes()); slim_middle_t *slim = dynamic_cast(mid.get()); - slim->ways_delete(way.id()); - slim->ways_set(way); + slim->ways_delete(way->id()); + slim->ways_set(*way); int status = 0; for (auto& out: outs) { status |= out->way_modify(way); } - slim->way_changed(way.id()); + slim->way_changed(way->id()); return status; } diff --git a/osmdata.hpp b/osmdata.hpp index 945ca626e..8759262d9 100644 --- a/osmdata.hpp +++ b/osmdata.hpp @@ -28,11 +28,11 @@ class osmdata_t { void stop(); int node_add(osmium::Node const &node); - int way_add(osmium::Way const &way); + int way_add(osmium::Way *way); int relation_add(osmium::Relation const &rel); int node_modify(osmium::Node const &node); - int way_modify(osmium::Way const &way); + int way_modify(osmium::Way *way); int relation_modify(osmium::Relation const &rel); int node_delete(osmid_t id); diff --git a/output-gazetteer.cpp b/output-gazetteer.cpp index 8125d6013..3715a77bc 100644 --- a/output-gazetteer.cpp +++ b/output-gazetteer.cpp @@ -684,23 +684,23 @@ int output_gazetteer_t::process_node(osmium::Node const &node) return 0; } -int output_gazetteer_t::process_way(osmium::Way const &way) +int output_gazetteer_t::process_way(osmium::Way *way) { - places.process_tags(way); + places.process_tags(*way); if (m_options.append) - delete_unused_classes('W', way.id()); + delete_unused_classes('W', way->id()); /* Are we interested in this item? */ if (places.has_data()) { /* Fetch the node details */ nodelist_t nodes; - m_mid->nodes_get_list(nodes, way.nodes(), reproj.get()); + m_mid->nodes_get_list(nodes, way->nodes(), reproj.get()); /* Get the geometry of the object */ auto geom = builder.get_wkb_simple(nodes, 1); if (geom.valid()) { - places.copy_out(way, geom.geom, buffer); + places.copy_out(*way, geom.geom, buffer); flush_place_buffer(); } } diff --git a/output-gazetteer.hpp b/output-gazetteer.hpp index e90a08f5d..e59fb8abc 100644 --- a/output-gazetteer.hpp +++ b/output-gazetteer.hpp @@ -184,7 +184,7 @@ class output_gazetteer_t : public output_t { return process_node(node); } - int way_add(osmium::Way const &way) override + int way_add(osmium::Way *way) override { return process_way(way); } @@ -199,7 +199,7 @@ class output_gazetteer_t : public output_t { return process_node(node); } - int way_modify(osmium::Way const &way) override + int way_modify(osmium::Way *way) override { return process_way(way); } @@ -234,7 +234,7 @@ class output_gazetteer_t : public output_t { void delete_unused_classes(char osm_type, osmid_t osm_id); void delete_place(char osm_type, osmid_t osm_id); int process_node(osmium::Node const &node); - int process_way(osmium::Way const &way); + int process_way(osmium::Way *way); int process_relation(osmium::Relation const &rel); int connect(); diff --git a/output-multi.cpp b/output-multi.cpp index 6df394c36..649e113e3 100644 --- a/output-multi.cpp +++ b/output-multi.cpp @@ -194,8 +194,8 @@ int output_multi_t::node_add(osmium::Node const &node) return 0; } -int output_multi_t::way_add(osmium::Way const &way) { - if (m_processor->interests(geometry_processor::interest_way) && way.nodes().size() > 1) { +int output_multi_t::way_add(osmium::Way *way) { + if (m_processor->interests(geometry_processor::interest_way) && way->nodes().size() > 1) { return process_way(way); } return 0; @@ -224,10 +224,10 @@ int output_multi_t::node_modify(osmium::Node const &node) return 0; } -int output_multi_t::way_modify(osmium::Way const &way) { +int output_multi_t::way_modify(osmium::Way *way) { if (m_processor->interests(geometry_processor::interest_way)) { // TODO - need to know it's a way? - delete_from_output(way.id()); + delete_from_output(way->id()); // TODO: need to mark any relations using it - depends on what // type of output this is... delegate to the geometry processor?? @@ -322,15 +322,15 @@ int output_multi_t::reprocess_way(osmium::Way const &way, bool exists) return 0; } -int output_multi_t::process_way(osmium::Way const &way) { +int output_multi_t::process_way(osmium::Way *way) { //check if we are keeping this way int polygon = 0, roads = 0; taglist_t outtags; - auto filter = m_tagtransform->filter_tags(way, &polygon, &roads, + auto filter = m_tagtransform->filter_tags(*way, &polygon, &roads, *m_export_list.get(), outtags, true); if (!filter) { //get the geom from the middle - if (m_way_helper.set(way.nodes(), m_mid, m_proj.get()) < 1) + if (m_way_helper.set(way->nodes(), m_mid, m_proj.get()) < 1) return 0; //grab its geom auto geom = m_processor->process_way(m_way_helper.node_cache); @@ -339,11 +339,11 @@ int output_multi_t::process_way(osmium::Way const &way) { //if we are also interested in relations we need to mark //this way pending just in case it shows up in one if (m_processor->interests(geometry_processor::interest_relation)) { - ways_pending_tracker.mark(way.id()); + ways_pending_tracker.mark(way->id()); } else { // We wouldn't be interested in this as a relation, so no need to mark it pending. // TODO: Does this imply anything for non-multipolygon relations? - copy_to_table(way.id(), geom, outtags, polygon); + copy_to_table(way->id(), geom, outtags, polygon); } } } diff --git a/output-multi.hpp b/output-multi.hpp index 4e093959b..3ac13cbf0 100644 --- a/output-multi.hpp +++ b/output-multi.hpp @@ -45,11 +45,11 @@ class output_multi_t : public output_t { int pending_relation(osmid_t id, int exists) override; int node_add(osmium::Node const &node) override; - int way_add(osmium::Way const &way) override; + int way_add(osmium::Way *way) override; int relation_add(osmium::Relation const &rel) override; int node_modify(osmium::Node const &node) override; - int way_modify(osmium::Way const &way) override; + int way_modify(osmium::Way *way) override; int relation_modify(osmium::Relation const &rel) override; int node_delete(osmid_t id) override; @@ -65,7 +65,7 @@ class output_multi_t : public output_t { void delete_from_output(osmid_t id); int process_node(osmium::Node const &node); - int process_way(osmium::Way const &way); + int process_way(osmium::Way *way); int reprocess_way(osmium::Way const &way, bool exists); int process_relation(osmium::Relation const &rel, bool exists, bool pending=false); void copy_node_to_table(osmid_t id, const std::string &geom, taglist_t &tags); diff --git a/output-null.cpp b/output-null.cpp index e29b6fad7..81d571537 100644 --- a/output-null.cpp +++ b/output-null.cpp @@ -33,7 +33,7 @@ int output_null_t::pending_relation(osmid_t, int) { int output_null_t::node_add(osmium::Node const &) { return 0; } -int output_null_t::way_add(osmium::Way const &) { +int output_null_t::way_add(osmium::Way *) { return 0; } @@ -55,7 +55,7 @@ int output_null_t::relation_delete(osmid_t) { int output_null_t::node_modify(osmium::Node const &) { return 0; } -int output_null_t::way_modify(osmium::Way const &) { +int output_null_t::way_modify(osmium::Way *) { return 0; } diff --git a/output-null.hpp b/output-null.hpp index d56eaf2d9..14f5eb68c 100644 --- a/output-null.hpp +++ b/output-null.hpp @@ -26,11 +26,11 @@ class output_null_t : public output_t { int pending_relation(osmid_t id, int exists) override; int node_add(osmium::Node const &node) override; - int way_add(osmium::Way const &way) override; + int way_add(osmium::Way *way) override; int relation_add(osmium::Relation const &rel) override; int node_modify(osmium::Node const &node) override; - int way_modify(osmium::Way const &way) override; + int way_modify(osmium::Way *way) override; int relation_modify(osmium::Relation const &rel) override; int node_delete(osmid_t id) override; diff --git a/output-pgsql.cpp b/output-pgsql.cpp index 77e69c921..99a2155c3 100644 --- a/output-pgsql.cpp +++ b/output-pgsql.cpp @@ -362,26 +362,26 @@ int output_pgsql_t::node_add(osmium::Node const &node) return 0; } -int output_pgsql_t::way_add(osmium::Way const &way) +int output_pgsql_t::way_add(osmium::Way *way) { int polygon = 0; int roads = 0; taglist_t outtags; /* Check whether the way is: (1) Exportable, (2) Maybe a polygon */ - auto filter = m_tagtransform->filter_tags(way, &polygon, &roads, + auto filter = m_tagtransform->filter_tags(*way, &polygon, &roads, *m_export_list.get(), outtags); /* If this isn't a polygon then it can not be part of a multipolygon Hence only polygons are "pending" */ - if (!filter && polygon) { ways_pending_tracker.mark(way.id()); } + if (!filter && polygon) { ways_pending_tracker.mark(way->id()); } if( !polygon && !filter ) { /* Get actual node data and generate output */ nodelist_t nodes; - m_mid->nodes_get_list(nodes, way.nodes(), reproj.get()); - pgsql_out_way(way.id(), outtags, nodes, polygon, roads); + m_mid->nodes_get_list(nodes, way->nodes(), reproj.get()); + pgsql_out_way(way->id(), outtags, nodes, polygon, roads); } return 0; } @@ -545,14 +545,14 @@ int output_pgsql_t::node_modify(osmium::Node const &node) return 0; } -int output_pgsql_t::way_modify(osmium::Way const &way) +int output_pgsql_t::way_modify(osmium::Way *way) { if( !m_options.slim ) { fprintf( stderr, "Cannot apply diffs unless in slim mode\n" ); util::exit_nicely(); } - way_delete(way.id()); + way_delete(way->id()); way_add(way); return 0; diff --git a/output-pgsql.hpp b/output-pgsql.hpp index d80e991ae..e1145db0f 100644 --- a/output-pgsql.hpp +++ b/output-pgsql.hpp @@ -40,11 +40,11 @@ class output_pgsql_t : public output_t { int pending_relation(osmid_t id, int exists) override; int node_add(osmium::Node const &node) override; - int way_add(osmium::Way const &way) override; + int way_add(osmium::Way *way) override; int relation_add(osmium::Relation const &rel) override; int node_modify(osmium::Node const &node) override; - int way_modify(osmium::Way const &way) override; + int way_modify(osmium::Way *way) override; int relation_modify(osmium::Relation const &rel) override; int node_delete(osmid_t id) override; diff --git a/output.hpp b/output.hpp index 681fe7e7b..19c02f7c3 100644 --- a/output.hpp +++ b/output.hpp @@ -50,11 +50,11 @@ class output_t : public boost::noncopyable { virtual int pending_relation(osmid_t id, int exists) = 0; virtual int node_add(osmium::Node const &node) = 0; - virtual int way_add(osmium::Way const &way) = 0; + virtual int way_add(osmium::Way *way) = 0; virtual int relation_add(osmium::Relation const &rel) = 0; virtual int node_modify(osmium::Node const &node) = 0; - virtual int way_modify(osmium::Way const &way) = 0; + virtual int way_modify(osmium::Way *way) = 0; virtual int relation_modify(osmium::Relation const &rel) = 0; virtual int node_delete(osmid_t id) = 0; diff --git a/parse-osmium.cpp b/parse-osmium.cpp index 46d1be579..b404f3185 100644 --- a/parse-osmium.cpp +++ b/parse-osmium.cpp @@ -129,7 +129,7 @@ void parse_osmium_t::stream_file(const std::string &filename, const std::string reader.close(); } -void parse_osmium_t::node(osmium::Node& node) +void parse_osmium_t::node(osmium::Node const &node) { if (node.deleted()) { m_data->node_delete(node.id()); @@ -163,15 +163,15 @@ void parse_osmium_t::way(osmium::Way& way) m_data->way_delete(way.id()); } else { if (m_append) { - m_data->way_modify(way); + m_data->way_modify(&way); } else { - m_data->way_add(way); + m_data->way_add(&way); } } m_stats.add_way(way.id()); } -void parse_osmium_t::relation(osmium::Relation& rel) +void parse_osmium_t::relation(osmium::Relation const &rel) { if (rel.deleted()) { m_data->relation_delete(rel.id()); diff --git a/parse-osmium.hpp b/parse-osmium.hpp index be1bc3af6..4b1e23aea 100644 --- a/parse-osmium.hpp +++ b/parse-osmium.hpp @@ -113,9 +113,9 @@ class parse_osmium_t: public osmium::handler::Handler void stream_file(const std::string &filename, const std::string &fmt); - void node(osmium::Node& node); + void node(osmium::Node const &node); void way(osmium::Way& way); - void relation(osmium::Relation& rel); + void relation(osmium::Relation const &rel); parse_stats_t const &stats() const { diff --git a/tests/test-parse-diff.cpp b/tests/test-parse-diff.cpp index 9d751c9b6..c3d98a43e 100644 --- a/tests/test-parse-diff.cpp +++ b/tests/test-parse-diff.cpp @@ -48,8 +48,8 @@ struct test_output_t : public dummy_output_t { return 0; } - int way_add(osmium::Way const &w) override { - assert(w.id() > 0); + int way_add(osmium::Way *w) override { + assert(w->id() > 0); ++way.added; return 0; } @@ -65,7 +65,7 @@ struct test_output_t : public dummy_output_t { ++node.modified; return 0; } - int way_modify(osmium::Way const &) override { + int way_modify(osmium::Way *) override { ++way.modified; return 0; } diff --git a/tests/test-parse-xml2.cpp b/tests/test-parse-xml2.cpp index 70862375b..e9db2ac96 100644 --- a/tests/test-parse-xml2.cpp +++ b/tests/test-parse-xml2.cpp @@ -50,12 +50,12 @@ struct test_output_t : public output_null_t { return 0; } - int way_add(osmium::Way const &way) override { - assert(way.id() > 0); - sum_ids += (unsigned) way.id(); + int way_add(osmium::Way *way) override { + assert(way->id() > 0); + sum_ids += (unsigned) way->id(); num_ways += 1; - assert(way.nodes().size() >= 0); - num_nds += uint64_t(way.nodes().size()); + assert(way->nodes().size() >= 0); + num_nds += uint64_t(way->nodes().size()); return 0; } From 33f4cc70fb8f20588e4b6239703458552838f1dc Mon Sep 17 00:00:00 2001 From: Sarah Hoffmann Date: Fri, 13 Jan 2017 19:03:40 +0100 Subject: [PATCH 02/30] add node retrival functions without projection --- middle-pgsql.cpp | 57 +++++++++++++++++++++++++++++++++++++++ middle-pgsql.hpp | 2 ++ middle-ram.cpp | 15 +++++++++++ middle-ram.hpp | 1 + middle.hpp | 11 ++++++++ node-persistent-cache.cpp | 23 ++++++++++++++++ node-persistent-cache.hpp | 1 + tests/mockups.hpp | 2 ++ 8 files changed, 112 insertions(+) diff --git a/middle-pgsql.cpp b/middle-pgsql.cpp index f3f3731d0..c8766ce04 100644 --- a/middle-pgsql.cpp +++ b/middle-pgsql.cpp @@ -372,6 +372,56 @@ size_t middle_pgsql_t::local_nodes_get_list(nodelist_t &out, return wrtidx; } +size_t middle_pgsql_t::local_nodes_get_list(osmium::WayNodeList *nodes) const +{ + size_t count = 0; + std::string buffer("{"); + std::unordered_map node_pos; + + // get nodes where possible from cache, + // at the same time build a list for querying missing nodes from DB + size_t pos = 0; + for (auto &n : *nodes) { + auto loc = cache->get(n.ref()); + if (loc.valid()) { + n.set_location(loc); + ++count; + } else { + node_pos.emplace(n.ref(), pos); + buffer += std::to_string(n.ref()); + buffer += ','; + } + ++pos; + } + + if (node_pos.empty()) { + return count; // all ids found in cache, nothing more to do + } + + // get any remaining nodes from the DB + buffer[buffer.size() - 1] = '}'; + + pgsql_endCopy(node_table); + + PGconn *sql_conn = node_table->sql_conn; + + char const *paramValues[1]; + paramValues[0] = buffer.c_str(); + PGresult *res = pgsql_execPrepared(sql_conn, "get_node_list", 1, + paramValues, PGRES_TUPLES_OK); + auto countPG = PQntuples(res); + + for (int i = 0; i < countPG; ++i) { + auto &npos = node_pos.at(strtoosmid(PQgetvalue(res, i, 0), nullptr, 10)); + + osmium::Location loc((int)strtol(PQgetvalue(res, i, 2), nullptr, 10), + (int)strtol(PQgetvalue(res, i, 1), nullptr, 10)); + (*nodes)[npos].set_location(loc); + } + + return count + (size_t) countPG; +} + void middle_pgsql_t::nodes_set(osmium::Node const &node) { cache->set(node.id(), node.location()); @@ -392,6 +442,13 @@ size_t middle_pgsql_t::nodes_get_list(nodelist_t &out, : local_nodes_get_list(out, nds, proj); } +size_t middle_pgsql_t::nodes_get_list(osmium::WayNodeList *nodes) const +{ + return (out_options->flat_node_cache_enabled) + ? persistent_cache->get_list(nodes) + : local_nodes_get_list(nodes); +} + void middle_pgsql_t::local_nodes_delete(osmid_t osm_id) { char const *paramValues[1]; diff --git a/middle-pgsql.hpp b/middle-pgsql.hpp index d9f63e29b..13f661447 100644 --- a/middle-pgsql.hpp +++ b/middle-pgsql.hpp @@ -29,6 +29,7 @@ struct middle_pgsql_t : public slim_middle_t { void nodes_set(osmium::Node const &node) override; size_t nodes_get_list(nodelist_t &out, osmium::WayNodeList const &nds, reprojection const *proj) const override; + size_t nodes_get_list(osmium::WayNodeList *nodes) const override; void nodes_delete(osmid_t id) override; void node_changed(osmid_t id) override; @@ -90,6 +91,7 @@ struct middle_pgsql_t : public slim_middle_t { void local_nodes_set(osmium::Node const &node); size_t local_nodes_get_list(nodelist_t &out, osmium::WayNodeList const &nds, reprojection const *proj) const; + size_t local_nodes_get_list(osmium::WayNodeList *nodes) const; void local_nodes_delete(osmid_t osm_id); std::vector tables; diff --git a/middle-ram.cpp b/middle-ram.cpp index 8d593915b..8ee5dbbd1 100644 --- a/middle-ram.cpp +++ b/middle-ram.cpp @@ -62,6 +62,21 @@ size_t middle_ram_t::nodes_get_list(nodelist_t &out, return out.size(); } +size_t middle_ram_t::nodes_get_list(osmium::WayNodeList *nodes) const +{ + size_t count = 0; + + for (auto &n : *nodes) { + auto loc = cache->get(n.ref()); + n.set_location(loc); + if (loc.valid()) { + ++count; + } + } + + return count; +} + void middle_ram_t::iterate_relations(pending_processor& pf) { //TODO: just dont do anything diff --git a/middle-ram.hpp b/middle-ram.hpp index ca9e58296..8cca7ded9 100644 --- a/middle-ram.hpp +++ b/middle-ram.hpp @@ -93,6 +93,7 @@ struct middle_ram_t : public middle_t { void nodes_set(osmium::Node const &node) override; size_t nodes_get_list(nodelist_t &out, osmium::WayNodeList const &nds, reprojection const *proj) const override; + size_t nodes_get_list(osmium::WayNodeList *nodes) const override; int nodes_delete(osmid_t id); int node_changed(osmid_t id); diff --git a/middle.hpp b/middle.hpp index 7b5f76ffa..e757e6736 100644 --- a/middle.hpp +++ b/middle.hpp @@ -23,10 +23,21 @@ struct options_t; struct middle_query_t { virtual ~middle_query_t() {} + /** + * Retrives node locations for the given node list and + * reprojects them. + */ virtual size_t nodes_get_list(nodelist_t &out, osmium::WayNodeList const &nds, reprojection const *proj) const = 0; + /** + * Retrives node locations for the given node list. + * + * The locations are saved directly in the input list. + */ + virtual size_t nodes_get_list(osmium::WayNodeList *nodes) const = 0; + /** * Retrives a single way from the ways storage * and stores it in the given osmium buffer. diff --git a/node-persistent-cache.cpp b/node-persistent-cache.cpp index 8d962646c..3659f5b27 100644 --- a/node-persistent-cache.cpp +++ b/node-persistent-cache.cpp @@ -51,6 +51,29 @@ size_t node_persistent_cache::get_list(nodelist_t &out, return out.size(); } +size_t node_persistent_cache::get_list(osmium::WayNodeList *nodes) +{ + size_t count = 0; + + for (auto &n : *nodes) { + auto loc = m_ram_cache->get(n.ref()); + /* Check cache first */ + if (!loc.valid() && n.ref() >= 0) { + try { + loc = m_index->get( + static_cast(n.ref())); + } catch (osmium::not_found const &) { + } + } + n.set_location(loc); + if (loc.valid()) { + ++count; + } + } + + return count; +} + node_persistent_cache::node_persistent_cache( const options_t *options, std::shared_ptr ptr) : m_ram_cache(ptr), m_fd(-1) diff --git a/node-persistent-cache.hpp b/node-persistent-cache.hpp index e0b10b6e4..cb4f7d551 100644 --- a/node-persistent-cache.hpp +++ b/node-persistent-cache.hpp @@ -23,6 +23,7 @@ class node_persistent_cache osmium::Location get(osmid_t id); size_t get_list(nodelist_t &out, osmium::WayNodeList const &nds, reprojection const *proj); + size_t get_list(osmium::WayNodeList *nodes); private: // Dense node cache for unsigned IDs only diff --git a/tests/mockups.hpp b/tests/mockups.hpp index 1efed9c3f..6583c7a55 100644 --- a/tests/mockups.hpp +++ b/tests/mockups.hpp @@ -20,6 +20,7 @@ struct dummy_middle_t : public middle_t { { return 0; } + size_t nodes_get_list(osmium::WayNodeList *) const override { return 0; } void ways_set(osmium::Way const &) override { } bool ways_get(osmid_t, osmium::memory::Buffer &) const override { return true; } @@ -57,6 +58,7 @@ struct dummy_slim_middle_t : public slim_middle_t { { return 0; } + size_t nodes_get_list(osmium::WayNodeList *) const override { return 0; } void ways_set(osmium::Way const &) override { } bool ways_get(osmid_t, osmium::memory::Buffer &) const override { return true; } From c8bc82f808aa3b9056eab01331099507880fc58c Mon Sep 17 00:00:00 2001 From: Sarah Hoffmann Date: Sat, 21 Jan 2017 00:28:16 +0100 Subject: [PATCH 03/30] initial version of osmium geometry building in output-pgsql --- CMakeLists.txt | 1 + contrib/libosmium/osmium/area/assembler.hpp | 37 ++ .../osmium/area/detail/segment_list.hpp | 37 +- osmium-builder.cpp | 213 ++++++++++++ osmium-builder.hpp | 43 +++ output-pgsql.cpp | 326 ++++++++---------- output-pgsql.hpp | 23 +- reprojection.hpp | 11 + table.cpp | 46 +++ table.hpp | 2 + wkb-parser.hpp | 148 ++++++++ 11 files changed, 690 insertions(+), 197 deletions(-) create mode 100644 osmium-builder.cpp create mode 100644 osmium-builder.hpp create mode 100644 wkb-parser.hpp diff --git a/CMakeLists.txt b/CMakeLists.txt index ae96f4ddd..4f003ccdb 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -177,6 +177,7 @@ set(osm2pgsql_lib_SOURCES node-ram-cache.cpp options.cpp osmdata.cpp + osmium-builder.cpp output-gazetteer.cpp output-multi.cpp output-null.cpp diff --git a/contrib/libosmium/osmium/area/assembler.hpp b/contrib/libosmium/osmium/area/assembler.hpp index 3509b57bc..7495503d5 100644 --- a/contrib/libosmium/osmium/area/assembler.hpp +++ b/contrib/libosmium/osmium/area/assembler.hpp @@ -1467,6 +1467,43 @@ namespace osmium { } } + bool make_area(const osmium::Way &way, + osmium::memory::Buffer &out_buffer) + { + m_segment_list.extract_segments_from_way( + m_config.problem_reporter, way, true); + + if (!create_rings()) { + return false; + } + + osmium::builder::AreaBuilder builder{out_buffer}; + builder.initialize_from_object(way); + add_rings_to_area(builder); + out_buffer.commit(); + return true; + } + + bool make_area(const osmium::Relation &relation, + osmium::memory::Buffer const &ways, + osmium::memory::Buffer &out_buffer) + { + for (auto const &w : ways.select()) { + m_segment_list.extract_segments_from_way( + m_config.problem_reporter, w, true); + } + + if (!create_rings()) { + return false; + } + + osmium::builder::AreaBuilder builder{out_buffer}; + builder.initialize_from_object(relation); + add_rings_to_area(builder); + out_buffer.commit(); + return true; + } + /** * Assemble an area from the given relation and its members. * All members are to be found in the in_buffer at the offsets diff --git a/contrib/libosmium/osmium/area/detail/segment_list.hpp b/contrib/libosmium/osmium/area/detail/segment_list.hpp index 64b2b4e5a..45a2a0ac0 100644 --- a/contrib/libosmium/osmium/area/detail/segment_list.hpp +++ b/contrib/libosmium/osmium/area/detail/segment_list.hpp @@ -110,11 +110,18 @@ namespace osmium { }); } - uint32_t extract_segments_from_way_impl(osmium::area::ProblemReporter* problem_reporter, const osmium::Way& way, role_type role) { + uint32_t extract_segments_from_way_impl( + osmium::area::ProblemReporter *problem_reporter, + const osmium::Way &way, role_type role, + bool ignore_invalid = false) + { uint32_t duplicate_nodes = 0; osmium::NodeRef previous_nr; for (const osmium::NodeRef& nr : way.nodes()) { + if (ignore_invalid && !nr.location().valid()) { + continue; + } if (previous_nr.location()) { if (previous_nr.location() != nr.location()) { m_segments.emplace_back(previous_nr, nr, role, &way); @@ -213,19 +220,29 @@ namespace osmium { * same node or different nodes with same location) are * removed after reporting the duplicate node. */ - uint32_t extract_segments_from_way(osmium::area::ProblemReporter* problem_reporter, const osmium::Way& way) { + uint32_t extract_segments_from_way( + osmium::area::ProblemReporter *problem_reporter, + const osmium::Way &way, bool ignore_invalid = false) + { if (way.nodes().empty()) { return 0; } m_segments.reserve(way.nodes().size() - 1); - return extract_segments_from_way_impl(problem_reporter, way, role_type::outer); + return extract_segments_from_way_impl(problem_reporter, way, + role_type::outer, + ignore_invalid); } /** * Extract all segments from all ways that make up this * multipolygon relation and add them to the list. */ - uint32_t extract_segments_from_ways(osmium::area::ProblemReporter* problem_reporter, const osmium::Relation& relation, const std::vector& members) { + uint32_t extract_segments_from_ways( + osmium::area::ProblemReporter *problem_reporter, + const osmium::Relation &relation, + const std::vector &members, + bool ignore_invalid = false) + { assert(relation.members().size() >= members.size()); const size_t num_segments = get_num_segments(members); @@ -235,9 +252,15 @@ namespace osmium { m_segments.reserve(num_segments); uint32_t duplicate_nodes = 0; - for_each_member(relation, members, [this, &problem_reporter, &duplicate_nodes](const osmium::RelationMember& member, const osmium::Way& way) { - duplicate_nodes += extract_segments_from_way_impl(problem_reporter, way, parse_role(member.role())); - }); + for_each_member( + relation, members, + [this, &problem_reporter, &duplicate_nodes, + ignore_invalid](const osmium::RelationMember &member, + const osmium::Way &way) { + duplicate_nodes += extract_segments_from_way_impl( + problem_reporter, way, + parse_role(member.role()), ignore_invalid); + }); return duplicate_nodes; } diff --git a/osmium-builder.cpp b/osmium-builder.cpp new file mode 100644 index 000000000..1e04b4723 --- /dev/null +++ b/osmium-builder.cpp @@ -0,0 +1,213 @@ +#include +#include + +#include "osmium-builder.hpp" + +namespace { + +osmium::area::AssemblerConfig area_config; + +inline double distance(osmium::geom::Coordinates p1, + osmium::geom::Coordinates p2) +{ + double dx = p1.x - p2.x; + double dy = p1.y - p2.y; + return std::sqrt(dx * dx + dy * dy); +} + +inline osmium::geom::Coordinates interpolate(osmium::geom::Coordinates p1, + osmium::geom::Coordinates p2, + double frac) +{ + return osmium::geom::Coordinates(frac * (p1.x - p2.x) + p2.x, + frac * (p1.y - p2.y) + p2.y); +} +} + +namespace geom { + +using WKBWriter = osmium::geom::detail::WKBFactoryImpl; + +osmium_builder_t::wkb_t +osmium_builder_t::get_wkb_node(osmium::Location const &loc) +{ + return m_writer.make_point(m_proj->reproject(loc)); +} + +osmium_builder_t::wkbs_t osmium_builder_t::get_wkb_split(osmium::Way const &way) +{ + wkbs_t ret; + + double const split_at = m_proj->target_latlon() ? 1 : 100 * 1000; + + double dist = 0; + osmium::geom::Coordinates prev_pt; + m_writer.linestring_start(); + size_t curlen = 0; + + for (auto const &node : way.nodes()) { + if (!node.location().valid()) + continue; + + auto const this_pt = m_proj->reproject(node.location()); + if (prev_pt.valid()) { + if (prev_pt == this_pt) { + continue; + } + double const delta = distance(prev_pt, this_pt); + + // figure out if the addition of this point would take the total + // length of the line in `segment` over the `split_at` distance. + + if (dist + delta > split_at) { + size_t const splits = + (size_t)std::floor((dist + delta) / split_at); + // use the splitting distance to split the current segment up + // into as many parts as necessary to keep each part below + // the `split_at` distance. + osmium::geom::Coordinates ipoint; + for (size_t j = 0; j < splits; ++j) { + double const frac = + ((double)(j + 1) * split_at - dist) / delta; + ipoint = interpolate(this_pt, prev_pt, frac); + m_writer.linestring_add_location(ipoint); + ret.push_back(m_writer.linestring_finish(curlen)); + // start a new segment + m_writer.linestring_start(); + m_writer.linestring_add_location(ipoint); + } + // reset the distance based on the final splitting point for + // the next iteration. + if (this_pt == ipoint) { + dist = 0; + m_writer.linestring_start(); + curlen = 0; + } else { + dist = distance(this_pt, ipoint); + curlen = 1; + } + } else { + dist += delta; + } + } + + m_writer.linestring_add_location(this_pt); + ++curlen; + + prev_pt = this_pt; + } + + if (curlen > 1) { + ret.push_back(m_writer.linestring_finish(curlen)); + } + + return ret; +} + +osmium_builder_t::wkb_t +osmium_builder_t::get_wkb_polygon(osmium::Way const &way) +{ + osmium::area::Assembler assembler{area_config}; + + m_buffer.clear(); + if (!assembler.make_area(way, m_buffer)) { + return wkb_t(); + } + + auto wkbs = create_multipolygon(m_buffer.get(0)); + + return wkbs.empty() ? wkb_t() : wkbs[0]; +} + +osmium_builder_t::wkbs_t +osmium_builder_t::get_wkb_multipolygon(osmium::Relation const &rel, + osmium::memory::Buffer const &ways) +{ + wkbs_t ret; + osmium::area::Assembler assembler{area_config}; + + m_buffer.clear(); + if (assembler.make_area(rel, ways, m_buffer)) { + ret = create_multipolygon(m_buffer.get(0)); + } + + return ret; +} + +osmium_builder_t::wkbs_t +osmium_builder_t::get_wkb_multiline(osmium::memory::Buffer const &ways, bool) +{ + wkbs_t ret; + + // XXX need to combine ways + // XXX need to do non-split version + wkbs_t linewkbs; + for (auto &w : ways.select()) { + linewkbs = get_wkb_split(w); + std::move(linewkbs.begin(), linewkbs.end(), + std::inserter(ret, ret.end())); + linewkbs.clear(); + } + + return ret; +} + +void osmium_builder_t::add_mp_points(const osmium::NodeRefList &nodes) +{ + osmium::Location last_location; + for (const osmium::NodeRef &node_ref : nodes) { + if (node_ref.location().valid() && + last_location != node_ref.location()) { + last_location = node_ref.location(); + m_writer.multipolygon_add_location( + m_proj->reproject(last_location)); + } + } +} + +osmium_builder_t::wkbs_t +osmium_builder_t::create_multipolygon(osmium::Area const &area) +{ + wkbs_t ret; + + // XXX need to split into polygons + + try { + size_t num_polygons = 0; + size_t num_rings = 0; + m_writer.multipolygon_start(); + + for (auto it = area.cbegin(); it != area.cend(); ++it) { + if (it->type() == osmium::item_type::outer_ring) { + auto &ring = static_cast(*it); + if (num_polygons > 0) { + m_writer.multipolygon_polygon_finish(); + } + m_writer.multipolygon_polygon_start(); + m_writer.multipolygon_outer_ring_start(); + add_mp_points(ring); + m_writer.multipolygon_outer_ring_finish(); + ++num_rings; + ++num_polygons; + } else if (it->type() == osmium::item_type::inner_ring) { + auto &ring = static_cast(*it); + m_writer.multipolygon_inner_ring_start(); + add_mp_points(ring); + m_writer.multipolygon_inner_ring_finish(); + ++num_rings; + } + } + + // if there are no rings, this area is invalid + if (num_rings > 0) { + m_writer.multipolygon_polygon_finish(); + ret.push_back(m_writer.multipolygon_finish()); + } + + } catch (osmium::geometry_error &e) { /* ignored */ + } + + return ret; +} + +} // name space diff --git a/osmium-builder.hpp b/osmium-builder.hpp new file mode 100644 index 000000000..93ecb3fdf --- /dev/null +++ b/osmium-builder.hpp @@ -0,0 +1,43 @@ +#include +#include +#include + +#include +#include +#include + +#include "reprojection.hpp" + +namespace geom { + +class osmium_builder_t +{ +public: + typedef std::string wkb_t; + typedef std::vector wkbs_t; + + explicit osmium_builder_t(std::shared_ptr const &proj) + : m_proj(proj), m_buffer(1024, osmium::memory::Buffer::auto_grow::yes), + m_writer(m_proj->epsg(), osmium::geom::wkb_type::ewkb) + { + } + + wkb_t get_wkb_node(osmium::Location const &loc); + wkbs_t get_wkb_split(osmium::Way const &way); + wkb_t get_wkb_polygon(osmium::Way const &way); + + wkbs_t get_wkb_multipolygon(osmium::Relation const &rel, + osmium::memory::Buffer const &ways); + wkbs_t get_wkb_multiline(osmium::memory::Buffer const &ways, bool split); + +private: + wkbs_t create_multipolygon(osmium::Area const &area); + void add_mp_points(const osmium::NodeRefList &nodes); + + std::shared_ptr m_proj; + // inernal buffer for creating areas + osmium::memory::Buffer m_buffer; + osmium::geom::detail::WKBFactoryImpl m_writer; +}; + +} // namespace diff --git a/output-pgsql.cpp b/output-pgsql.cpp index 99a2155c3..65c926d19 100644 --- a/output-pgsql.cpp +++ b/output-pgsql.cpp @@ -38,6 +38,7 @@ #include "tagtransform.hpp" #include "util.hpp" #include "wildcmp.hpp" +#include "wkb-parser.hpp" /* make the diagnostic information work with older versions of * boost - the function signature changed at version 1.54. @@ -70,127 +71,36 @@ E4C1421D5BF24D06053E7DF4940 212696 Oswald Road \N \N \N \N \N \N minor \N \N \N \N \N \N \N 0102000020E610000004000000467D923B6C22D5BFA359D93EE4DF4940B3976DA7AD11D5BF84BBB376DBDF4940997FF44D9A06D5BF4223D8B8FEDF49404D158C4AEA04D 5BF5BB39597FCDF4940 */ -int output_pgsql_t::pgsql_out_way(osmid_t id, taglist_t &outtags, - const nodelist_t &nodes, - int polygon, int roads) +void output_pgsql_t::pgsql_out_way(osmium::Way const &way, taglist_t *tags, + bool polygon, bool roads) { - /* Split long ways after around 1 degree or 100km */ - double split_at; - if (m_options.projection->target_latlon()) - split_at = 1; - else - split_at = 100 * 1000; - - char tmp[32]; - auto wkbs = builder.get_wkb_split(nodes, polygon, split_at); - for (const auto& wkb: wkbs) { - /* FIXME: there should be a better way to detect polygons */ - if (wkb.is_polygon()) { - expire.from_nodes_poly(nodes, id); - if ((wkb.area > 0.0) && m_enable_way_area) { - snprintf(tmp, sizeof(tmp), "%g", wkb.area); - outtags.push_override(tag_t("way_area", tmp)); + if (polygon && way.is_closed()) { + auto wkb = m_builder.get_wkb_polygon(way); + if (!wkb.empty()) { + expire.from_wkb(wkb.c_str(), way.id()); + if (m_enable_way_area) { + auto const area = + m_options.reproject_area + ? ewkb_parser_t(wkb).get_area( + m_options.projection.get()) + : ewkb_parser_t(wkb) + .get_area(); + tags->push_override(tag_t("way_area", std::to_string(area))); } - m_tables[t_poly]->write_row(id, outtags, wkb.geom); - } else { - expire.from_nodes_line(nodes); - m_tables[t_line]->write_row(id, outtags, wkb.geom); - if (roads) - m_tables[t_roads]->write_row(id, outtags, wkb.geom); + m_tables[t_poly]->write_row_wkb(way.id(), *tags, wkb); } - } - - return 0; -} - -int output_pgsql_t::pgsql_out_relation(osmid_t id, const taglist_t &rel_tags, - const multinodelist_t &xnodes, const multitaglist_t & xtags, - const idlist_t &xid, const rolelist_t &xrole, - bool pending) -{ - if (xnodes.empty()) - return 0; - - int roads = 0; - int make_polygon = 0; - int make_boundary = 0; - double split_at; - - std::vector members_superseded(xnodes.size(), 0); - taglist_t outtags; - - //if its a route relation make_boundary and make_polygon will be false otherwise one or the other will be true - if (m_tagtransform->filter_rel_member_tags(rel_tags, xtags, xrole, - &(members_superseded[0]), &make_boundary, &make_polygon, &roads, - *m_export_list.get(), outtags)) { - return 0; - } - - /* Split long linear ways after around 1 degree or 100km (polygons not effected) */ - if (m_options.projection->target_latlon()) - split_at = 1; - else - split_at = 100 * 1000; - - //this will either make lines or polygons (unless the lines arent a ring or are less than 3 pts) depending on the tag transform above - //TODO: pick one or the other based on which we expect to care about - auto wkbs = builder.build_both(xnodes, make_polygon, m_options.enable_multi, split_at, id); - - if (wkbs.empty()) { - return 0; - } - - char tmp[32]; - for (const auto& wkb: wkbs) { - expire.from_wkb(wkb.geom.c_str(), -id); - /* FIXME: there should be a better way to detect polygons */ - if (wkb.is_polygon()) { - if ((wkb.area > 0.0) && m_enable_way_area) { - snprintf(tmp, sizeof(tmp), "%g", wkb.area); - outtags.push_override(tag_t("way_area", tmp)); - } - m_tables[t_poly]->write_row(-id, outtags, wkb.geom); - } else { - m_tables[t_line]->write_row(-id, outtags, wkb.geom); - if (roads) - m_tables[t_roads]->write_row(-id, outtags, wkb.geom); - } - } - - /* Tagtransform will have marked those member ways of the relation that - * have fully been dealt with as part of the multi-polygon entry. - * Set them in the database as done and delete their entry to not - * have duplicates */ - //dont do this when working with pending relations as its not needed - if (make_polygon) { - for (size_t i=0; i < xid.size(); i++) { - if (members_superseded[i]) { - pgsql_delete_way_from_output(xid[i]); - if(!pending) - ways_done_tracker->mark(xid[i]); + } else { + for (auto const &wkb : m_builder.get_wkb_split(way)) { + expire.from_wkb(wkb.c_str(), way.id()); + m_tables[t_poly]->write_row_wkb(way.id(), *tags, wkb); + if (roads) { + m_tables[t_roads]->write_row_wkb(way.id(), *tags, wkb); } } - } - // If the tag transform said the polygon looked like a boundary we want to make that as well - // If we are making a boundary then also try adding any relations which form complete rings - // The linear variants will have already been processed above - if (make_boundary) { - wkbs = builder.build_polygons(xnodes, m_options.enable_multi, id); - for (const auto& wkb: wkbs) { - expire.from_wkb(wkb.geom.c_str(), -id); - if ((wkb.area > 0.0) && m_enable_way_area) { - snprintf(tmp, sizeof(tmp), "%g", wkb.area); - outtags.push_override(tag_t("way_area", tmp)); - } - m_tables[t_poly]->write_row(-id, outtags, wkb.geom); - } } - - return 0; } - void output_pgsql_t::enqueue_ways(pending_queue_t &job_queue, osmid_t id, size_t output_id, size_t& added) { osmid_t const prev = ways_pending_tracker.last_returned(); if (id_tracker::is_valid(prev) && prev >= id) { @@ -252,9 +162,11 @@ int output_pgsql_t::pending_way(osmid_t id, int exists) { auto &way = buffer.get(0); if (!m_tagtransform->filter_tags(way, &polygon, &roads, *m_export_list.get(), outtags)) { - nodelist_t nodes; - m_mid->nodes_get_list(nodes, way.nodes(), reproj.get()); - return pgsql_out_way(id, outtags, nodes, polygon, roads); + auto nnodes = m_mid->nodes_get_list(&(way.nodes())); + if (nnodes > 1) { + pgsql_out_way(way, &outtags, polygon, roads); + return 1; + } } } @@ -305,8 +217,13 @@ int output_pgsql_t::pending_relation(osmid_t id, int exists) { // might be relocated when more data is added. rels_buffer.clear(); if (m_mid->relations_get(id, rels_buffer)) { + // If the flag says this object may exist already, delete it first. + if (exists) { + pgsql_delete_relation_from_output(id); + } + auto const &rel = rels_buffer.get(0); - return pgsql_process_relation(rel, exists, true); + return pgsql_process_relation(rel, true); } return 0; @@ -355,9 +272,9 @@ int output_pgsql_t::node_add(osmium::Node const &node) *m_export_list.get(), outtags)) return 1; - auto c = reproj->reproject(node.location()); - expire.from_bbox(c.x, c.y, c.x, c.y); - m_tables[t_point]->write_node(node.id(), outtags, c.y, c.x); + auto wkb = m_builder.get_wkb_node(node.location()); + expire.from_wkb(wkb.c_str(), node.id()); + m_tables[t_point]->write_row_wkb(node.id(), outtags, wkb); return 0; } @@ -379,9 +296,10 @@ int output_pgsql_t::way_add(osmium::Way *way) if( !polygon && !filter ) { /* Get actual node data and generate output */ - nodelist_t nodes; - m_mid->nodes_get_list(nodes, way->nodes(), reproj.get()); - pgsql_out_way(way->id(), outtags, nodes, polygon, roads); + auto nnodes = m_mid->nodes_get_list(&(way->nodes())); + if (nnodes > 1) { + pgsql_out_way(*way, &outtags, polygon, roads); + } } return 0; } @@ -389,38 +307,35 @@ int output_pgsql_t::way_add(osmium::Way *way) /* This is the workhorse of pgsql_add_relation, split out because it is used as the callback for iterate relations */ int output_pgsql_t::pgsql_process_relation(osmium::Relation const &rel, - bool exists, bool pending) + bool pending) { - /* If the flag says this object may exist already, delete it first */ - if(exists) - pgsql_delete_relation_from_output(rel.id()); - - taglist_t outtags; - if (m_tagtransform->filter_tags(rel, nullptr, nullptr, *m_export_list.get(), outtags)) - return 1; - - idlist_t xid2; - for (auto const &m : rel.members()) - { - /* Need to handle more than just ways... */ - if (m.type() == osmium::item_type::way) { - xid2.push_back(m.ref()); + taglist_t prefiltered_tags; + if (m_tagtransform->filter_tags(rel, nullptr, nullptr, *m_export_list.get(), + prefiltered_tags)) { + return 1; } + idlist_t xid2; + for (auto const &m : rel.members()) { + /* Need to handle more than just ways... */ + if (m.type() == osmium::item_type::way) { + xid2.push_back(m.ref()); + } + } buffer.clear(); auto num_ways = m_mid->ways_get_list(xid2, buffer); + + if (num_ways == 0) + return 0; + multitaglist_t xtags(num_ways, taglist_t()); rolelist_t xrole(num_ways); - multinodelist_t xnodes(num_ways, nodelist_t()); - idlist_t xid; size_t i = 0; for (auto const &w : buffer.select()) { assert(i < num_ways); - xid.push_back(w.id()); - m_mid->nodes_get_list(xnodes[i], w.nodes(), reproj.get()); //filter the tags on this member because we got it from the middle //and since the middle is no longer tied to the output it no longer @@ -440,8 +355,75 @@ int output_pgsql_t::pgsql_process_relation(osmium::Relation const &rel, ++i; } - /* At some point we might want to consider storing the retrieved data in the members, rather than as separate arrays */ - pgsql_out_relation(rel.id(), outtags, xnodes, xtags, xid, xrole, pending); + ////// pgsql_out_relation + //pgsql_out_relation(rel, outtags, xtags, xid, xrole, pending); + + int roads = 0; + int make_polygon = 0; + int make_boundary = 0; + std::vector members_superseded(num_ways, 0); + taglist_t outtags; + + // If it's a route relation make_boundary and make_polygon will be false + // otherwise one or the other will be true. + if (m_tagtransform->filter_rel_member_tags( + prefiltered_tags, xtags, xrole, &(members_superseded[0]), + &make_boundary, &make_polygon, &roads, *m_export_list.get(), + outtags)) { + return 0; + } + + for (auto &w : buffer.select()) { + m_mid->nodes_get_list(&(w.nodes())); + } + + // multipolygons and boundaries + if (make_boundary || make_polygon) { + auto wkbs = m_builder.get_wkb_multipolygon(rel, buffer); + + for (auto const &wkb : wkbs) { + expire.from_wkb(wkb.c_str(), -rel.id()); + if (m_enable_way_area) { + auto const area = + m_options.reproject_area + ? ewkb_parser_t(wkb).get_area( + m_options.projection.get()) + : ewkb_parser_t(wkb) + .get_area(); + outtags.push_override(tag_t("way_area", std::to_string(area))); + } + m_tables[t_poly]->write_row_wkb(-rel.id(), outtags, wkb); + } + + /* Tagtransform will have marked those member ways of the relation that + * have fully been dealt with as part of the multi-polygon entry. + * Set them in the database as done and delete their entry to not + * have duplicates */ + if (make_polygon) { + size_t j = 0; + for (auto &w : buffer.select()) { + if (members_superseded[j]) { + pgsql_delete_way_from_output(w.id()); + // When working with pending relations this is not needed. + if (!pending) { + ways_done_tracker->mark(w.id()); + } + } + ++j; + } + } + } + + // linear features and boundaries + if (!make_polygon) { + auto wkbs = m_builder.get_wkb_multiline(buffer, true); + for (auto const &wkb : wkbs) { + expire.from_wkb(wkb.c_str(), -rel.id()); + m_tables[t_line]->write_row_wkb(-rel.id(), outtags, wkb); + if (roads) + m_tables[t_roads]->write_row_wkb(-rel.id(), outtags, wkb); + } + } return 0; } @@ -460,7 +442,7 @@ int output_pgsql_t::relation_add(osmium::Relation const &rel) return 0; } - return pgsql_process_relation(rel, 0); + return pgsql_process_relation(rel, false); } /* Delete is easy, just remove all traces of this object. We don't need to @@ -588,17 +570,13 @@ std::shared_ptr output_pgsql_t::clone(const middle_query_t* cloned_mid return std::shared_ptr(clone); } -output_pgsql_t::output_pgsql_t(const middle_query_t* mid, const options_t &o) - : output_t(mid, o), - expire(o.expire_tiles_zoom, o.expire_tiles_max_bbox, o.projection), - ways_done_tracker(new id_tracker()), - buffer(32768, osmium::memory::Buffer::auto_grow::yes), - rels_buffer(1024, osmium::memory::Buffer::auto_grow::yes) +output_pgsql_t::output_pgsql_t(const middle_query_t *mid, const options_t &o) +: output_t(mid, o), m_builder(o.projection), + expire(o.expire_tiles_zoom, o.expire_tiles_max_bbox, o.projection), + ways_done_tracker(new id_tracker()), + buffer(32768, osmium::memory::Buffer::auto_grow::yes), + rels_buffer(1024, osmium::memory::Buffer::auto_grow::yes) { - reproj = m_options.projection; - builder.set_exclude_broken_polygon(m_options.excludepoly); - if (m_options.reproject_area) builder.set_reprojection(reproj.get()); - m_export_list.reset(new export_list()); m_enable_way_area = read_style_file( m_options.style, m_export_list.get() ); @@ -650,31 +628,29 @@ output_pgsql_t::output_pgsql_t(const middle_query_t* mid, const options_t &o) //tremble in awe of this massive constructor! seriously we are trying to avoid passing an //options object because we want to make use of the table_t in output_mutli_t which could //have a different tablespace/hstores/etc per table - m_tables.push_back(std::shared_ptr( - new table_t( - m_options.database_options.conninfo(), name, type, columns, m_options.hstore_columns, - reproj->target_srs(), - m_options.append, m_options.slim, m_options.droptemp, m_options.hstore_mode, - m_options.enable_hstore_index, m_options.tblsmain_data, m_options.tblsmain_index - ) - )); + m_tables.push_back(std::shared_ptr(new table_t( + m_options.database_options.conninfo(), name, type, columns, + m_options.hstore_columns, m_options.projection->target_srs(), + m_options.append, m_options.slim, m_options.droptemp, + m_options.hstore_mode, m_options.enable_hstore_index, + m_options.tblsmain_data, m_options.tblsmain_index))); } } -output_pgsql_t::output_pgsql_t(const output_pgsql_t& other): - output_t(other.m_mid, other.m_options), m_tagtransform(new tagtransform(&m_options)), m_enable_way_area(other.m_enable_way_area), - m_export_list(new export_list(*other.m_export_list)), - expire(m_options.expire_tiles_zoom, m_options.expire_tiles_max_bbox, - m_options.projection), - reproj(other.reproj), - //NOTE: we need to know which ways were used by relations so each thread - //must have a copy of the original marked done ways, its read only so its ok - ways_done_tracker(other.ways_done_tracker), - buffer(1024, osmium::memory::Buffer::auto_grow::yes), - rels_buffer(1024, osmium::memory::Buffer::auto_grow::yes) +output_pgsql_t::output_pgsql_t(const output_pgsql_t &other) +: output_t(other.m_mid, other.m_options), + m_tagtransform(new tagtransform(&m_options)), + m_enable_way_area(other.m_enable_way_area), + m_export_list(new export_list(*other.m_export_list)), + m_builder(m_options.projection), + expire(m_options.expire_tiles_zoom, m_options.expire_tiles_max_bbox, + m_options.projection), + //NOTE: we need to know which ways were used by relations so each thread + //must have a copy of the original marked done ways, its read only so its ok + ways_done_tracker(other.ways_done_tracker), + buffer(1024, osmium::memory::Buffer::auto_grow::yes), + rels_buffer(1024, osmium::memory::Buffer::auto_grow::yes) { - builder.set_exclude_broken_polygon(m_options.excludepoly); - if (m_options.reproject_area) builder.set_reprojection(reproj.get()); for(std::vector >::const_iterator t = other.m_tables.begin(); t != other.m_tables.end(); ++t) { //copy constructor will just connect to the already there table m_tables.push_back(std::shared_ptr(new table_t(**t))); diff --git a/output-pgsql.hpp b/output-pgsql.hpp index e1145db0f..ccbee75be 100644 --- a/output-pgsql.hpp +++ b/output-pgsql.hpp @@ -6,13 +6,13 @@ #ifndef OUTPUT_PGSQL_H #define OUTPUT_PGSQL_H -#include "output.hpp" -#include "tagtransform.hpp" -#include "geometry-builder.hpp" -#include "reprojection.hpp" #include "expire-tiles.hpp" #include "id-tracker.hpp" +#include "osmium-builder.hpp" +#include "output.hpp" +#include "reprojection.hpp" #include "table.hpp" +#include "tagtransform.hpp" #include #include @@ -57,14 +57,9 @@ class output_pgsql_t : public output_t { void merge_expire_trees(output_t *other) override; protected: - int pgsql_out_way(osmid_t id, taglist_t &tags, const nodelist_t &nodes, - int polygons, int roads); - int pgsql_out_relation(osmid_t id, const taglist_t &rel_tags, - const multinodelist_t &xnodes, const multitaglist_t & xtags, - const idlist_t &xid, const rolelist_t &xrole, - bool pending); - int pgsql_process_relation(osmium::Relation const &rel, - bool exists, bool pending=false); + void pgsql_out_way(osmium::Way const &way, taglist_t *tags, bool polygon, + bool roads); + int pgsql_process_relation(osmium::Relation const &rel, bool pending); int pgsql_delete_way_from_output(osmid_t osm_id); int pgsql_delete_relation_from_output(osmid_t osm_id); @@ -77,11 +72,9 @@ class output_pgsql_t : public output_t { std::unique_ptr m_export_list; - geometry_builder builder; + geom::osmium_builder_t m_builder; expire_tiles expire; - std::shared_ptr reproj; - id_tracker ways_pending_tracker, rels_pending_tracker; std::shared_ptr ways_done_tracker; osmium::memory::Buffer buffer; diff --git a/reprojection.hpp b/reprojection.hpp index c0609dfd9..f37dd54d9 100644 --- a/reprojection.hpp +++ b/reprojection.hpp @@ -50,6 +50,17 @@ class reprojection : public boost::noncopyable return target_srs() == PROJ_LATLONG; } + // interface for osmium GeometryFactory + + osmium::geom::Coordinates operator()(osmium::Location location) const + { + return reproject(location); + } + + int epsg() const { return target_srs(); } + + std::string proj_string() const { return target_desc(); } + /** * Create a reprojection object with target srs `srs`. * diff --git a/table.cpp b/table.cpp index 6196e40d4..a14cac991 100644 --- a/table.cpp +++ b/table.cpp @@ -344,6 +344,52 @@ void table_t::write_row(const osmid_t id, const taglist_t &tags, const std::stri } } +static const char *lookup_hex = "0123456789ABCDEF"; + +void table_t::write_row_wkb(const osmid_t id, const taglist_t &tags, + const std::string &geom) +{ + //add the osm id + buffer.append((single_fmt % id).str()); + buffer.push_back('\t'); + + // used to remember which columns have been written out already. + std::vector used; + + if (hstore_mode != HSTORE_NONE) + used.assign(tags.size(), false); + + //get the regular columns' values + write_columns(tags, buffer, hstore_mode == HSTORE_NORM ? &used : nullptr); + + //get the hstore columns' values + write_hstore_columns(tags, buffer); + + //get the key value pairs for the tags column + if (hstore_mode != HSTORE_NONE) + write_tags_column(tags, buffer, used); + + //add the geometry - encoding it to hex along the way + for (char c : geom) { + buffer += lookup_hex[(c >> 4) & 0xf]; + buffer += lookup_hex[c & 0xf]; + } + //we need \n because we are copying from stdin + buffer.push_back('\n'); + + //tell the db we are copying if for some reason we arent already + if (!copyMode) { + pgsql_exec_simple(sql_conn, PGRES_COPY_IN, copystr); + copyMode = true; + } + + //send all the data to postgres + if (buffer.length() > BUFFER_SEND_SIZE) { + pgsql_CopyData(name.c_str(), sql_conn, buffer); + buffer.clear(); + } +} + void table_t::write_columns(const taglist_t &tags, string& values, std::vector *used) { //for each column diff --git a/table.hpp b/table.hpp index f4b2423f4..bc41044e7 100644 --- a/table.hpp +++ b/table.hpp @@ -32,6 +32,8 @@ class table_t void commit(); void write_row(const osmid_t id, const taglist_t &tags, const std::string &geom); + void write_row_wkb(const osmid_t id, const taglist_t &tags, + const std::string &geom); void write_node(const osmid_t id, const taglist_t &tags, double lat, double lon); void delete_row(const osmid_t id); diff --git a/wkb-parser.hpp b/wkb-parser.hpp new file mode 100644 index 000000000..a9ab80d15 --- /dev/null +++ b/wkb-parser.hpp @@ -0,0 +1,148 @@ + +#include +#include +#include +#include + +#include +#include + +/** + * Class that allows to iterate over the elements of a ewkb geometry. + * + * Note: this class assumes that the wkb was created by osmium::geom::WKBFactory. + * It implements the exact opposite decoding. + */ +class ewkb_parser_t +{ +public: + enum geometry_type + { + wkb_point = 1, + wkb_line = 2, + wkb_polygon = 3, + wkb_multi_point = 4, + wkb_multi_line = 5, + wkb_multi_polygon = 6, + wkb_collection = 7, + }; + + explicit ewkb_parser_t(char const *wkb) : m_wkb(wkb), m_pos(0) {} + explicit ewkb_parser_t(std::string const &wkb) + : m_wkb(wkb.c_str()), m_pos(0) + { + } + + int read_header() + { + m_pos += sizeof(uint8_t); // skip endianess + + auto type = read_data(); + + m_pos += sizeof(int); // skip srid + + return type & 0xff; + } + + uint32_t read_length() { return read_data(); } + + osmium::geom::Coordinates read_point() + { + auto x = read_data(); + auto y = read_data(); + + return osmium::geom::Coordinates(x, y); + } + + template + double get_area(PROJ *proj = nullptr) + { + double total = 0; + + auto type = read_header(); + + if (type == wkb_polygon) { + total = get_polygon_area(proj); + } else if (type == wkb_multi_polygon) { + auto num_poly = read_length(); + for (unsigned i = 0; i < num_poly; ++i) { + auto ptype = read_header(); + (void)ptype; + assert(ptype == wkb_polygon); + + total += get_polygon_area(proj); + } + } + + return total; + } + +private: + template + double get_polygon_area(PROJ *proj) + { + auto num_rings = read_length(); + assert(num_rings > 0); + + double total = get_ring_area(proj); + + for (unsigned i = 1; i < num_rings; ++i) { + total -= get_ring_area(proj); + } + + return total; + } + + template + double get_ring_area(PROJ *proj) + { + // Algorithm borrowed from + // http://stackoverflow.com/questions/451426/how-do-i-calculate-the-area-of-a-2d-polygon + auto num_pts = read_length(); + assert(num_pts > 3); + + double total = 0; + + auto prev = read_point(); + proj->target_to_tile(&prev.y, &prev.x); + for (unsigned i = 1; i < num_pts; ++i) { + auto cur = read_point(); + proj->target_to_tile(&cur.y, &cur.x); + total += prev.x * cur.y - cur.x * prev.y; + prev = cur; + } + + return std::abs(total) * 0.5; + } + + double get_ring_area(osmium::geom::IdentityProjection *) + { + // Algorithm borrowed from + // http://stackoverflow.com/questions/451426/how-do-i-calculate-the-area-of-a-2d-polygon + auto num_pts = read_length(); + assert(num_pts > 3); + + double total = 0; + + auto prev = read_point(); + for (unsigned i = 1; i < num_pts; ++i) { + auto cur = read_point(); + total += prev.x * cur.y - cur.x * prev.y; + prev = cur; + } + + return std::abs(total) * 0.5; + } + + template + T read_data() + { + auto *data = reinterpret_cast(m_wkb + m_pos); + m_pos += sizeof(T); + + return *data; + } + + char const *m_wkb; + unsigned m_pos; +}; From cf7e8246f6e8f523cb72dba37e261bc6ad0d5c79 Mon Sep 17 00:00:00 2001 From: Sarah Hoffmann Date: Mon, 23 Jan 2017 00:13:54 +0100 Subject: [PATCH 04/30] expire directly from wkb Don't go through the conversation into a geos geometry but use our new wkb reader instead. --- expire-tiles.cpp | 115 ++++++++++++++++++++++++++++++++++++++++------- expire-tiles.hpp | 7 ++- wkb-parser.hpp | 5 +++ 3 files changed, 109 insertions(+), 18 deletions(-) diff --git a/expire-tiles.cpp b/expire-tiles.cpp index 28c1f3bc7..8653b1a69 100644 --- a/expire-tiles.cpp +++ b/expire-tiles.cpp @@ -16,9 +16,9 @@ #include "expire-tiles.hpp" #include "options.hpp" -#include "geometry-builder.hpp" #include "reprojection.hpp" #include "table.hpp" +#include "wkb-parser.hpp" #define EARTH_CIRCUMFERENCE 40075016.68 #define HALF_EARTH_CIRCUMFERENCE (EARTH_CIRCUMFERENCE / 2) @@ -377,30 +377,113 @@ void expire_tiles::from_nodes_poly(const nodelist_t &nodes, osmid_t osm_id) } } -void expire_tiles::from_xnodes_poly(const multinodelist_t &xnodes, osmid_t osm_id) +void expire_tiles::from_wkb(const char *wkb, osmid_t osm_id) { - for (multinodelist_t::const_iterator it = xnodes.begin(); it != xnodes.end(); ++it) - from_nodes_poly(*it, osm_id); + if (maxzoom < 0) { + return; + } + + auto parse = ewkb_parser_t(wkb); + + switch (parse.read_header()) { + case ewkb_parser_t::wkb_point: + from_wkb_point(&parse); + break; + case ewkb_parser_t::wkb_line: + from_wkb_line(&parse); + break; + case ewkb_parser_t::wkb_polygon: + from_wkb_polygon(&parse, osm_id); + break; + case ewkb_parser_t::wkb_multi_line: { + auto num = parse.read_length(); + for (unsigned i = 0; i < num; ++i) { + parse.read_header(); + from_wkb_line(&parse); + } + break; + } + case ewkb_parser_t::wkb_multi_polygon: { + auto num = parse.read_length(); + for (unsigned i = 0; i < num; ++i) { + parse.read_header(); + from_wkb_polygon(&parse, osm_id); + } + break; + } + default: + fprintf(stderr, "OSM id %" PRIdOSMID + ": Unknown geometry type, cannot expire.\n", + osm_id); + } } -void expire_tiles::from_xnodes_line(const multinodelist_t &xnodes) +void expire_tiles::from_wkb_point(ewkb_parser_t *wkb) { - for (multinodelist_t::const_iterator it = xnodes.begin(); it != xnodes.end(); ++it) - from_nodes_line(*it); + auto c = wkb->read_point(); + from_bbox(c.x, c.y, c.x, c.y); } -void expire_tiles::from_wkb(const char* wkb, osmid_t osm_id) +void expire_tiles::from_wkb_line(ewkb_parser_t *wkb) { - if (maxzoom < 0) return; + auto sz = wkb->read_length(); + + if (sz == 0) { + return; + } - multinodelist_t xnodes; - bool polygon; + if (sz == 1) { + from_wkb_point(wkb); + } else { + auto prev = wkb->read_point(); + for (size_t i = 1; i < sz; ++i) { + auto cur = wkb->read_point(); + from_line(prev.x, prev.y, cur.x, cur.y); + prev = cur; + } + } +} - if (geometry_builder::parse_wkb(wkb, xnodes, &polygon) == 0) { - if (polygon) - from_xnodes_poly(xnodes, osm_id); - else - from_xnodes_line(xnodes); +void expire_tiles::from_wkb_polygon(ewkb_parser_t *wkb, osmid_t osm_id) +{ + auto num_rings = wkb->read_length(); + assert(num_rings > 0); + + auto start = wkb->save_pos(); + + auto num_pt = wkb->read_length(); + auto initpt = wkb->read_point(); + + osmium::geom::Coordinates min{initpt}, max{initpt}; + + for (size_t i = 1; i < num_pt; ++i) { + auto c = wkb->read_point(); + if (c.x < min.x) + min.x = c.x; + if (c.y < min.y) + min.y = c.y; + if (c.x > max.x) + max.x = c.x; + if (c.y > max.y) + max.y = c.y; + } + + if (from_bbox(min.x, min.y, max.x, max.y)) { + /* Bounding box too big - just expire tiles on the line */ + fprintf(stderr, + "\rLarge polygon (%.0f x %.0f metres, OSM ID %" PRIdOSMID + ") - only expiring perimeter\n", + max.x - min.x, max.y - min.y, osm_id); + wkb->rewind(start); + for (unsigned ring = 0; ring < num_rings; ++ring) { + from_wkb_line(wkb); + } + } else { + // ignore inner rings + for (unsigned ring = 1; ring < num_rings; ++ring) { + auto inum_pt = wkb->read_length(); + wkb->skip_points(inum_pt); + } } } diff --git a/expire-tiles.hpp b/expire-tiles.hpp index 840e4fd9b..c74d22141 100644 --- a/expire-tiles.hpp +++ b/expire-tiles.hpp @@ -8,6 +8,7 @@ class reprojection; class table_t; class tile; +class ewkb_parser_t; struct expire_tiles { @@ -47,8 +48,10 @@ struct expire_tiles void expire_tile(int x, int y); int normalise_tile_x_coord(int x); void from_line(double lon_a, double lat_a, double lon_b, double lat_b); - void from_xnodes_poly(const multinodelist_t &xnodes, osmid_t osm_id); - void from_xnodes_line(const multinodelist_t &xnodes); + + void from_wkb_point(ewkb_parser_t *wkb); + void from_wkb_line(ewkb_parser_t *wkb); + void from_wkb_polygon(ewkb_parser_t *wkb, osmid_t osm_id); double tile_width; double max_bbox; diff --git a/wkb-parser.hpp b/wkb-parser.hpp index a9ab80d15..05093203a 100644 --- a/wkb-parser.hpp +++ b/wkb-parser.hpp @@ -33,6 +33,9 @@ class ewkb_parser_t { } + unsigned save_pos() const { return m_pos; } + void rewind(unsigned pos) { m_pos = pos; } + int read_header() { m_pos += sizeof(uint8_t); // skip endianess @@ -54,6 +57,8 @@ class ewkb_parser_t return osmium::geom::Coordinates(x, y); } + void skip_points(size_t num) { m_pos += sizeof(double) * 2 * num; } + template double get_area(PROJ *proj = nullptr) { From 005776787e413698e4f4d815aa59db9221bee780 Mon Sep 17 00:00:00 2001 From: Sarah Hoffmann Date: Tue, 24 Jan 2017 22:53:22 +0100 Subject: [PATCH 05/30] add linemerger for route relations --- osmium-builder.cpp | 164 +++++++++++++++++++++++++++++++++++++++++++-- osmium-builder.hpp | 6 +- output-pgsql.cpp | 4 +- 3 files changed, 163 insertions(+), 11 deletions(-) diff --git a/osmium-builder.cpp b/osmium-builder.cpp index 1e04b4723..3fe511f40 100644 --- a/osmium-builder.cpp +++ b/osmium-builder.cpp @@ -1,3 +1,8 @@ +#include +#include +#include +#include + #include #include @@ -22,19 +27,39 @@ inline osmium::geom::Coordinates interpolate(osmium::geom::Coordinates p1, return osmium::geom::Coordinates(frac * (p1.x - p2.x) + p2.x, frac * (p1.y - p2.y) + p2.y); } + +template +inline void add_nodes_to_builder(osmium::builder::WayNodeListBuilder &builder, + ITERATOR const &begin, ITERATOR const &end, + bool skip_first) +{ + auto it = begin; + if (skip_first) { + ++it; + } + + while (it != end) { + if (it->location().valid()) { + builder.add_node_ref(*it); + } + ++it; + } } +} // name space + namespace geom { using WKBWriter = osmium::geom::detail::WKBFactoryImpl; osmium_builder_t::wkb_t -osmium_builder_t::get_wkb_node(osmium::Location const &loc) +osmium_builder_t::get_wkb_node(osmium::Location const &loc) const { return m_writer.make_point(m_proj->reproject(loc)); } -osmium_builder_t::wkbs_t osmium_builder_t::get_wkb_split(osmium::Way const &way) +osmium_builder_t::wkbs_t +osmium_builder_t::get_wkb_split(osmium::WayNodeList const &nodes) { wkbs_t ret; @@ -45,7 +70,7 @@ osmium_builder_t::wkbs_t osmium_builder_t::get_wkb_split(osmium::Way const &way) m_writer.linestring_start(); size_t curlen = 0; - for (auto const &node : way.nodes()) { + for (auto const &node : nodes) { if (!node.location().valid()) continue; @@ -139,16 +164,143 @@ osmium_builder_t::get_wkb_multiline(osmium::memory::Buffer const &ways, bool) { wkbs_t ret; - // XXX need to combine ways + // make a list of all endpoints + using endpoint_t = std::tuple; + std::vector endpoints; + // and a list of way connections + enum lmt : size_t + { + NOCONN = -1UL + }; + std::vector> conns; + + // initialise the two lists + for (auto const &w : ways.select()) { + if (w.nodes().size() > 1) { + endpoints.emplace_back(w.nodes().front().ref(), conns.size(), true); + endpoints.emplace_back(w.nodes().back().ref(), conns.size(), false); + conns.emplace_back(NOCONN, &w, NOCONN); + } + } + // sort by node id + std::sort(endpoints.begin(), endpoints.end()); + // now fill the connection list based on the sorted list + { + endpoint_t const *prev = nullptr; + for (auto const &pt : endpoints) { + if (prev) { + if (std::get<0>(*prev) == std::get<0>(pt)) { + auto previd = std::get<1>(*prev); + auto ptid = std::get<1>(pt); + if (std::get<2>(*prev)) { + std::get<0>(conns[previd]) = ptid; + } else { + std::get<2>(conns[previd]) = ptid; + } + if (std::get<2>(pt)) { + std::get<0>(conns[ptid]) = previd; + } else { + std::get<2>(conns[ptid]) = previd; + } + prev = nullptr; + continue; + } + } + + prev = &pt; + } + } + // XXX need to do non-split version + size_t done_ways = 0; + size_t todo_ways = conns.size(); wkbs_t linewkbs; - for (auto &w : ways.select()) { - linewkbs = get_wkb_split(w); + for (size_t i = 0; i < todo_ways; ++i) { + if (!std::get<1>(conns[i]) || (std::get<0>(conns[i]) != NOCONN && + std::get<2>(conns[i]) != NOCONN)) { + continue; // way already done or not the beginning of a segment + } + + m_buffer.clear(); + osmium::builder::WayNodeListBuilder wnl_builder(m_buffer); + size_t prev = NOCONN; + size_t cur = i; + + do { + auto &conn = conns[cur]; + assert(std::get<1>(conn)); + auto &nl = std::get<1>(conn)->nodes(); + bool skip_first = prev != NOCONN; + bool forward = std::get<0>(conn) == prev; + prev = cur; + // add way nodes + if (forward) { + add_nodes_to_builder(wnl_builder, nl.cbegin(), nl.cend(), + skip_first); + cur = std::get<2>(conn); + } else { + add_nodes_to_builder(wnl_builder, nl.crbegin(), nl.crend(), + skip_first); + cur = std::get<0>(conn); + } + // mark way as done + std::get<1>(conns[prev]) = nullptr; + ++done_ways; + } while (cur != NOCONN); + + // found a line end, create the wkbs + m_buffer.commit(); + linewkbs = get_wkb_split(m_buffer.get(0)); std::move(linewkbs.begin(), linewkbs.end(), std::inserter(ret, ret.end())); linewkbs.clear(); } + if (done_ways < todo_ways) { + // oh dear, there must be circular ways without an end + // need to do the same shebang again + for (size_t i = 0; i < todo_ways; ++i) { + if (!std::get<1>(conns[i])) { + continue; // way already done + } + + m_buffer.clear(); + osmium::builder::WayNodeListBuilder wnl_builder(m_buffer); + size_t prev = std::get<0>(conns[i]); + size_t cur = i; + bool skip_first = false; + + do { + auto &conn = conns[cur]; + assert(std::get<1>(conn)); + auto &nl = std::get<1>(conn)->nodes(); + bool forward = std::get<0>(conn) == prev; + prev = cur; + if (forward) { + // add way forwards + add_nodes_to_builder(wnl_builder, nl.cbegin(), nl.cend(), + skip_first); + cur = std::get<2>(conn); + } else { + // add way backwards + add_nodes_to_builder(wnl_builder, nl.crbegin(), nl.crend(), + skip_first); + cur = std::get<0>(conn); + } + // mark way as done + std::get<1>(conns[prev]) = nullptr; + skip_first = true; + } while (cur != i); + + // found a line end, create the wkbs + m_buffer.commit(); + linewkbs = get_wkb_split(m_buffer.get(0)); + std::move(linewkbs.begin(), linewkbs.end(), + std::inserter(ret, ret.end())); + linewkbs.clear(); + } + } + return ret; } diff --git a/osmium-builder.hpp b/osmium-builder.hpp index 93ecb3fdf..e02ad505f 100644 --- a/osmium-builder.hpp +++ b/osmium-builder.hpp @@ -22,8 +22,8 @@ class osmium_builder_t { } - wkb_t get_wkb_node(osmium::Location const &loc); - wkbs_t get_wkb_split(osmium::Way const &way); + wkb_t get_wkb_node(osmium::Location const &loc) const; + wkbs_t get_wkb_split(osmium::WayNodeList const &way); wkb_t get_wkb_polygon(osmium::Way const &way); wkbs_t get_wkb_multipolygon(osmium::Relation const &rel, @@ -35,7 +35,7 @@ class osmium_builder_t void add_mp_points(const osmium::NodeRefList &nodes); std::shared_ptr m_proj; - // inernal buffer for creating areas + // internal buffer for creating areas osmium::memory::Buffer m_buffer; osmium::geom::detail::WKBFactoryImpl m_writer; }; diff --git a/output-pgsql.cpp b/output-pgsql.cpp index 65c926d19..2aa468a15 100644 --- a/output-pgsql.cpp +++ b/output-pgsql.cpp @@ -90,9 +90,9 @@ void output_pgsql_t::pgsql_out_way(osmium::Way const &way, taglist_t *tags, m_tables[t_poly]->write_row_wkb(way.id(), *tags, wkb); } } else { - for (auto const &wkb : m_builder.get_wkb_split(way)) { + for (auto const &wkb : m_builder.get_wkb_split(way.nodes())) { expire.from_wkb(wkb.c_str(), way.id()); - m_tables[t_poly]->write_row_wkb(way.id(), *tags, wkb); + m_tables[t_line]->write_row_wkb(way.id(), *tags, wkb); if (roads) { m_tables[t_roads]->write_row_wkb(way.id(), *tags, wkb); } From 728a4ff36c5bac1638ce3da4beb2c18e434ac346 Mon Sep 17 00:00:00 2001 From: Sarah Hoffmann Date: Wed, 25 Jan 2017 23:06:45 +0100 Subject: [PATCH 06/30] add multipolygon split-up --- contrib/libosmium/osmium/geom/wkb.hpp | 15 +++++++ osmium-builder.cpp | 55 +++++++++++++++++++++----- osmium-builder.hpp | 10 +++-- output-pgsql.cpp | 4 +- tests/test-output-pgsql-tablespace.cpp | 4 +- tests/test-output-pgsql-validgeom.cpp | 2 +- tests/test-output-pgsql.cpp | 12 +++--- wkb-parser.hpp | 1 + 8 files changed, 80 insertions(+), 23 deletions(-) diff --git a/contrib/libosmium/osmium/geom/wkb.hpp b/contrib/libosmium/osmium/geom/wkb.hpp index 39105fc47..604e3a412 100644 --- a/contrib/libosmium/osmium/geom/wkb.hpp +++ b/contrib/libosmium/osmium/geom/wkb.hpp @@ -199,6 +199,21 @@ namespace osmium { } } + /* Polygon */ + polygon_type polygon_finish() { + set_size(m_polygon_size_offset, m_rings); + std::string data; + + using std::swap; + swap(data, m_data); + + if (m_out_type == out_type::hex) { + return convert_to_hex(data); + } else { + return data; + } + } + /* MultiPolygon */ void multipolygon_start() { diff --git a/osmium-builder.cpp b/osmium-builder.cpp index 3fe511f40..00a1e396e 100644 --- a/osmium-builder.cpp +++ b/osmium-builder.cpp @@ -96,10 +96,11 @@ osmium_builder_t::get_wkb_split(osmium::WayNodeList const &nodes) ((double)(j + 1) * split_at - dist) / delta; ipoint = interpolate(this_pt, prev_pt, frac); m_writer.linestring_add_location(ipoint); - ret.push_back(m_writer.linestring_finish(curlen)); + ret.push_back(m_writer.linestring_finish(curlen + 1)); // start a new segment m_writer.linestring_start(); m_writer.linestring_add_location(ipoint); + curlen = 1; } // reset the distance based on the final splitting point for // the next iteration. @@ -109,7 +110,6 @@ osmium_builder_t::get_wkb_split(osmium::WayNodeList const &nodes) curlen = 0; } else { dist = distance(this_pt, ipoint); - curlen = 1; } } else { dist += delta; @@ -139,7 +139,7 @@ osmium_builder_t::get_wkb_polygon(osmium::Way const &way) return wkb_t(); } - auto wkbs = create_multipolygon(m_buffer.get(0)); + auto wkbs = create_polygons(m_buffer.get(0)); return wkbs.empty() ? wkb_t() : wkbs[0]; } @@ -153,7 +153,11 @@ osmium_builder_t::get_wkb_multipolygon(osmium::Relation const &rel, m_buffer.clear(); if (assembler.make_area(rel, ways, m_buffer)) { - ret = create_multipolygon(m_buffer.get(0)); + if (m_build_multigeoms) { + ret.push_back(create_multipolygon(m_buffer.get(0))); + } else { + ret = create_polygons(m_buffer.get(0)); + } } return ret; @@ -317,12 +321,10 @@ void osmium_builder_t::add_mp_points(const osmium::NodeRefList &nodes) } } -osmium_builder_t::wkbs_t +osmium_builder_t::wkb_t osmium_builder_t::create_multipolygon(osmium::Area const &area) { - wkbs_t ret; - - // XXX need to split into polygons + wkb_t ret; try { size_t num_polygons = 0; @@ -353,7 +355,7 @@ osmium_builder_t::create_multipolygon(osmium::Area const &area) // if there are no rings, this area is invalid if (num_rings > 0) { m_writer.multipolygon_polygon_finish(); - ret.push_back(m_writer.multipolygon_finish()); + ret = m_writer.multipolygon_finish(); } } catch (osmium::geometry_error &e) { /* ignored */ @@ -362,4 +364,39 @@ osmium_builder_t::create_multipolygon(osmium::Area const &area) return ret; } +osmium_builder_t::wkbs_t +osmium_builder_t::create_polygons(osmium::Area const &area) +{ + wkbs_t ret; + + try { + size_t num_polygons = 0; + + for (auto it = area.cbegin(); it != area.cend(); ++it) { + if (it->type() == osmium::item_type::outer_ring) { + auto &ring = static_cast(*it); + if (num_polygons > 0) { + ret.push_back(m_writer.polygon_finish()); + } + m_writer.multipolygon_polygon_start(); + m_writer.multipolygon_outer_ring_start(); + add_mp_points(ring); + m_writer.multipolygon_outer_ring_finish(); + ++num_polygons; + } else if (it->type() == osmium::item_type::inner_ring) { + auto &ring = static_cast(*it); + m_writer.multipolygon_inner_ring_start(); + add_mp_points(ring); + m_writer.multipolygon_inner_ring_finish(); + } + } + + ret.push_back(m_writer.polygon_finish()); + + } catch (osmium::geometry_error &e) { /* ignored */ + } + + return ret; +} + } // name space diff --git a/osmium-builder.hpp b/osmium-builder.hpp index e02ad505f..57bb45f72 100644 --- a/osmium-builder.hpp +++ b/osmium-builder.hpp @@ -16,9 +16,11 @@ class osmium_builder_t typedef std::string wkb_t; typedef std::vector wkbs_t; - explicit osmium_builder_t(std::shared_ptr const &proj) + explicit osmium_builder_t(std::shared_ptr const &proj, + bool build_multigeoms) : m_proj(proj), m_buffer(1024, osmium::memory::Buffer::auto_grow::yes), - m_writer(m_proj->epsg(), osmium::geom::wkb_type::ewkb) + m_writer(m_proj->epsg(), osmium::geom::wkb_type::ewkb), + m_build_multigeoms(build_multigeoms) { } @@ -31,13 +33,15 @@ class osmium_builder_t wkbs_t get_wkb_multiline(osmium::memory::Buffer const &ways, bool split); private: - wkbs_t create_multipolygon(osmium::Area const &area); + wkb_t create_multipolygon(osmium::Area const &area); + wkbs_t create_polygons(osmium::Area const &area); void add_mp_points(const osmium::NodeRefList &nodes); std::shared_ptr m_proj; // internal buffer for creating areas osmium::memory::Buffer m_buffer; osmium::geom::detail::WKBFactoryImpl m_writer; + bool m_build_multigeoms; }; } // namespace diff --git a/output-pgsql.cpp b/output-pgsql.cpp index 2aa468a15..bf28c96fb 100644 --- a/output-pgsql.cpp +++ b/output-pgsql.cpp @@ -571,7 +571,7 @@ std::shared_ptr output_pgsql_t::clone(const middle_query_t* cloned_mid } output_pgsql_t::output_pgsql_t(const middle_query_t *mid, const options_t &o) -: output_t(mid, o), m_builder(o.projection), +: output_t(mid, o), m_builder(o.projection, o.enable_multi), expire(o.expire_tiles_zoom, o.expire_tiles_max_bbox, o.projection), ways_done_tracker(new id_tracker()), buffer(32768, osmium::memory::Buffer::auto_grow::yes), @@ -642,7 +642,7 @@ output_pgsql_t::output_pgsql_t(const output_pgsql_t &other) m_tagtransform(new tagtransform(&m_options)), m_enable_way_area(other.m_enable_way_area), m_export_list(new export_list(*other.m_export_list)), - m_builder(m_options.projection), + m_builder(m_options.projection, other.m_options.enable_multi), expire(m_options.expire_tiles_zoom, m_options.expire_tiles_max_bbox, m_options.projection), //NOTE: we need to know which ways were used by relations so each thread diff --git a/tests/test-output-pgsql-tablespace.cpp b/tests/test-output-pgsql-tablespace.cpp index 090209e6c..1d897fd28 100644 --- a/tests/test-output-pgsql-tablespace.cpp +++ b/tests/test-output-pgsql-tablespace.cpp @@ -89,9 +89,9 @@ void test_regression_simple() { db->assert_has_table("osm2pgsql_test_roads"); db->check_count(1342, "SELECT count(*) FROM osm2pgsql_test_point"); - db->check_count(3300, "SELECT count(*) FROM osm2pgsql_test_line"); + db->check_count(3231, "SELECT count(*) FROM osm2pgsql_test_line"); db->check_count( 375, "SELECT count(*) FROM osm2pgsql_test_roads"); - db->check_count(4128, "SELECT count(*) FROM osm2pgsql_test_polygon"); + db->check_count(4127, "SELECT count(*) FROM osm2pgsql_test_polygon"); } } // anonymous namespace diff --git a/tests/test-output-pgsql-validgeom.cpp b/tests/test-output-pgsql-validgeom.cpp index 44da9fcd7..14056fb86 100644 --- a/tests/test-output-pgsql-validgeom.cpp +++ b/tests/test-output-pgsql-validgeom.cpp @@ -83,7 +83,7 @@ void test_z_order() { db->assert_has_table("osm2pgsql_test_polygon"); db->assert_has_table("osm2pgsql_test_roads"); - db->check_count(6, "SELECT COUNT(*) FROM osm2pgsql_test_polygon"); + db->check_count(10, "SELECT COUNT(*) FROM osm2pgsql_test_polygon"); db->check_count(0, "SELECT COUNT(*) FROM osm2pgsql_test_polygon WHERE NOT ST_IsValid(way)"); db->check_count(0, "SELECT COUNT(*) FROM osm2pgsql_test_polygon WHERE ST_IsEmpty(way)"); } diff --git a/tests/test-output-pgsql.cpp b/tests/test-output-pgsql.cpp index c5aad142d..98092f714 100644 --- a/tests/test-output-pgsql.cpp +++ b/tests/test-output-pgsql.cpp @@ -88,9 +88,9 @@ void test_regression_simple() { db->assert_has_table("osm2pgsql_test_roads"); db->check_count(1342, "SELECT count(*) FROM osm2pgsql_test_point"); - db->check_count(3300, "SELECT count(*) FROM osm2pgsql_test_line"); + db->check_count(3231, "SELECT count(*) FROM osm2pgsql_test_line"); db->check_count( 375, "SELECT count(*) FROM osm2pgsql_test_roads"); - db->check_count(4128, "SELECT count(*) FROM osm2pgsql_test_polygon"); + db->check_count(4127, "SELECT count(*) FROM osm2pgsql_test_polygon"); // Check size of lines db->check_number(1696.04, "SELECT ST_Length(way) FROM osm2pgsql_test_line WHERE osm_id = 44822682"); @@ -140,9 +140,9 @@ void test_latlong() { db->assert_has_table("osm2pgsql_test_roads"); db->check_count(1342, "SELECT count(*) FROM osm2pgsql_test_point"); - db->check_count(3298, "SELECT count(*) FROM osm2pgsql_test_line"); + db->check_count(3229, "SELECT count(*) FROM osm2pgsql_test_line"); db->check_count(374, "SELECT count(*) FROM osm2pgsql_test_roads"); - db->check_count(4128, "SELECT count(*) FROM osm2pgsql_test_polygon"); + db->check_count(4127, "SELECT count(*) FROM osm2pgsql_test_polygon"); // Check size of lines db->check_number(0.0105343, "SELECT ST_Length(way) FROM osm2pgsql_test_line WHERE osm_id = 44822682"); @@ -277,9 +277,9 @@ void test_clone() { db->assert_has_table("osm2pgsql_test_roads"); db->check_count(1342, "SELECT count(*) FROM osm2pgsql_test_point"); - db->check_count(3300, "SELECT count(*) FROM osm2pgsql_test_line"); + db->check_count(3231, "SELECT count(*) FROM osm2pgsql_test_line"); db->check_count( 375, "SELECT count(*) FROM osm2pgsql_test_roads"); - db->check_count(4128, "SELECT count(*) FROM osm2pgsql_test_polygon"); + db->check_count(4127, "SELECT count(*) FROM osm2pgsql_test_polygon"); } } // anonymous namespace diff --git a/wkb-parser.hpp b/wkb-parser.hpp index 05093203a..3a77a00c3 100644 --- a/wkb-parser.hpp +++ b/wkb-parser.hpp @@ -103,6 +103,7 @@ class ewkb_parser_t { // Algorithm borrowed from // http://stackoverflow.com/questions/451426/how-do-i-calculate-the-area-of-a-2d-polygon + // XXX numerically not stable (useless for latlon) auto num_pts = read_length(); assert(num_pts > 3); From e1b75a5775d410b4e66cdef76d20c335b43706b3 Mon Sep 17 00:00:00 2001 From: Sarah Hoffmann Date: Thu, 2 Feb 2017 23:22:15 +0100 Subject: [PATCH 07/30] convert remaining outputs to libosmium --- contrib/libosmium/osmium/geom/wkb.hpp | 25 +++++++ expire-tiles.cpp | 1 + geometry-processor.cpp | 88 ++++++++-------------- geometry-processor.hpp | 33 +++------ osmium-builder.cpp | 96 +++++++++++++----------- osmium-builder.hpp | 7 +- output-gazetteer.cpp | 71 +++++++++--------- output-gazetteer.hpp | 34 +++------ output-multi.cpp | 103 ++++++++++++-------------- output-multi.hpp | 6 +- output-pgsql.cpp | 64 ++++++++-------- output-pgsql.hpp | 1 - processor-line.cpp | 20 +++-- processor-line.hpp | 14 ++-- processor-point.cpp | 14 ++-- processor-point.hpp | 12 ++- processor-polygon.cpp | 20 ++--- processor-polygon.hpp | 16 ++-- tests/test-output-multi-line.cpp | 4 +- 19 files changed, 310 insertions(+), 319 deletions(-) diff --git a/contrib/libosmium/osmium/geom/wkb.hpp b/contrib/libosmium/osmium/geom/wkb.hpp index 604e3a412..3f055a735 100644 --- a/contrib/libosmium/osmium/geom/wkb.hpp +++ b/contrib/libosmium/osmium/geom/wkb.hpp @@ -199,6 +199,31 @@ namespace osmium { } } + /* MultiLineString */ + + void multilinestring_start() { + m_data.clear(); + m_multipolygon_size_offset = header(m_data, wkbMultiLineString, true); + } + + void add_part(std::string const &part) { + m_data.append(part); + } + + linestring_type multilinestring_finish(size_t num_lines) { + set_size(m_multipolygon_size_offset, num_lines); + std::string data; + + using std::swap; + swap(data, m_data); + + if (m_out_type == out_type::hex) { + return convert_to_hex(data); + } else { + return data; + } + } + /* Polygon */ polygon_type polygon_finish() { set_size(m_polygon_size_offset, m_rings); diff --git a/expire-tiles.cpp b/expire-tiles.cpp index 8653b1a69..665b066a5 100644 --- a/expire-tiles.cpp +++ b/expire-tiles.cpp @@ -508,6 +508,7 @@ int expire_tiles::from_db(table_t* table, osmid_t osm_id) { //dirty the stuff const char* wkb = nullptr; while((wkb = wkbs.get_next())) + // XXX need to convert from hex to binary from_wkb(wkb, osm_id); //return how many rows were affected diff --git a/geometry-processor.cpp b/geometry-processor.cpp index a5191828d..76323d45c 100644 --- a/geometry-processor.cpp +++ b/geometry-processor.cpp @@ -14,16 +14,16 @@ std::shared_ptr geometry_processor::create(const std::string &type, const options_t *options) { std::shared_ptr ptr; - int srid = options->projection->target_srs(); if (type == "point") { - ptr = std::make_shared(srid); + ptr = std::make_shared(options->projection); } else if (type == "line") { - ptr = std::make_shared(srid); + ptr = std::make_shared(options->projection); } else if (type == "polygon") { - ptr = std::make_shared(srid, options->enable_multi); + ptr = std::make_shared(options->projection, + options->enable_multi); } else { throw std::runtime_error((boost::format("Unable to construct geometry processor " @@ -38,8 +38,7 @@ geometry_processor::geometry_processor(int srid, const std::string &type, unsign : m_srid(srid), m_type(type), m_interests(interests) { } -geometry_processor::~geometry_processor() { -} +geometry_processor::~geometry_processor() = default; int geometry_processor::srid() const { return m_srid; @@ -57,40 +56,29 @@ bool geometry_processor::interests(unsigned int interested) const { return (interested & m_interests) == interested; } -geometry_builder::pg_geom_t geometry_processor::process_node(double, double) { - return geometry_builder::pg_geom_t(); -} - -geometry_builder::pg_geom_t geometry_processor::process_way(const nodelist_t &) { - return geometry_builder::pg_geom_t(); -} - -geometry_builder::pg_geoms_t geometry_processor::process_relation(const multinodelist_t &) { - return geometry_builder::pg_geoms_t(); +geometry_processor::wkb_t +geometry_processor::process_node(osmium::Location const &) +{ + return wkb_t(); } -way_helper::way_helper() +geometry_processor::wkb_t geometry_processor::process_way(osmium::Way const &) { + return wkb_t(); } -way_helper::~way_helper() + +geometry_processor::wkbs_t +geometry_processor::process_relation(osmium::Relation const &, + osmium::memory::Buffer const &) { + return wkbs_t(); } -size_t way_helper::set(osmium::WayNodeList const &node_ids, - middle_query_t const *mid, reprojection const *proj) -{ - node_cache.clear(); - mid->nodes_get_list(node_cache, node_ids, proj); - // equivalent to returning node_count for complete ways, different for partial - // extracts - return node_cache.size(); -} relation_helper::relation_helper() : data(1024, osmium::memory::Buffer::auto_grow::yes) {} -relation_helper::~relation_helper() = default; size_t relation_helper::set(osmium::RelationMemberList const &member_list, middle_t const *mid) { @@ -100,13 +88,10 @@ size_t relation_helper::set(osmium::RelationMemberList const &member_list, middl roles.clear(); //grab the way members' ids - size_t num_input = member_list.size(); - input_way_ids.reserve(num_input); - roles.reserve(num_input); - for (auto const &member : member_list) { - if (member.type() == osmium::item_type::way) { - input_way_ids.push_back(member.ref()); - roles.push_back(member.role()); + for (auto const &m : member_list) { + /* Need to handle more than just ways... */ + if (m.type() == osmium::item_type::way) { + input_way_ids.push_back(m.ref()); } } @@ -118,21 +103,14 @@ size_t relation_helper::set(osmium::RelationMemberList const &member_list, middl auto num_ways = mid->ways_get_list(input_way_ids, data); //grab the roles of each way - if (num_ways < input_way_ids.size()) { - size_t memberpos = 0; - size_t waypos = 0; - for (auto const &w : data.select()) { - while (memberpos < input_way_ids.size()) { - if (input_way_ids[memberpos] == w.id()) { - roles[waypos] = roles[memberpos]; - ++memberpos; - break; - } - ++memberpos; + for (auto const &w : data.select()) { + for (auto const &member : member_list) { + if (member.ref() == w.id() && + member.type() == osmium::item_type::way) { + roles.emplace_back(member.role()); + break; } - ++waypos; } - roles.resize(num_ways); } //mark the ends of each so whoever uses them will know where they end.. @@ -147,21 +125,15 @@ multitaglist_t relation_helper::get_filtered_tags(tagtransform *transform, expor size_t i = 0; for (auto const &w : data.select()) { - transform->filter_tags(w, nullptr, nullptr, el, filtered[i++]); + transform->filter_tags(w, nullptr, nullptr, el, filtered[++i]); } return filtered; } -multinodelist_t relation_helper::get_nodes(middle_t const *mid, - reprojection const *proj) const +void relation_helper::add_way_locations(middle_t const *mid) { - multinodelist_t nodes(roles.size()); - - size_t i = 0; - for (auto const &w : data.select()) { - mid->nodes_get_list(nodes[i++], w.nodes(), proj); + for (auto &w : data.select()) { + mid->nodes_get_list(&(w.nodes())); } - - return nodes; } diff --git a/geometry-processor.hpp b/geometry-processor.hpp index efb876bad..0cf66793f 100644 --- a/geometry-processor.hpp +++ b/geometry-processor.hpp @@ -8,7 +8,7 @@ #include -#include "geometry-builder.hpp" +#include "osmium-builder.hpp" #include "osmtypes.hpp" #include "tagtransform.hpp" @@ -18,6 +18,8 @@ struct options_t; class reprojection; struct geometry_processor { + using wkb_t = geom::osmium_builder_t::wkb_t; + using wkbs_t = geom::osmium_builder_t::wkbs_t; // factory method for creating various types of geometry processors either by name or by geometry column type static std::shared_ptr create(const std::string &type, const options_t *options); @@ -45,17 +47,18 @@ struct geometry_processor { // process a node, optionally returning a WKB string describing // geometry to be inserted into the table. - virtual geometry_builder::pg_geom_t process_node(double lat, double lon); + virtual wkb_t process_node(osmium::Location const &loc); // process a way // position data and optionally returning WKB-encoded geometry // for insertion into the table. - virtual geometry_builder::pg_geom_t process_way(const nodelist_t &nodes); + virtual wkb_t process_way(osmium::Way const &way); // process a way, taking a middle query object to get way and // node position data. optionally returns an array of WKB-encoded geometry // for insertion into the table. - virtual geometry_builder::pg_geoms_t process_relation(const multinodelist_t &nodes); + virtual wkbs_t process_relation(osmium::Relation const &rel, + osmium::memory::Buffer const &ways); // returns the SRID of the output geometry. int srid() const; @@ -75,29 +78,15 @@ struct geometry_processor { }; -//various bits for continuous processing of ways -struct way_helper -{ - way_helper(); - ~way_helper(); - size_t set(osmium::WayNodeList const &node_ids, middle_query_t const *mid, - reprojection const *proj); - - nodelist_t node_cache; -}; - //various bits for continuous processing of members of relations -struct relation_helper +class relation_helper { +public: relation_helper(); - ~relation_helper(); + size_t set(osmium::RelationMemberList const &member_list, middle_t const *mid); multitaglist_t get_filtered_tags(tagtransform *transform, export_list const &el) const; - multinodelist_t get_nodes(middle_t const *mid, - reprojection const *proj) const; - - osmium::memory::ItemIteratorRange way_iterator() const - { return data.select(); } + void add_way_locations(middle_t const *mid); rolelist_t roles; std::vector superseded; diff --git a/osmium-builder.cpp b/osmium-builder.cpp index 00a1e396e..37863c1f8 100644 --- a/osmium-builder.cpp +++ b/osmium-builder.cpp @@ -59,7 +59,7 @@ osmium_builder_t::get_wkb_node(osmium::Location const &loc) const } osmium_builder_t::wkbs_t -osmium_builder_t::get_wkb_split(osmium::WayNodeList const &nodes) +osmium_builder_t::get_wkb_line(osmium::WayNodeList const &nodes, bool do_split) { wkbs_t ret; @@ -79,40 +79,43 @@ osmium_builder_t::get_wkb_split(osmium::WayNodeList const &nodes) if (prev_pt == this_pt) { continue; } - double const delta = distance(prev_pt, this_pt); - - // figure out if the addition of this point would take the total - // length of the line in `segment` over the `split_at` distance. - - if (dist + delta > split_at) { - size_t const splits = - (size_t)std::floor((dist + delta) / split_at); - // use the splitting distance to split the current segment up - // into as many parts as necessary to keep each part below - // the `split_at` distance. - osmium::geom::Coordinates ipoint; - for (size_t j = 0; j < splits; ++j) { - double const frac = - ((double)(j + 1) * split_at - dist) / delta; - ipoint = interpolate(this_pt, prev_pt, frac); - m_writer.linestring_add_location(ipoint); - ret.push_back(m_writer.linestring_finish(curlen + 1)); - // start a new segment - m_writer.linestring_start(); - m_writer.linestring_add_location(ipoint); - curlen = 1; - } - // reset the distance based on the final splitting point for - // the next iteration. - if (this_pt == ipoint) { - dist = 0; - m_writer.linestring_start(); - curlen = 0; + + if (do_split) { + double const delta = distance(prev_pt, this_pt); + + // figure out if the addition of this point would take the total + // length of the line in `segment` over the `split_at` distance. + + if (dist + delta > split_at) { + size_t const splits = + (size_t)std::floor((dist + delta) / split_at); + // use the splitting distance to split the current segment up + // into as many parts as necessary to keep each part below + // the `split_at` distance. + osmium::geom::Coordinates ipoint; + for (size_t j = 0; j < splits; ++j) { + double const frac = + ((double)(j + 1) * split_at - dist) / delta; + ipoint = interpolate(this_pt, prev_pt, frac); + m_writer.linestring_add_location(ipoint); + ret.push_back(m_writer.linestring_finish(curlen + 1)); + // start a new segment + m_writer.linestring_start(); + m_writer.linestring_add_location(ipoint); + curlen = 1; + } + // reset the distance based on the final splitting point for + // the next iteration. + if (this_pt == ipoint) { + dist = 0; + m_writer.linestring_start(); + curlen = 0; + } else { + dist = distance(this_pt, ipoint); + } } else { - dist = distance(this_pt, ipoint); + dist += delta; } - } else { - dist += delta; } } @@ -164,10 +167,9 @@ osmium_builder_t::get_wkb_multipolygon(osmium::Relation const &rel, } osmium_builder_t::wkbs_t -osmium_builder_t::get_wkb_multiline(osmium::memory::Buffer const &ways, bool) +osmium_builder_t::get_wkb_multiline(osmium::memory::Buffer const &ways, + bool do_split) { - wkbs_t ret; - // make a list of all endpoints using endpoint_t = std::tuple; std::vector endpoints; @@ -215,10 +217,10 @@ osmium_builder_t::get_wkb_multiline(osmium::memory::Buffer const &ways, bool) } } - // XXX need to do non-split version + wkbs_t ret; + size_t done_ways = 0; size_t todo_ways = conns.size(); - wkbs_t linewkbs; for (size_t i = 0; i < todo_ways; ++i) { if (!std::get<1>(conns[i]) || (std::get<0>(conns[i]) != NOCONN && std::get<2>(conns[i]) != NOCONN)) { @@ -254,10 +256,10 @@ osmium_builder_t::get_wkb_multiline(osmium::memory::Buffer const &ways, bool) // found a line end, create the wkbs m_buffer.commit(); - linewkbs = get_wkb_split(m_buffer.get(0)); + auto linewkbs = + get_wkb_line(m_buffer.get(0), do_split); std::move(linewkbs.begin(), linewkbs.end(), std::inserter(ret, ret.end())); - linewkbs.clear(); } if (done_ways < todo_ways) { @@ -298,13 +300,23 @@ osmium_builder_t::get_wkb_multiline(osmium::memory::Buffer const &ways, bool) // found a line end, create the wkbs m_buffer.commit(); - linewkbs = get_wkb_split(m_buffer.get(0)); + auto linewkbs = + get_wkb_line(m_buffer.get(0), do_split); std::move(linewkbs.begin(), linewkbs.end(), std::inserter(ret, ret.end())); - linewkbs.clear(); } } + if (!do_split && !ret.empty()) { + auto num_lines = ret.size(); + m_writer.multilinestring_start(); + for (auto const &line : ret) { + m_writer.add_part(line); + } + ret.clear(); + ret.push_back(m_writer.multilinestring_finish(num_lines)); + } + return ret; } diff --git a/osmium-builder.hpp b/osmium-builder.hpp index 57bb45f72..d4f1e9c35 100644 --- a/osmium-builder.hpp +++ b/osmium-builder.hpp @@ -1,3 +1,6 @@ +#ifndef OSMIUM_BUILDER_H +#define OSMIUM_BUILDER_H + #include #include #include @@ -25,7 +28,7 @@ class osmium_builder_t } wkb_t get_wkb_node(osmium::Location const &loc) const; - wkbs_t get_wkb_split(osmium::WayNodeList const &way); + wkbs_t get_wkb_line(osmium::WayNodeList const &way, bool do_split); wkb_t get_wkb_polygon(osmium::Way const &way); wkbs_t get_wkb_multipolygon(osmium::Relation const &rel, @@ -45,3 +48,5 @@ class osmium_builder_t }; } // namespace + +#endif // OSMIUM_BUILDER_H diff --git a/output-gazetteer.cpp b/output-gazetteer.cpp index 3715a77bc..aec9eace9 100644 --- a/output-gazetteer.cpp +++ b/output-gazetteer.cpp @@ -5,7 +5,6 @@ #include "osmtypes.hpp" #include "middle.hpp" #include "pgsql.hpp" -#include "reprojection.hpp" #include "output-gazetteer.hpp" #include "options.hpp" #include "util.hpp" @@ -37,6 +36,7 @@ #define CREATE_PLACE_ID_INDEX \ "CREATE INDEX place_id_idx ON place USING BTREE (osm_type, osm_id) %s %s" +static const char *lookup_hex = "0123456789ABCDEF"; void place_tag_processor::clear() { @@ -486,9 +486,11 @@ void place_tag_processor::copy_out(osmium::OSMObject const &o, } buffer += "\t"; } - // geometry - buffer += srid_str; - buffer += geom; + // add the geometry - encoding it to hex along the way + for (char c : geom) { + buffer += lookup_hex[(c >> 4) & 0xf]; + buffer += lookup_hex[c & 0xf]; + } buffer += '\n'; } } @@ -611,7 +613,6 @@ int output_gazetteer_t::connect() { int output_gazetteer_t::start() { int srid = m_options.projection->target_srs(); - builder.set_exclude_broken_polygon(m_options.excludepoly); places.srid_str = (boost::format("SRID=%1%;") % srid).str(); @@ -675,9 +676,8 @@ int output_gazetteer_t::process_node(osmium::Node const &node) /* Are we interested in this item? */ if (places.has_data()) { - auto c = reproj->reproject(node.location()); - std::string wkt = (point_fmt % c.x % c.y).str(); - places.copy_out(node, wkt, buffer); + auto wkb = m_builder.get_wkb_node(node.location()); + places.copy_out(node, wkb, buffer); flush_place_buffer(); } @@ -695,14 +695,24 @@ int output_gazetteer_t::process_way(osmium::Way *way) if (places.has_data()) { /* Fetch the node details */ nodelist_t nodes; - m_mid->nodes_get_list(nodes, way->nodes(), reproj.get()); + m_mid->nodes_get_list(&(way->nodes())); /* Get the geometry of the object */ - auto geom = builder.get_wkb_simple(nodes, 1); - if (geom.valid()) { - places.copy_out(*way, geom.geom, buffer); - flush_place_buffer(); + geom::osmium_builder_t::wkb_t geom; + if (way->is_closed()) { + geom = m_builder.get_wkb_polygon(*way); + } + if (geom.empty()) { + auto wkbs = m_builder.get_wkb_line(way->nodes(), false); + if (wkbs.empty()) { + return 0; + } + + geom = wkbs[0]; } + + places.copy_out(*way, geom, buffer); + flush_place_buffer(); } return 0; @@ -743,38 +753,25 @@ int output_gazetteer_t::process_relation(osmium::Relation const &rel) xid2.push_back(member.ref()); } - if (xid2.empty()) { + osmium_buffer.clear(); + if (xid2.empty() || (m_mid->ways_get_list(xid2, osmium_buffer) == 0)) { if (m_options.append) delete_unused_full('R', rel.id()); return 0; } - osmium_buffer.clear(); - auto num_ways = m_mid->ways_get_list(xid2, osmium_buffer); - multinodelist_t xnodes(num_ways); - size_t i = 0; - for (auto const &w : osmium_buffer.select()) { - m_mid->nodes_get_list(xnodes[i++], w.nodes(), reproj.get()); + for (auto &w : osmium_buffer.select()) { + m_mid->nodes_get_list(&(w.nodes())); } - if (!is_waterway) { - auto geoms = builder.build_both(xnodes, 1, 1, 1000000, rel.id()); - for (const auto& geom: geoms) { - if (geom.is_polygon()) { - places.copy_out(rel, geom.geom, buffer); - flush_place_buffer(); - } else { - /* add_polygon_error('R', id, "boundary", "adminitrative", &names, countrycode, wkt); */ - } - } - } else { - /* waterways result in multilinestrings */ - auto geom = builder.build_multilines(xnodes, rel.id()); - if (geom.valid()) { - places.copy_out(rel, geom.geom, buffer); - flush_place_buffer(); - } + auto geoms = is_waterway + ? m_builder.get_wkb_multiline(osmium_buffer, false) + : m_builder.get_wkb_multipolygon(rel, osmium_buffer); + + if (!geoms.empty()) { + places.copy_out(rel, geoms[0], buffer); + flush_place_buffer(); } return 0; diff --git a/output-gazetteer.hpp b/output-gazetteer.hpp index e59fb8abc..57cd82d98 100644 --- a/output-gazetteer.hpp +++ b/output-gazetteer.hpp @@ -8,7 +8,7 @@ #include #include -#include "geometry-builder.hpp" +#include "osmium-builder.hpp" #include "osmtypes.hpp" #include "output.hpp" #include "pgsql.hpp" @@ -130,33 +130,22 @@ class place_tag_processor class output_gazetteer_t : public output_t { public: - output_gazetteer_t(const middle_query_t* mid_, const options_t &options_) - : output_t(mid_, options_), - Connection(NULL), - ConnectionDelete(NULL), - ConnectionError(NULL), - copy_active(false), - reproj(options_.projection), - single_fmt("%1%"), - point_fmt("POINT(%.15g %.15g)"), + output_gazetteer_t(const middle_query_t *mid_, const options_t &options_) + : output_t(mid_, options_), Connection(NULL), ConnectionDelete(NULL), + ConnectionError(NULL), copy_active(false), + m_builder(options_.projection, true), single_fmt("%1%"), osmium_buffer(PLACE_BUFFER_SIZE, osmium::memory::Buffer::auto_grow::yes) { buffer.reserve(PLACE_BUFFER_SIZE); } - output_gazetteer_t(const output_gazetteer_t& other) - : output_t(other.m_mid, other.m_options), - Connection(NULL), - ConnectionDelete(NULL), - ConnectionError(NULL), - copy_active(false), - reproj(other.reproj), - single_fmt(other.single_fmt), - point_fmt(other.point_fmt), + output_gazetteer_t(const output_gazetteer_t &other) + : output_t(other.m_mid, other.m_options), Connection(NULL), + ConnectionDelete(NULL), ConnectionError(NULL), copy_active(false), + m_builder(other.m_options.projection, true), single_fmt(other.single_fmt), osmium_buffer(PLACE_BUFFER_SIZE, osmium::memory::Buffer::auto_grow::yes) { buffer.reserve(PLACE_BUFFER_SIZE); - builder.set_exclude_broken_polygon(m_options.excludepoly); connect(); } @@ -267,14 +256,11 @@ class output_gazetteer_t : public output_t { std::string buffer; place_tag_processor places; - geometry_builder builder; - - std::shared_ptr reproj; + geom::osmium_builder_t m_builder; // string formatters // Need to be part of the class, so we have one per thread. boost::format single_fmt; - boost::format point_fmt; osmium::memory::Buffer osmium_buffer; }; diff --git a/output-multi.cpp b/output-multi.cpp index 649e113e3..ce6d92be8 100644 --- a/output-multi.cpp +++ b/output-multi.cpp @@ -1,12 +1,13 @@ #include "output-multi.hpp" -#include "taginfo_impl.hpp" +#include "expire-tiles.hpp" +#include "geometry-builder.hpp" +#include "id-tracker.hpp" +#include "middle.hpp" +#include "options.hpp" #include "table.hpp" +#include "taginfo_impl.hpp" #include "tagtransform.hpp" -#include "options.hpp" -#include "middle.hpp" -#include "id-tracker.hpp" -#include "geometry-builder.hpp" -#include "expire-tiles.hpp" +#include "wkb-parser.hpp" #include #include @@ -117,7 +118,7 @@ int output_multi_t::pending_way(osmid_t id, int exists) { buffer.clear(); if (m_mid->ways_get(id, buffer)) { // Output the way - ret = reprocess_way(buffer.get(0), exists); + ret = reprocess_way(&buffer.get(0), exists); } return ret; @@ -282,24 +283,24 @@ int output_multi_t::process_node(osmium::Node const &node) auto filter = m_tagtransform->filter_tags(node, 0, 0, *m_export_list.get(), outtags, true); if (!filter) { - auto c = m_proj->reproject(node.location()); // grab its geom - auto geom = m_processor->process_node(c.y, c.x); - if (geom.valid()) { - m_expire.from_bbox(c.x, c.y, c.x, c.y); - copy_node_to_table(node.id(), geom.geom, outtags); + auto geom = m_processor->process_node(node.location()); + if (!geom.empty()) { + m_expire.from_wkb(geom.c_str(), node.id()); + copy_node_to_table(node.id(), geom, outtags); } } return 0; } -int output_multi_t::reprocess_way(osmium::Way const &way, bool exists) +int output_multi_t::reprocess_way(osmium::Way *way, bool exists) { //if the way could exist already we have to make the relation pending and reprocess it later //but only if we actually care about relations if(m_processor->interests(geometry_processor::interest_relation) && exists) { - way_delete(way.id()); - const std::vector rel_ids = m_mid->relations_using_way(way.id()); + way_delete(way->id()); + const std::vector rel_ids = + m_mid->relations_using_way(way->id()); for (std::vector::const_iterator itr = rel_ids.begin(); itr != rel_ids.end(); ++itr) { rels_pending_tracker.mark(*itr); } @@ -308,15 +309,13 @@ int output_multi_t::reprocess_way(osmium::Way const &way, bool exists) //check if we are keeping this way int polygon = 0; taglist_t outtags; - unsigned int filter = m_tagtransform->filter_tags(way, &polygon, 0, - *m_export_list.get(), outtags, true); + unsigned int filter = m_tagtransform->filter_tags( + *way, &polygon, 0, *m_export_list.get(), outtags, true); if (!filter) { - //grab its geom - nodelist_t nodes; - m_mid->nodes_get_list(nodes, way.nodes(), m_proj.get()); - auto geom = m_processor->process_way(nodes); - if (geom.valid()) { - copy_to_table(way.id(), geom, outtags, polygon); + m_mid->nodes_get_list(&(way->nodes())); + auto geom = m_processor->process_way(*way); + if (!geom.empty()) { + copy_to_table(way->id(), geom, outtags, polygon); } } return 0; @@ -330,12 +329,12 @@ int output_multi_t::process_way(osmium::Way *way) { *m_export_list.get(), outtags, true); if (!filter) { //get the geom from the middle - if (m_way_helper.set(way->nodes(), m_mid, m_proj.get()) < 1) + if (m_mid->nodes_get_list(&(way->nodes())) < 1) return 0; //grab its geom - auto geom = m_processor->process_way(m_way_helper.node_cache); + auto geom = m_processor->process_way(*way); - if (geom.valid()) { + if (!geom.empty()) { //if we are also interested in relations we need to mark //this way pending just in case it shows up in one if (m_processor->interests(geometry_processor::interest_relation)) { @@ -384,7 +383,7 @@ int output_multi_t::process_relation(osmium::Relation const &rel, //if the export list did not include the type tag. //TODO: find a less hacky way to do the matching/superseded and tag copying stuff without //all this trickery - int make_boundary, make_polygon = 1; + int make_boundary, make_polygon; taglist_t outtags; filter = m_tagtransform->filter_rel_member_tags(rel_outtags, filtered, m_relation_helper.roles, &m_relation_helper.superseded.front(), @@ -392,23 +391,17 @@ int output_multi_t::process_relation(osmium::Relation const &rel, *m_export_list.get(), outtags, true); if (!filter) { - auto nodes = - m_relation_helper.get_nodes((middle_t *)m_mid, m_proj.get()); - auto geoms = m_processor->process_relation(nodes); + m_relation_helper.add_way_locations((middle_t *)m_mid); + auto geoms = + m_processor->process_relation(rel, m_relation_helper.data); for (const auto geom : geoms) { - // TODO: we actually have the nodes in the m_relation_helper and - // could use them - // instead of having to reparse the wkb in the expiry code - m_expire.from_wkb(geom.geom.c_str(), -rel.id()); - // what part of the code relies on relation members getting negative - // ids? copy_to_table(-rel.id(), geom, outtags, make_polygon); } //TODO: should this loop be inside the if above just in case? //take a look at each member to see if its superseded (tags on it matched the tags on the relation) size_t i = 0; - for (auto const &w : m_relation_helper.way_iterator()) { + for (auto const &w : m_relation_helper.data.select()) { //tags matched so we are keeping this one with this relation if (m_relation_helper.superseded[i]) { //just in case it wasnt previously with this relation we get rid of them @@ -426,8 +419,10 @@ int output_multi_t::process_relation(osmium::Relation const &rel, return 0; } -void output_multi_t::copy_node_to_table(osmid_t id, const std::string &geom, taglist_t &tags) { - m_table->write_row(id, tags, geom); +void output_multi_t::copy_node_to_table(osmid_t id, std::string const &geom, + taglist_t &tags) +{ + m_table->write_row_wkb(id, tags, geom); } /** @@ -439,25 +434,23 @@ void output_multi_t::copy_node_to_table(osmid_t id, const std::string &geom, tag * * \pre geom must be valid. */ -void output_multi_t::copy_to_table(const osmid_t id, const geometry_builder::pg_geom_t &geom, taglist_t &tags, int polygon) { - if (geom.is_polygon()) { +void output_multi_t::copy_to_table(const osmid_t id, + geometry_processor::wkb_t const &geom, + taglist_t &tags, int polygon) +{ + // XXX really should depend on expected output type + if (polygon) { // It's a polygon table (implied by it turning into a poly), // and it got formed into a polygon, so expire as a polygon and write the geom - m_expire.from_nodes_poly(m_way_helper.node_cache, id); - if (geom.area > 0.0) { - char tmp[32]; - snprintf(tmp, sizeof(tmp), "%g", geom.area); - tags.push_override(tag_t("way_area", tmp)); - } - m_table->write_row(id, tags, geom.geom); - } else { - // Linestring - if (!polygon) { - // non-polygons are okay - m_expire.from_nodes_line(m_way_helper.node_cache); - m_table->write_row(id, tags, geom.geom); - } + auto area = + ewkb_parser_t(geom).get_area(); + char tmp[32]; + snprintf(tmp, sizeof(tmp), "%g", area); + tags.push_override(tag_t("way_area", tmp)); } + + m_expire.from_wkb(geom.c_str(), id); + m_table->write_row_wkb(id, tags, geom); } void output_multi_t::delete_from_output(osmid_t id) { diff --git a/output-multi.hpp b/output-multi.hpp index 3ac13cbf0..804ebe0e0 100644 --- a/output-multi.hpp +++ b/output-multi.hpp @@ -66,10 +66,11 @@ class output_multi_t : public output_t { void delete_from_output(osmid_t id); int process_node(osmium::Node const &node); int process_way(osmium::Way *way); - int reprocess_way(osmium::Way const &way, bool exists); + int reprocess_way(osmium::Way *way, bool exists); int process_relation(osmium::Relation const &rel, bool exists, bool pending=false); void copy_node_to_table(osmid_t id, const std::string &geom, taglist_t &tags); - void copy_to_table(const osmid_t id, const geometry_builder::pg_geom_t &geom, taglist_t &tags, int polygon); + void copy_to_table(const osmid_t id, geometry_processor::wkb_t const &geom, + taglist_t &tags, int polygon); std::unique_ptr m_tagtransform; std::unique_ptr m_export_list; @@ -80,7 +81,6 @@ class output_multi_t : public output_t { id_tracker ways_pending_tracker, rels_pending_tracker; std::shared_ptr ways_done_tracker; expire_tiles m_expire; - way_helper m_way_helper; relation_helper m_relation_helper; osmium::memory::Buffer buffer; }; diff --git a/output-pgsql.cpp b/output-pgsql.cpp index bf28c96fb..6a2eee369 100644 --- a/output-pgsql.cpp +++ b/output-pgsql.cpp @@ -90,7 +90,7 @@ void output_pgsql_t::pgsql_out_way(osmium::Way const &way, taglist_t *tags, m_tables[t_poly]->write_row_wkb(way.id(), *tags, wkb); } } else { - for (auto const &wkb : m_builder.get_wkb_split(way.nodes())) { + for (auto const &wkb : m_builder.get_wkb_line(way.nodes(), true)) { expire.from_wkb(wkb.c_str(), way.id()); m_tables[t_line]->write_row_wkb(way.id(), *tags, wkb); if (roads) { @@ -321,43 +321,41 @@ int output_pgsql_t::pgsql_process_relation(osmium::Relation const &rel, if (m.type() == osmium::item_type::way) { xid2.push_back(m.ref()); } + } - } - - buffer.clear(); - auto num_ways = m_mid->ways_get_list(xid2, buffer); + buffer.clear(); + auto num_ways = m_mid->ways_get_list(xid2, buffer); - if (num_ways == 0) - return 0; + if (num_ways == 0) + return 0; - multitaglist_t xtags(num_ways, taglist_t()); - rolelist_t xrole(num_ways); - - size_t i = 0; - for (auto const &w : buffer.select()) { - assert(i < num_ways); - - //filter the tags on this member because we got it from the middle - //and since the middle is no longer tied to the output it no longer - //shares any kind of tag transform and therefore all original tags - //will come back and need to be filtered by individual outputs before - //using these ways - m_tagtransform->filter_tags(w, nullptr, nullptr, *m_export_list.get(), xtags[i]); - //TODO: if the filter says that this member is now not interesting we - //should decrement the count and remove his nodes and tags etc. for - //now we'll just keep him with no tags so he will get filtered later - for (auto const &member : rel.members()) { - if (member.ref() == w.id() && member.type() == osmium::item_type::way) { - xrole[i] = member.role(); - break; - } - } - ++i; + multitaglist_t xtags(num_ways, taglist_t()); + rolelist_t xrole(num_ways); + + size_t i = 0; + for (auto const &w : buffer.select()) { + assert(i < num_ways); + + //filter the tags on this member because we got it from the middle + //and since the middle is no longer tied to the output it no longer + //shares any kind of tag transform and therefore all original tags + //will come back and need to be filtered by individual outputs before + //using these ways + m_tagtransform->filter_tags(w, nullptr, nullptr, *m_export_list.get(), + xtags[i]); + //TODO: if the filter says that this member is now not interesting we + //should decrement the count and remove his nodes and tags etc. for + //now we'll just keep him with no tags so he will get filtered later + for (auto const &member : rel.members()) { + if (member.ref() == w.id() && + member.type() == osmium::item_type::way) { + xrole[i] = member.role(); + break; + } + } + ++i; } - ////// pgsql_out_relation - //pgsql_out_relation(rel, outtags, xtags, xid, xrole, pending); - int roads = 0; int make_polygon = 0; int make_boundary = 0; diff --git a/output-pgsql.hpp b/output-pgsql.hpp index ccbee75be..561fb908e 100644 --- a/output-pgsql.hpp +++ b/output-pgsql.hpp @@ -10,7 +10,6 @@ #include "id-tracker.hpp" #include "osmium-builder.hpp" #include "output.hpp" -#include "reprojection.hpp" #include "table.hpp" #include "tagtransform.hpp" diff --git a/processor-line.cpp b/processor-line.cpp index bc1b23aa4..3c971039c 100644 --- a/processor-line.cpp +++ b/processor-line.cpp @@ -1,19 +1,23 @@ #include "processor-line.hpp" -processor_line::processor_line(int srid) : geometry_processor(srid, "LINESTRING", interest_way | interest_relation ) +processor_line::processor_line(std::shared_ptr const &proj) +: geometry_processor(proj->target_srs(), "LINESTRING", + interest_way | interest_relation), + m_builder(proj, false) { } -processor_line::~processor_line() +geometry_processor::wkb_t processor_line::process_way(osmium::Way const &way) { -} + auto wkbs = m_builder.get_wkb_line(way.nodes(), false); -geometry_builder::pg_geom_t processor_line::process_way(const nodelist_t &nodes) -{ - return builder.get_wkb_simple(nodes, false); + return wkbs.empty() ? wkb_t() : wkbs[0]; } -geometry_builder::pg_geoms_t processor_line::process_relation(const multinodelist_t &nodes) +geometry_processor::wkbs_t +processor_line::process_relation(osmium::Relation const &, + osmium::memory::Buffer const &ways) { - return builder.build_both(nodes, false, false, 1000000); + // XXX are multilines really acceptable? + return m_builder.get_wkb_multiline(ways, false); } diff --git a/processor-line.hpp b/processor-line.hpp index 1a011b658..883290241 100644 --- a/processor-line.hpp +++ b/processor-line.hpp @@ -3,15 +3,17 @@ #include "geometry-processor.hpp" -struct processor_line : public geometry_processor { - processor_line(int srid); - virtual ~processor_line(); +class processor_line : public geometry_processor +{ +public: + processor_line(std::shared_ptr const &proj); - geometry_builder::pg_geom_t process_way(const nodelist_t &nodes); - geometry_builder::pg_geoms_t process_relation(const multinodelist_t &nodes); + wkb_t process_way(osmium::Way const &way) override; + wkbs_t process_relation(osmium::Relation const &rel, + osmium::memory::Buffer const &ways) override; private: - geometry_builder builder; + geom::osmium_builder_t m_builder; }; #endif /* PROCESSOR_LINE_HPP */ diff --git a/processor-point.cpp b/processor-point.cpp index d40472536..faf66b327 100644 --- a/processor-point.cpp +++ b/processor-point.cpp @@ -5,14 +5,14 @@ #include "processor-point.hpp" #include "util.hpp" -processor_point::processor_point(int srid) - : geometry_processor(srid, "POINT", interest_node) { -} - -processor_point::~processor_point() { +processor_point::processor_point(std::shared_ptr const &proj) +: geometry_processor(proj->target_srs(), "POINT", interest_node), + m_builder(proj, false) +{ } -geometry_builder::pg_geom_t processor_point::process_node(double lat, double lon) +geometry_processor::wkb_t +processor_point::process_node(osmium::Location const &loc) { - return geometry_builder::pg_geom_t((boost::format("POINT(%.15g %.15g)") % lon % lat).str(), false); + return m_builder.get_wkb_node(loc); } diff --git a/processor-point.hpp b/processor-point.hpp index 6cdf58ea9..c265bbc64 100644 --- a/processor-point.hpp +++ b/processor-point.hpp @@ -3,11 +3,15 @@ #include "geometry-processor.hpp" -struct processor_point : public geometry_processor { - processor_point(int srid); - virtual ~processor_point(); +class processor_point : public geometry_processor +{ +public: + processor_point(std::shared_ptr const &proj); - geometry_builder::pg_geom_t process_node(double lat, double lon); + wkb_t process_node(osmium::Location const &loc) override; + +private: + geom::osmium_builder_t m_builder; }; #endif /* PROCESSOR_POINT_HPP */ diff --git a/processor-polygon.cpp b/processor-polygon.cpp index 73c339219..35cf5e195 100644 --- a/processor-polygon.cpp +++ b/processor-polygon.cpp @@ -1,19 +1,21 @@ #include "processor-polygon.hpp" -processor_polygon::processor_polygon(int srid, bool enable_multi) : geometry_processor(srid, "GEOMETRY", interest_way | interest_relation), enable_multi(enable_multi) +processor_polygon::processor_polygon(std::shared_ptr const &proj, + bool enable_multi) +: geometry_processor(proj->target_srs(), "GEOMETRY", + interest_way | interest_relation), + m_builder(proj, enable_multi) { } -processor_polygon::~processor_polygon() +geometry_processor::wkb_t processor_polygon::process_way(osmium::Way const &way) { + return m_builder.get_wkb_polygon(way); } -geometry_builder::pg_geom_t processor_polygon::process_way(const nodelist_t &nodes) +geometry_processor::wkbs_t +processor_polygon::process_relation(osmium::Relation const &rel, + osmium::memory::Buffer const &ways) { - return builder.get_wkb_simple(nodes, true); -} - -geometry_builder::pg_geoms_t processor_polygon::process_relation(const multinodelist_t &nodes) -{ - return builder.build_polygons(nodes, enable_multi, -1); + return m_builder.get_wkb_multipolygon(rel, ways); } diff --git a/processor-polygon.hpp b/processor-polygon.hpp index 6d562ebfe..d2273e412 100644 --- a/processor-polygon.hpp +++ b/processor-polygon.hpp @@ -3,16 +3,18 @@ #include "geometry-processor.hpp" -struct processor_polygon : public geometry_processor { - processor_polygon(int srid, bool enable_multi); - virtual ~processor_polygon(); +class processor_polygon : public geometry_processor +{ +public: + processor_polygon(std::shared_ptr const &proj, + bool enable_multi); - geometry_builder::pg_geom_t process_way(const nodelist_t &nodes); - geometry_builder::pg_geoms_t process_relation(const multinodelist_t &nodes); + wkb_t process_way(osmium::Way const &nodes) override; + wkbs_t process_relation(osmium::Relation const &rel, + osmium::memory::Buffer const &ways) override; private: - bool enable_multi; - geometry_builder builder; + geom::osmium_builder_t m_builder; }; #endif /* PROCESSOR_POLYGON_HPP */ diff --git a/tests/test-output-multi-line.cpp b/tests/test-output-multi-line.cpp index 57f0767f5..9208655b7 100644 --- a/tests/test-output-multi-line.cpp +++ b/tests/test-output-multi-line.cpp @@ -56,7 +56,7 @@ int main(int argc, char *argv[]) { // start a new connection to run tests on db->check_count(1, "select count(*) from pg_catalog.pg_class where relname = 'foobar_highways'"); - db->check_count(2752, "select count(*) from foobar_highways"); + db->check_count(2753, "select count(*) from foobar_highways"); //check that we have the right spread db->check_count(13, "select count(*) from foobar_highways where highway='bridleway'"); @@ -65,7 +65,7 @@ int main(int argc, char *argv[]) { db->check_count(249, "select count(*) from foobar_highways where highway='footway'"); db->check_count(18, "select count(*) from foobar_highways where highway='living_street'"); db->check_count(171, "select count(*) from foobar_highways where highway='path'"); - db->check_count(5, "select count(*) from foobar_highways where highway='pedestrian'"); + db->check_count(6, "select count(*) from foobar_highways where highway='pedestrian'"); db->check_count(81, "select count(*) from foobar_highways where highway='primary'"); db->check_count(842, "select count(*) from foobar_highways where highway='residential'"); db->check_count(3, "select count(*) from foobar_highways where highway='road'"); From 9bab69eb0316e423baf61cb336c8fe59fa82d4bd Mon Sep 17 00:00:00 2001 From: Sarah Hoffmann Date: Fri, 3 Feb 2017 23:43:35 +0100 Subject: [PATCH 08/30] move wkb writing in a separate class Role out our own wkb writer class instead of using libosmiums internal implementation details. Allows a bit of stream-lining and removes some of the hacks added to the libosmium copy. --- contrib/libosmium/osmium/geom/wkb.hpp | 40 --- expire-tiles.cpp | 20 +- expire-tiles.hpp | 10 +- osmium-builder.cpp | 79 +++--- osmium-builder.hpp | 9 +- output-gazetteer.cpp | 14 +- output-multi.cpp | 4 +- output-pgsql.cpp | 10 +- table.cpp | 23 +- wkb-parser.hpp | 154 ------------ wkb.hpp | 338 ++++++++++++++++++++++++++ 11 files changed, 423 insertions(+), 278 deletions(-) delete mode 100644 wkb-parser.hpp create mode 100644 wkb.hpp diff --git a/contrib/libosmium/osmium/geom/wkb.hpp b/contrib/libosmium/osmium/geom/wkb.hpp index 3f055a735..39105fc47 100644 --- a/contrib/libosmium/osmium/geom/wkb.hpp +++ b/contrib/libosmium/osmium/geom/wkb.hpp @@ -199,46 +199,6 @@ namespace osmium { } } - /* MultiLineString */ - - void multilinestring_start() { - m_data.clear(); - m_multipolygon_size_offset = header(m_data, wkbMultiLineString, true); - } - - void add_part(std::string const &part) { - m_data.append(part); - } - - linestring_type multilinestring_finish(size_t num_lines) { - set_size(m_multipolygon_size_offset, num_lines); - std::string data; - - using std::swap; - swap(data, m_data); - - if (m_out_type == out_type::hex) { - return convert_to_hex(data); - } else { - return data; - } - } - - /* Polygon */ - polygon_type polygon_finish() { - set_size(m_polygon_size_offset, m_rings); - std::string data; - - using std::swap; - swap(data, m_data); - - if (m_out_type == out_type::hex) { - return convert_to_hex(data); - } else { - return data; - } - } - /* MultiPolygon */ void multipolygon_start() { diff --git a/expire-tiles.cpp b/expire-tiles.cpp index 665b066a5..f8336bac5 100644 --- a/expire-tiles.cpp +++ b/expire-tiles.cpp @@ -18,7 +18,7 @@ #include "options.hpp" #include "reprojection.hpp" #include "table.hpp" -#include "wkb-parser.hpp" +#include "wkb.hpp" #define EARTH_CIRCUMFERENCE 40075016.68 #define HALF_EARTH_CIRCUMFERENCE (EARTH_CIRCUMFERENCE / 2) @@ -383,19 +383,19 @@ void expire_tiles::from_wkb(const char *wkb, osmid_t osm_id) return; } - auto parse = ewkb_parser_t(wkb); + auto parse = ewkb::parser_t(wkb); switch (parse.read_header()) { - case ewkb_parser_t::wkb_point: + case ewkb::wkb_point: from_wkb_point(&parse); break; - case ewkb_parser_t::wkb_line: + case ewkb::wkb_line: from_wkb_line(&parse); break; - case ewkb_parser_t::wkb_polygon: + case ewkb::wkb_polygon: from_wkb_polygon(&parse, osm_id); break; - case ewkb_parser_t::wkb_multi_line: { + case ewkb::wkb_multi_line: { auto num = parse.read_length(); for (unsigned i = 0; i < num; ++i) { parse.read_header(); @@ -403,7 +403,7 @@ void expire_tiles::from_wkb(const char *wkb, osmid_t osm_id) } break; } - case ewkb_parser_t::wkb_multi_polygon: { + case ewkb::wkb_multi_polygon: { auto num = parse.read_length(); for (unsigned i = 0; i < num; ++i) { parse.read_header(); @@ -418,13 +418,13 @@ void expire_tiles::from_wkb(const char *wkb, osmid_t osm_id) } } -void expire_tiles::from_wkb_point(ewkb_parser_t *wkb) +void expire_tiles::from_wkb_point(ewkb::parser_t *wkb) { auto c = wkb->read_point(); from_bbox(c.x, c.y, c.x, c.y); } -void expire_tiles::from_wkb_line(ewkb_parser_t *wkb) +void expire_tiles::from_wkb_line(ewkb::parser_t *wkb) { auto sz = wkb->read_length(); @@ -444,7 +444,7 @@ void expire_tiles::from_wkb_line(ewkb_parser_t *wkb) } } -void expire_tiles::from_wkb_polygon(ewkb_parser_t *wkb, osmid_t osm_id) +void expire_tiles::from_wkb_polygon(ewkb::parser_t *wkb, osmid_t osm_id) { auto num_rings = wkb->read_length(); assert(num_rings > 0); diff --git a/expire-tiles.hpp b/expire-tiles.hpp index c74d22141..448585778 100644 --- a/expire-tiles.hpp +++ b/expire-tiles.hpp @@ -8,7 +8,9 @@ class reprojection; class table_t; class tile; -class ewkb_parser_t; +namespace ewkb { +class parser_t; +} struct expire_tiles { @@ -49,9 +51,9 @@ struct expire_tiles int normalise_tile_x_coord(int x); void from_line(double lon_a, double lat_a, double lon_b, double lat_b); - void from_wkb_point(ewkb_parser_t *wkb); - void from_wkb_line(ewkb_parser_t *wkb); - void from_wkb_polygon(ewkb_parser_t *wkb, osmid_t osm_id); + void from_wkb_point(ewkb::parser_t *wkb); + void from_wkb_line(ewkb::parser_t *wkb); + void from_wkb_polygon(ewkb::parser_t *wkb, osmid_t osm_id); double tile_width; double max_bbox; diff --git a/osmium-builder.cpp b/osmium-builder.cpp index 37863c1f8..914ac8ff3 100644 --- a/osmium-builder.cpp +++ b/osmium-builder.cpp @@ -4,7 +4,6 @@ #include #include -#include #include "osmium-builder.hpp" @@ -50,8 +49,6 @@ inline void add_nodes_to_builder(osmium::builder::WayNodeListBuilder &builder, namespace geom { -using WKBWriter = osmium::geom::detail::WKBFactoryImpl; - osmium_builder_t::wkb_t osmium_builder_t::get_wkb_node(osmium::Location const &loc) const { @@ -97,11 +94,11 @@ osmium_builder_t::get_wkb_line(osmium::WayNodeList const &nodes, bool do_split) double const frac = ((double)(j + 1) * split_at - dist) / delta; ipoint = interpolate(this_pt, prev_pt, frac); - m_writer.linestring_add_location(ipoint); + m_writer.add_location(ipoint); ret.push_back(m_writer.linestring_finish(curlen + 1)); // start a new segment m_writer.linestring_start(); - m_writer.linestring_add_location(ipoint); + m_writer.add_location(ipoint); curlen = 1; } // reset the distance based on the final splitting point for @@ -119,7 +116,7 @@ osmium_builder_t::get_wkb_line(osmium::WayNodeList const &nodes, bool do_split) } } - m_writer.linestring_add_location(this_pt); + m_writer.add_location(this_pt); ++curlen; prev_pt = this_pt; @@ -311,7 +308,7 @@ osmium_builder_t::get_wkb_multiline(osmium::memory::Buffer const &ways, auto num_lines = ret.size(); m_writer.multilinestring_start(); for (auto const &line : ret) { - m_writer.add_part(line); + m_writer.add_sub_geometry(line); } ret.clear(); ret.push_back(m_writer.multilinestring_finish(num_lines)); @@ -320,17 +317,20 @@ osmium_builder_t::get_wkb_multiline(osmium::memory::Buffer const &ways, return ret; } -void osmium_builder_t::add_mp_points(const osmium::NodeRefList &nodes) +size_t osmium_builder_t::add_mp_points(const osmium::NodeRefList &nodes) { + size_t num_points = 0; osmium::Location last_location; for (const osmium::NodeRef &node_ref : nodes) { if (node_ref.location().valid() && last_location != node_ref.location()) { last_location = node_ref.location(); - m_writer.multipolygon_add_location( - m_proj->reproject(last_location)); + m_writer.add_location(m_proj->reproject(last_location)); + ++num_points; } } + + return num_points; } osmium_builder_t::wkb_t @@ -347,30 +347,33 @@ osmium_builder_t::create_multipolygon(osmium::Area const &area) if (it->type() == osmium::item_type::outer_ring) { auto &ring = static_cast(*it); if (num_polygons > 0) { - m_writer.multipolygon_polygon_finish(); + m_writer.multipolygon_polygon_finish(num_rings); + num_rings = 0; } - m_writer.multipolygon_polygon_start(); - m_writer.multipolygon_outer_ring_start(); - add_mp_points(ring); - m_writer.multipolygon_outer_ring_finish(); + m_writer.polygon_start(); + m_writer.polygon_ring_start(); + auto num_points = add_mp_points(ring); + m_writer.polygon_ring_finish(num_points); ++num_rings; - ++num_polygons; } else if (it->type() == osmium::item_type::inner_ring) { auto &ring = static_cast(*it); - m_writer.multipolygon_inner_ring_start(); - add_mp_points(ring); - m_writer.multipolygon_inner_ring_finish(); + m_writer.polygon_ring_start(); + auto num_points = add_mp_points(ring); + m_writer.polygon_ring_finish(num_points); ++num_rings; } } - // if there are no rings, this area is invalid - if (num_rings > 0) { - m_writer.multipolygon_polygon_finish(); - ret = m_writer.multipolygon_finish(); + // if there are no polygons, this area is invalid + if (num_polygons > 0) { + m_writer.multipolygon_polygon_finish(num_rings); + ret = m_writer.multipolygon_finish(num_polygons); + } else { + ret.clear(); } - } catch (osmium::geometry_error &e) { /* ignored */ + } catch (osmium::geometry_error &e) { + /* ignored */ } return ret; @@ -382,28 +385,32 @@ osmium_builder_t::create_polygons(osmium::Area const &area) wkbs_t ret; try { - size_t num_polygons = 0; + size_t num_rings = 0; for (auto it = area.cbegin(); it != area.cend(); ++it) { if (it->type() == osmium::item_type::outer_ring) { auto &ring = static_cast(*it); - if (num_polygons > 0) { - ret.push_back(m_writer.polygon_finish()); + if (num_rings > 0) { + ret.push_back(m_writer.polygon_finish(num_rings)); + num_rings = 0; } - m_writer.multipolygon_polygon_start(); - m_writer.multipolygon_outer_ring_start(); - add_mp_points(ring); - m_writer.multipolygon_outer_ring_finish(); - ++num_polygons; + m_writer.polygon_start(); + m_writer.polygon_ring_start(); + auto num_points = add_mp_points(ring); + m_writer.polygon_ring_finish(num_points); + ++num_rings; } else if (it->type() == osmium::item_type::inner_ring) { auto &ring = static_cast(*it); - m_writer.multipolygon_inner_ring_start(); - add_mp_points(ring); - m_writer.multipolygon_inner_ring_finish(); + m_writer.polygon_ring_start(); + auto num_points = add_mp_points(ring); + m_writer.polygon_ring_finish(num_points); + ++num_rings; } } - ret.push_back(m_writer.polygon_finish()); + if (num_rings > 0) { + ret.push_back(m_writer.polygon_finish(num_rings)); + } } catch (osmium::geometry_error &e) { /* ignored */ } diff --git a/osmium-builder.hpp b/osmium-builder.hpp index d4f1e9c35..0e5759b38 100644 --- a/osmium-builder.hpp +++ b/osmium-builder.hpp @@ -5,11 +5,11 @@ #include #include -#include #include #include #include "reprojection.hpp" +#include "wkb.hpp" namespace geom { @@ -22,8 +22,7 @@ class osmium_builder_t explicit osmium_builder_t(std::shared_ptr const &proj, bool build_multigeoms) : m_proj(proj), m_buffer(1024, osmium::memory::Buffer::auto_grow::yes), - m_writer(m_proj->epsg(), osmium::geom::wkb_type::ewkb), - m_build_multigeoms(build_multigeoms) + m_writer(m_proj->epsg()), m_build_multigeoms(build_multigeoms) { } @@ -38,12 +37,12 @@ class osmium_builder_t private: wkb_t create_multipolygon(osmium::Area const &area); wkbs_t create_polygons(osmium::Area const &area); - void add_mp_points(const osmium::NodeRefList &nodes); + size_t add_mp_points(const osmium::NodeRefList &nodes); std::shared_ptr m_proj; // internal buffer for creating areas osmium::memory::Buffer m_buffer; - osmium::geom::detail::WKBFactoryImpl m_writer; + ewkb::writer_t m_writer; bool m_build_multigeoms; }; diff --git a/output-gazetteer.cpp b/output-gazetteer.cpp index aec9eace9..8225004b2 100644 --- a/output-gazetteer.cpp +++ b/output-gazetteer.cpp @@ -2,12 +2,13 @@ #include #include -#include "osmtypes.hpp" #include "middle.hpp" -#include "pgsql.hpp" -#include "output-gazetteer.hpp" #include "options.hpp" +#include "osmtypes.hpp" +#include "output-gazetteer.hpp" +#include "pgsql.hpp" #include "util.hpp" +#include "wkb.hpp" #include #include @@ -36,8 +37,6 @@ #define CREATE_PLACE_ID_INDEX \ "CREATE INDEX place_id_idx ON place USING BTREE (osm_type, osm_id) %s %s" -static const char *lookup_hex = "0123456789ABCDEF"; - void place_tag_processor::clear() { // set members to sane defaults @@ -487,10 +486,7 @@ void place_tag_processor::copy_out(osmium::OSMObject const &o, buffer += "\t"; } // add the geometry - encoding it to hex along the way - for (char c : geom) { - buffer += lookup_hex[(c >> 4) & 0xf]; - buffer += lookup_hex[c & 0xf]; - } + ewkb::writer_t::write_as_hex(buffer, geom); buffer += '\n'; } } diff --git a/output-multi.cpp b/output-multi.cpp index ce6d92be8..03fd311fe 100644 --- a/output-multi.cpp +++ b/output-multi.cpp @@ -7,7 +7,7 @@ #include "table.hpp" #include "taginfo_impl.hpp" #include "tagtransform.hpp" -#include "wkb-parser.hpp" +#include "wkb.hpp" #include #include @@ -443,7 +443,7 @@ void output_multi_t::copy_to_table(const osmid_t id, // It's a polygon table (implied by it turning into a poly), // and it got formed into a polygon, so expire as a polygon and write the geom auto area = - ewkb_parser_t(geom).get_area(); + ewkb::parser_t(geom).get_area(); char tmp[32]; snprintf(tmp, sizeof(tmp), "%g", area); tags.push_override(tag_t("way_area", tmp)); diff --git a/output-pgsql.cpp b/output-pgsql.cpp index 6a2eee369..27d2f46e3 100644 --- a/output-pgsql.cpp +++ b/output-pgsql.cpp @@ -38,7 +38,7 @@ #include "tagtransform.hpp" #include "util.hpp" #include "wildcmp.hpp" -#include "wkb-parser.hpp" +#include "wkb.hpp" /* make the diagnostic information work with older versions of * boost - the function signature changed at version 1.54. @@ -81,9 +81,9 @@ void output_pgsql_t::pgsql_out_way(osmium::Way const &way, taglist_t *tags, if (m_enable_way_area) { auto const area = m_options.reproject_area - ? ewkb_parser_t(wkb).get_area( + ? ewkb::parser_t(wkb).get_area( m_options.projection.get()) - : ewkb_parser_t(wkb) + : ewkb::parser_t(wkb) .get_area(); tags->push_override(tag_t("way_area", std::to_string(area))); } @@ -384,9 +384,9 @@ int output_pgsql_t::pgsql_process_relation(osmium::Relation const &rel, if (m_enable_way_area) { auto const area = m_options.reproject_area - ? ewkb_parser_t(wkb).get_area( + ? ewkb::parser_t(wkb).get_area( m_options.projection.get()) - : ewkb_parser_t(wkb) + : ewkb::parser_t(wkb) .get_area(); outtags.push_override(tag_t("way_area", std::to_string(area))); } diff --git a/table.cpp b/table.cpp index a14cac991..f922de1d5 100644 --- a/table.cpp +++ b/table.cpp @@ -1,8 +1,3 @@ -#include "table.hpp" -#include "options.hpp" -#include "util.hpp" -#include "taginfo.hpp" - #include #include #include @@ -10,6 +5,12 @@ #include #include +#include "options.hpp" +#include "table.hpp" +#include "taginfo.hpp" +#include "util.hpp" +#include "wkb.hpp" + using std::string; typedef boost::format fmt; @@ -344,10 +345,8 @@ void table_t::write_row(const osmid_t id, const taglist_t &tags, const std::stri } } -static const char *lookup_hex = "0123456789ABCDEF"; - -void table_t::write_row_wkb(const osmid_t id, const taglist_t &tags, - const std::string &geom) +void table_t::write_row_wkb(osmid_t const id, taglist_t const &tags, + std::string const &geom) { //add the osm id buffer.append((single_fmt % id).str()); @@ -370,10 +369,8 @@ void table_t::write_row_wkb(const osmid_t id, const taglist_t &tags, write_tags_column(tags, buffer, used); //add the geometry - encoding it to hex along the way - for (char c : geom) { - buffer += lookup_hex[(c >> 4) & 0xf]; - buffer += lookup_hex[c & 0xf]; - } + ewkb::writer_t::write_as_hex(buffer, geom); + //we need \n because we are copying from stdin buffer.push_back('\n'); diff --git a/wkb-parser.hpp b/wkb-parser.hpp deleted file mode 100644 index 3a77a00c3..000000000 --- a/wkb-parser.hpp +++ /dev/null @@ -1,154 +0,0 @@ - -#include -#include -#include -#include - -#include -#include - -/** - * Class that allows to iterate over the elements of a ewkb geometry. - * - * Note: this class assumes that the wkb was created by osmium::geom::WKBFactory. - * It implements the exact opposite decoding. - */ -class ewkb_parser_t -{ -public: - enum geometry_type - { - wkb_point = 1, - wkb_line = 2, - wkb_polygon = 3, - wkb_multi_point = 4, - wkb_multi_line = 5, - wkb_multi_polygon = 6, - wkb_collection = 7, - }; - - explicit ewkb_parser_t(char const *wkb) : m_wkb(wkb), m_pos(0) {} - explicit ewkb_parser_t(std::string const &wkb) - : m_wkb(wkb.c_str()), m_pos(0) - { - } - - unsigned save_pos() const { return m_pos; } - void rewind(unsigned pos) { m_pos = pos; } - - int read_header() - { - m_pos += sizeof(uint8_t); // skip endianess - - auto type = read_data(); - - m_pos += sizeof(int); // skip srid - - return type & 0xff; - } - - uint32_t read_length() { return read_data(); } - - osmium::geom::Coordinates read_point() - { - auto x = read_data(); - auto y = read_data(); - - return osmium::geom::Coordinates(x, y); - } - - void skip_points(size_t num) { m_pos += sizeof(double) * 2 * num; } - - template - double get_area(PROJ *proj = nullptr) - { - double total = 0; - - auto type = read_header(); - - if (type == wkb_polygon) { - total = get_polygon_area(proj); - } else if (type == wkb_multi_polygon) { - auto num_poly = read_length(); - for (unsigned i = 0; i < num_poly; ++i) { - auto ptype = read_header(); - (void)ptype; - assert(ptype == wkb_polygon); - - total += get_polygon_area(proj); - } - } - - return total; - } - -private: - template - double get_polygon_area(PROJ *proj) - { - auto num_rings = read_length(); - assert(num_rings > 0); - - double total = get_ring_area(proj); - - for (unsigned i = 1; i < num_rings; ++i) { - total -= get_ring_area(proj); - } - - return total; - } - - template - double get_ring_area(PROJ *proj) - { - // Algorithm borrowed from - // http://stackoverflow.com/questions/451426/how-do-i-calculate-the-area-of-a-2d-polygon - // XXX numerically not stable (useless for latlon) - auto num_pts = read_length(); - assert(num_pts > 3); - - double total = 0; - - auto prev = read_point(); - proj->target_to_tile(&prev.y, &prev.x); - for (unsigned i = 1; i < num_pts; ++i) { - auto cur = read_point(); - proj->target_to_tile(&cur.y, &cur.x); - total += prev.x * cur.y - cur.x * prev.y; - prev = cur; - } - - return std::abs(total) * 0.5; - } - - double get_ring_area(osmium::geom::IdentityProjection *) - { - // Algorithm borrowed from - // http://stackoverflow.com/questions/451426/how-do-i-calculate-the-area-of-a-2d-polygon - auto num_pts = read_length(); - assert(num_pts > 3); - - double total = 0; - - auto prev = read_point(); - for (unsigned i = 1; i < num_pts; ++i) { - auto cur = read_point(); - total += prev.x * cur.y - cur.x * prev.y; - prev = cur; - } - - return std::abs(total) * 0.5; - } - - template - T read_data() - { - auto *data = reinterpret_cast(m_wkb + m_pos); - m_pos += sizeof(T); - - return *data; - } - - char const *m_wkb; - unsigned m_pos; -}; diff --git a/wkb.hpp b/wkb.hpp new file mode 100644 index 000000000..4569226d9 --- /dev/null +++ b/wkb.hpp @@ -0,0 +1,338 @@ +#ifndef OSM2PGSQL_WKB_HPP +#define OSM2PGSQL_WKB_HPP + +#include +#include +#include +#include + +#include +#include + +namespace ewkb { + +enum geometry_type : uint32_t +{ + wkb_point = 1, + wkb_line = 2, + wkb_polygon = 3, + wkb_multi_point = 4, + wkb_multi_line = 5, + wkb_multi_polygon = 6, + wkb_collection = 7, + + wkb_srid = 0x20000000 // SRID-presence flag (EWKB) +}; + +/** + * Writer for EWKB data suitable for postgres. + * + * Code has been largely derived from osmium::geom::WKBFactoryImpl. + */ +class writer_t +{ + enum wkb_byte_order_type_t : uint8_t + { + XDR = 0, // Big Endian + NDR = 1 // Little Endian + }; + + std::string m_data; + int m_srid; + + size_t m_geometry_size_offset = 0; + size_t m_multigeometry_size_offset = 0; + size_t m_ring_size_offset = 0; + + size_t header(std::string &str, geometry_type type, bool add_length) const + { +#if __BYTE_ORDER == __LITTLE_ENDIAN + str_push(str, NDR); +#else + str_push(str, XDR); +#endif + str_push(str, type | wkb_srid); + str_push(str, m_srid); + + const size_t offset = str.size(); + if (add_length) { + str_push(str, static_cast(0)); + } + return offset; + } + + void set_size(const size_t offset, const size_t size) + { + uint32_t s = static_cast(size); + std::copy_n(reinterpret_cast(&s), sizeof(uint32_t), + &m_data[offset]); + } + + template + inline static void str_push(std::string &str, T data) + { + str.append(reinterpret_cast(&data), sizeof(T)); + } + +public: + inline static void write_as_hex(std::string &out, std::string const &wkb) + { + static char const *lookup_hex = "0123456789ABCDEF"; + + for (char c : wkb) { + out += lookup_hex[(c >> 4) & 0xf]; + out += lookup_hex[c & 0xf]; + } + } + + explicit writer_t(int srid) : m_srid(srid) {} + + void add_sub_geometry(std::string const &part) { m_data.append(part); } + + void add_location(const osmium::geom::Coordinates &xy) + { + str_push(m_data, xy.x); + str_push(m_data, xy.y); + } + + /* Point */ + + std::string make_point(const osmium::geom::Coordinates &xy) const + { + std::string data; + header(data, wkb_point, false); + str_push(data, xy.x); + str_push(data, xy.y); + + return data; + } + + /* LineString */ + + void linestring_start() + { + m_geometry_size_offset = header(m_data, wkb_line, true); + } + + std::string linestring_finish(size_t num_points) + { + set_size(m_geometry_size_offset, num_points); + std::string data; + + using std::swap; + swap(data, m_data); + + return data; + } + + /* MultiLineString */ + + void multilinestring_start() + { + m_multigeometry_size_offset = header(m_data, wkb_multi_line, true); + } + + void multilinestring_line_finish(size_t num_points) + { + set_size(m_geometry_size_offset, num_points); + } + + std::string multilinestring_finish(size_t num_lines) + { + set_size(m_multigeometry_size_offset, num_lines); + std::string data; + + using std::swap; + swap(data, m_data); + + return data; + } + + /* Polygon */ + + void polygon_start() + { + m_geometry_size_offset = header(m_data, wkb_polygon, true); + } + + void polygon_ring_start() + { + m_ring_size_offset = m_data.size(); + str_push(m_data, static_cast(0)); + } + + void polygon_ring_finish(size_t num_points) + { + set_size(m_ring_size_offset, num_points); + } + + std::string polygon_finish(size_t num_rings) + { + set_size(m_geometry_size_offset, num_rings); + std::string data; + + using std::swap; + swap(data, m_data); + + return data; + } + + /* MultiPolygon */ + + void multipolygon_start() + { + m_multigeometry_size_offset = header(m_data, wkb_multi_polygon, true); + } + + void multipolygon_polygon_finish(size_t num_rings) + { + set_size(m_geometry_size_offset, num_rings); + } + + std::string multipolygon_finish(size_t num_polygons) + { + set_size(m_multigeometry_size_offset, num_polygons); + std::string data; + + using std::swap; + swap(data, m_data); + + return data; + } +}; + +/** + * Class that allows to iterate over the elements of a ewkb geometry. + * + * Note: this class assumes that the wkb was created by ewkb::writer_t. + * It implements the exact opposite decoding. + */ +class parser_t +{ +public: + explicit parser_t(char const *wkb) : m_wkb(wkb), m_pos(0) {} + explicit parser_t(std::string const &wkb) : m_wkb(wkb.c_str()), m_pos(0) {} + + unsigned save_pos() const { return m_pos; } + void rewind(unsigned pos) { m_pos = pos; } + + int read_header() + { + m_pos += sizeof(uint8_t); // skip endianess + + auto type = read_data(); + + m_pos += sizeof(int); // skip srid + + return type & 0xff; + } + + uint32_t read_length() { return read_data(); } + + osmium::geom::Coordinates read_point() + { + auto x = read_data(); + auto y = read_data(); + + return osmium::geom::Coordinates(x, y); + } + + void skip_points(size_t num) { m_pos += sizeof(double) * 2 * num; } + + template + double get_area(PROJ *proj = nullptr) + { + double total = 0; + + auto type = read_header(); + + if (type == wkb_polygon) { + total = get_polygon_area(proj); + } else if (type == wkb_multi_polygon) { + auto num_poly = read_length(); + for (unsigned i = 0; i < num_poly; ++i) { + auto ptype = read_header(); + (void)ptype; + assert(ptype == wkb_polygon); + + total += get_polygon_area(proj); + } + } + + return total; + } + +private: + template + double get_polygon_area(PROJ *proj) + { + auto num_rings = read_length(); + assert(num_rings > 0); + + double total = get_ring_area(proj); + + for (unsigned i = 1; i < num_rings; ++i) { + total -= get_ring_area(proj); + } + + return total; + } + + template + double get_ring_area(PROJ *proj) + { + // Algorithm borrowed from + // http://stackoverflow.com/questions/451426/how-do-i-calculate-the-area-of-a-2d-polygon + // XXX numerically not stable (useless for latlon) + auto num_pts = read_length(); + assert(num_pts > 3); + + double total = 0; + + auto prev = read_point(); + proj->target_to_tile(&prev.y, &prev.x); + for (unsigned i = 1; i < num_pts; ++i) { + auto cur = read_point(); + proj->target_to_tile(&cur.y, &cur.x); + total += prev.x * cur.y - cur.x * prev.y; + prev = cur; + } + + return std::abs(total) * 0.5; + } + + double get_ring_area(osmium::geom::IdentityProjection *) + { + // Algorithm borrowed from + // http://stackoverflow.com/questions/451426/how-do-i-calculate-the-area-of-a-2d-polygon + auto num_pts = read_length(); + assert(num_pts > 3); + + double total = 0; + + auto prev = read_point(); + for (unsigned i = 1; i < num_pts; ++i) { + auto cur = read_point(); + total += prev.x * cur.y - cur.x * prev.y; + prev = cur; + } + + return std::abs(total) * 0.5; + } + + template + T read_data() + { + auto *data = reinterpret_cast(m_wkb + m_pos); + m_pos += sizeof(T); + + return *data; + } + + char const *m_wkb; + unsigned m_pos; +}; + +} // namespace + +#endif // OSM2PGSQL_WKB_HPP From 1c298b880f8a11ebfacea8671b61b5d052902abf Mon Sep 17 00:00:00 2001 From: Sarah Hoffmann Date: Sat, 4 Feb 2017 00:31:51 +0100 Subject: [PATCH 09/30] remove geometry builder and geos dependency --- CMakeLists.txt | 6 +- geometry-builder.cpp | 739 ------------------------------------------- geometry-builder.hpp | 114 ------- output-multi.cpp | 1 - 4 files changed, 3 insertions(+), 857 deletions(-) delete mode 100644 geometry-builder.cpp delete mode 100644 geometry-builder.hpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 4f003ccdb..e984054fd 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -92,7 +92,7 @@ endif() include_directories(${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_BINARY_DIR}) -find_package(Osmium REQUIRED COMPONENTS io geos proj) +find_package(Osmium REQUIRED COMPONENTS io proj) include_directories(SYSTEM ${OSMIUM_INCLUDE_DIRS}) if (WITH_LUA) @@ -167,7 +167,6 @@ endif() set(osm2pgsql_lib_SOURCES expire-tiles.cpp - geometry-builder.cpp geometry-processor.cpp id-tracker.cpp middle-pgsql.cpp @@ -196,7 +195,6 @@ set(osm2pgsql_lib_SOURCES util.cpp wildcmp.cpp expire-tiles.hpp - geometry-builder.hpp geometry-processor.hpp id-tracker.hpp middle-pgsql.hpp @@ -206,6 +204,7 @@ set(osm2pgsql_lib_SOURCES node-ram-cache.hpp options.hpp osmdata.hpp + osmium-builder.hpp osmtypes.hpp output-gazetteer.hpp output-multi.hpp @@ -225,6 +224,7 @@ set(osm2pgsql_lib_SOURCES tagtransform.hpp util.hpp wildcmp.hpp + wkb.hpp ) add_library(osm2pgsql_lib STATIC ${osm2pgsql_lib_SOURCES}) diff --git a/geometry-builder.cpp b/geometry-builder.cpp deleted file mode 100644 index 4dea8c8ea..000000000 --- a/geometry-builder.cpp +++ /dev/null @@ -1,739 +0,0 @@ -/* -#----------------------------------------------------------------------------- -# Part of osm2pgsql utility -#----------------------------------------------------------------------------- -# By Artem Pavlenko, Copyright 2007 -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License -# as published by the Free Software Foundation; either version 2 -# of the License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software -# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -#----------------------------------------------------------------------------- -*/ - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#if defined(__CYGWIN__) -#define GEOS_INLINE -#endif - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -using namespace geos::geom; -using namespace geos::util; -using namespace geos::operation::linemerge; - -#include "geometry-builder.hpp" -#include "reprojection.hpp" - -typedef std::unique_ptr geom_ptr; -typedef std::unique_ptr coord_ptr; - -namespace { - -void coords2nodes(CoordinateSequence * coords, nodelist_t &nodes) -{ - size_t num_coords = coords->getSize(); - nodes.reserve(num_coords); - - for (size_t i = 0; i < num_coords; i++) { - Coordinate coord = coords->getAt(i); - nodes.emplace_back(coord.x, coord.y); - } -} - -coord_ptr nodes2coords(GeometryFactory &gf, const nodelist_t &nodes) -{ - coord_ptr coords(gf.getCoordinateSequenceFactory()->create(size_t(0), size_t(2))); - - for (const auto& nd: nodes) { - coords->add(Coordinate(nd.x, nd.y), 0); - } - - return coords; -} - -geom_ptr create_multi_line(GeometryFactory &gf, const multinodelist_t &xnodes) -{ - // XXX leaks memory if an exception is thrown - std::unique_ptr > lines(new std::vector); - lines->reserve(xnodes.size()); - - for (const auto& nodes: xnodes) { - auto coords = nodes2coords(gf, nodes); - if (coords->getSize() > 1) { - lines->push_back(gf.createLineString(coords.release())); - } - } - - return geom_ptr(gf.createMultiLineString(lines.release())); -} - -bool is_polygon_line(CoordinateSequence * coords) -{ - return (coords->getSize() >= 4) - && (coords->getAt(coords->getSize() - 1).equals2D(coords->getAt(0))); -} - -/** - * Reprojects given Linear Ring from target projection to spherical mercator. - * Caller takes ownership of return value. - */ -LinearRing* reproject_linearring(const LineString *ls, const reprojection *proj) -{ - auto *gf = ls->getFactory(); - coord_ptr coords(gf->getCoordinateSequenceFactory()->create(size_t(0), size_t(2))); - for (auto i : *(ls->getCoordinatesRO()->toVector())) { - Coordinate c(i.x, i.y); - proj->target_to_tile(&c.y, &c.x); - coords->add(c); - } - return gf->createLinearRing(coords.release()); -} - - -/** - * Computes area of given polygonal geometry. - * \return the area in projected units, or in EPSG 3857 if area reprojection is enabled - */ -double get_area(const geos::geom::Geometry *geom, reprojection *proj) -{ - // reprojection is not necessary, or has not been asked for. - if (!proj) { - return geom->getArea(); - } - - // MultiPolygon - return sum of individual areas - if (const auto *multi = dynamic_cast(geom)) { - return std::accumulate(multi->begin(), multi->end(), 0.0, - [=](double a, const Geometry *g) { - return a + get_area(g, proj); - }); - } - - const auto *poly = dynamic_cast(geom); - if (!poly) { - return 0.0; - } - - // standard polygon - reproject rings individually, then assemble polygon and - // compute area. - - const auto *ext = poly->getExteriorRing(); - std::unique_ptr projectedExt(reproject_linearring(ext, proj)); - auto nholes = poly->getNumInteriorRing(); - std::unique_ptr > projectedHoles(new std::vector); - for (std::size_t i=0; i < nholes; i++) { - auto* hole = poly->getInteriorRingN(i); - projectedHoles->push_back(reproject_linearring(hole, proj)); - } - const geom_ptr projectedPoly(poly->getFactory()->createPolygon(projectedExt.release(), projectedHoles.release())); - - return projectedPoly->getArea(); -} - - -struct polygondata -{ - std::unique_ptr polygon; - std::unique_ptr ring; - double area; - bool iscontained; - unsigned containedbyid; - - polygondata(std::unique_ptr p, LinearRing* r, double a) - : polygon(std::move(p)), ring(r), area(a), - iscontained(false), containedbyid(0) - {} -}; - -struct polygondata_comparearea { - bool operator()(const polygondata& lhs, const polygondata& rhs) { - return lhs.area > rhs.area; - } -}; - -} // anonymous namespace - - -void geometry_builder::pg_geom_t::set(const geos::geom::Geometry *g, bool poly, - reprojection *p) -{ - geos::io::WKBWriter writer(2, getMachineByteOrder(), true); - std::stringstream stream(std::ios_base::out); - writer.writeHEX(*g, stream); - geom = stream.str(); - - if (valid()) { - area = poly ? get_area(g, p) : 0; - polygon = poly; - } -} - -geom_ptr geometry_builder::create_simple_poly(GeometryFactory &gf, - std::unique_ptr coords) const -{ - std::unique_ptr shell(gf.createLinearRing(coords.release())); - std::unique_ptr > empty(new std::vector); - geom_ptr geom(gf.createPolygon(shell.release(), empty.release())); - - if (geom->isEmpty()) { - throw std::runtime_error("Excluding empty polygon."); - } - if (!geom->isValid()) { - if (excludepoly) { - throw std::runtime_error("Excluding broken polygon."); - } else { - geom = geom_ptr(geom->buffer(0)); - if (geom->isEmpty() || !geom->isValid()) { - throw std::runtime_error("Excluding unrecoverable broken polygon."); - } - } - } - geom->normalize(); // Fix direction of ring - - return geom; -} - -geometry_builder::pg_geom_t geometry_builder::get_wkb_simple(const nodelist_t &nodes, int polygon) const -{ - pg_geom_t wkb; - - try - { - GeometryFactory gf; - auto coords = nodes2coords(gf, nodes); - if (polygon && is_polygon_line(coords.get())) { - auto geom = create_simple_poly(gf, std::move(coords)); - wkb.set(geom.get(), true, projection); - } else { - if (coords->getSize() < 2) - throw std::runtime_error("Excluding degenerate line."); - geom_ptr geom(gf.createLineString(coords.release())); - wkb.set(geom.get(), false); - } - } - catch (const std::bad_alloc&) - { - std::cerr << std::endl << "Exception caught processing way. You are likelly running out of memory." << std::endl; - std::cerr << "Try in slim mode, using -s parameter." << std::endl; - } - catch (const std::runtime_error& e) - { - //std::cerr << std::endl << "Exception caught processing way: " << e.what() << std::endl; - } - catch (...) - { - std::cerr << std::endl << "Exception caught processing way" << std::endl; - } - - return wkb; -} - -geometry_builder::pg_geoms_t geometry_builder::get_wkb_split(const nodelist_t &nodes, int polygon, double split_at) const -{ - //TODO: use count to get some kind of hint of how much we should reserve? - pg_geoms_t wkbs; - - try - { - GeometryFactory gf; - auto coords = nodes2coords(gf, nodes); - - if (polygon && is_polygon_line(coords.get())) { - auto geom = create_simple_poly(gf, std::move(coords)); - wkbs.emplace_back(geom.get(), true, projection); - } else { - if (coords->getSize() < 2) - throw std::runtime_error("Excluding degenerate line."); - - double distance = 0; - std::unique_ptr segment(gf.getCoordinateSequenceFactory()->create((size_t)0, (size_t)2)); - segment->add(coords->getAt(0)); - for(size_t i=1; igetSize(); i++) { - const Coordinate this_pt = coords->getAt(i); - const Coordinate prev_pt = coords->getAt(i-1); - const double delta = this_pt.distance(prev_pt); - assert(!std::isnan(delta)); - // figure out if the addition of this point would take the total - // length of the line in `segment` over the `split_at` distance. - - if (distance + delta > split_at) { - const size_t splits = (size_t) std::floor((distance + delta) / split_at); - // use the splitting distance to split the current segment up - // into as many parts as necessary to keep each part below - // the `split_at` distance. - for (size_t j = 0; j < splits; ++j) { - double frac = (double(j + 1) * split_at - distance) / delta; - const Coordinate interpolated(frac * (this_pt.x - prev_pt.x) + prev_pt.x, - frac * (this_pt.y - prev_pt.y) + prev_pt.y); - segment->add(interpolated); - geom_ptr geom(gf.createLineString(segment.release())); - - wkbs.emplace_back(geom.get(), false); - - segment.reset(gf.getCoordinateSequenceFactory()->create((size_t)0, (size_t)2)); - segment->add(interpolated); - } - // reset the distance based on the final splitting point for - // the next iteration. - distance = segment->getAt(0).distance(this_pt); - - } else { - // if not split then just push this point onto the sequence - // being saved up. - distance += delta; - } - - // always add this point - segment->add(this_pt); - - // on the last iteration, close out the line. - if (i == coords->getSize()-1) { - geom_ptr geom(gf.createLineString(segment.release())); - - wkbs.emplace_back(geom.get(), false); - } - } - } - } - catch (const std::bad_alloc&) - { - std::cerr << std::endl << "Exception caught processing way. You are likely running out of memory." << std::endl; - std::cerr << "Try in slim mode, using -s parameter." << std::endl; - } - catch (const std::runtime_error& e) - { - //std::cerr << std::endl << "Exception caught processing way: " << e.what() << std::endl; - } - catch (...) - { - std::cerr << std::endl << "Exception caught processing way" << std::endl; - } - return wkbs; -} - -int geometry_builder::parse_wkb(const char* wkb, multinodelist_t &nodes, bool *polygon) { - GeometryFactory gf; - geos::io::WKBReader reader(gf); - - *polygon = false; - std::stringstream stream(wkb, std::ios_base::in); - geom_ptr geometry(reader.readHEX(stream)); - switch (geometry->getGeometryTypeId()) { - // Single geometries - case GEOS_POLYGON: - // Drop through - case GEOS_LINEARRING: - *polygon = true; - // Drop through - case GEOS_POINT: - // Drop through - case GEOS_LINESTRING: - { - nodes.push_back(nodelist_t()); - coord_ptr coords(geometry->getCoordinates()); - coords2nodes(coords.get(), nodes.back()); - break; - } - // Geometry collections - case GEOS_MULTIPOLYGON: - *polygon = true; - // Drop through - case GEOS_MULTIPOINT: - // Drop through - case GEOS_MULTILINESTRING: - { - auto gc = dynamic_cast(geometry.get()); - size_t num_geometries = gc->getNumGeometries(); - nodes.assign(num_geometries, nodelist_t()); - for (size_t i = 0; i < num_geometries; i++) { - const Geometry *subgeometry = gc->getGeometryN(i); - coord_ptr coords(subgeometry->getCoordinates()); - coords2nodes(coords.get(), nodes[i]); - } - break; - } - default: - std::cerr << std::endl << "unexpected object type while processing PostGIS data" << std::endl; - return -1; - } - - return 0; -} - -geometry_builder::pg_geoms_t geometry_builder::build_polygons(const multinodelist_t &xnodes, - bool enable_multi, osmid_t osm_id) const -{ - pg_geoms_t wkbs; - - try - { - GeometryFactory gf; - geom_ptr mline = create_multi_line(gf, xnodes); - - //geom_ptr noded (segment->Union(mline.get())); - LineMerger merger; - //merger.add(noded.get()); - merger.add(mline.get()); - std::unique_ptr> merged(merger.getMergedLineStrings()); - - // Procces ways into lines or simple polygon list - std::vector polys; - polys.reserve(merged->size()); - - for (auto *line: *merged) { - // stuff into unique pointer for auto-destruct - std::unique_ptr pline(line); - if (pline->getNumPoints() > 3 && pline->isClosed()) { - std::unique_ptr poly(gf.createPolygon(gf.createLinearRing(pline->getCoordinates()),0)); - double area = get_area(poly.get(), projection); - if (area > 0.0) { - polys.emplace_back(std::move(poly), - gf.createLinearRing(pline->getCoordinates()), - area); - } - } - } - - if (!polys.empty()) - { - std::sort(polys.begin(), polys.end(), polygondata_comparearea()); - - unsigned toplevelpolygons = 0; - int istoplevelafterall; - size_t totalpolys = polys.size(); - - geos::geom::prep::PreparedGeometryFactory pgf; - for (unsigned i=0 ;i < totalpolys; ++i) - { - if (polys[i].iscontained) continue; - toplevelpolygons++; - const geos::geom::prep::PreparedGeometry* preparedtoplevelpolygon = pgf.create(polys[i].polygon.get()); - - for (unsigned j=i+1; j < totalpolys; ++j) - { - // Does preparedtoplevelpolygon contain the smaller polygon[j]? - if (polys[j].containedbyid == 0 && preparedtoplevelpolygon->contains(polys[j].polygon.get())) - { - // are we in a [i] contains [k] contains [j] situation - // which would actually make j top level - istoplevelafterall = 0; - for (unsigned k=i+1; k < j; ++k) - { - if (polys[k].iscontained && polys[k].containedbyid == i && polys[k].polygon->contains(polys[j].polygon.get())) - { - istoplevelafterall = 1; - break; - } - } - if (istoplevelafterall == 0) - { - polys[j].iscontained = true; - polys[j].containedbyid = i; - } - } - } - pgf.destroy(preparedtoplevelpolygon); - } - // polys now is a list of polygons tagged with which ones are inside each other - - // List of polygons for multipolygon - std::unique_ptr> polygons(new std::vector); - - // For each top level polygon create a new polygon including any holes - for (unsigned i=0 ;i < totalpolys; ++i) - { - if (polys[i].iscontained) continue; - - // List of holes for this top level polygon - std::unique_ptr > interior(new std::vector); - for (unsigned j=i+1; j < totalpolys; ++j) - { - if (polys[j].iscontained && polys[j].containedbyid == i) - { - interior->push_back(polys[j].ring.release()); - } - } - - Polygon* poly(gf.createPolygon(polys[i].ring.release(), interior.release())); - poly->normalize(); - polygons->push_back(poly); - } - - // Make a multipolygon if required - if ((toplevelpolygons > 1) && enable_multi) - { - geom_ptr multipoly(gf.createMultiPolygon(polygons.release())); - - if (!multipoly->isEmpty()) { - if (!multipoly->isValid() && !excludepoly) { - multipoly = geom_ptr(multipoly->buffer(0)); - multipoly->normalize(); - if (!multipoly->isEmpty() && multipoly->isValid()) { - wkbs.emplace_back(multipoly.get(), true, projection); - } - } else { - multipoly->normalize(); - wkbs.emplace_back(multipoly.get(), true, projection); - } - } - } else { - for(unsigned i=0; iat(i)); - if (!poly->isEmpty()) { - if (!poly->isValid() && !excludepoly) { - poly = geom_ptr(poly->buffer(0)); - poly->normalize(); - if (!poly->isEmpty() && poly->isValid()) { - wkbs.emplace_back(poly.get(), true, projection); - } - } else { - poly->normalize(); - wkbs.emplace_back(poly.get(), true, projection); - } - } - } - } - } - }//TODO: don't show in message id when osm_id == -1 - catch (const std::exception& e) - { - std::cerr << std::endl << "Standard exception processing relation_id="<< osm_id << ": " << e.what() << std::endl; - } - catch (...) - { - std::cerr << std::endl << "Exception caught processing relation id=" << osm_id << std::endl; - } - - return wkbs; -} - -geometry_builder::pg_geom_t geometry_builder::build_multilines(const multinodelist_t &xnodes, osmid_t osm_id) const -{ - pg_geom_t wkb; - - try - { - GeometryFactory gf; - geom_ptr mline = create_multi_line(gf, xnodes); - - wkb.set(mline.get(), false); - }//TODO: don't show in message id when osm_id == -1 - catch (const std::exception& e) - { - std::cerr << std::endl << "Standard exception processing relation_id="<< osm_id << ": " << e.what() << std::endl; - } - catch (...) - { - std::cerr << std::endl << "Exception caught processing relation id=" << osm_id << std::endl; - } - return wkb; -} - -geometry_builder::pg_geoms_t geometry_builder::build_both(const multinodelist_t &xnodes, - int make_polygon, int enable_multi, - double split_at, osmid_t osm_id) const -{ - pg_geoms_t wkbs; - - try - { - GeometryFactory gf; - geom_ptr mline = create_multi_line(gf, xnodes); - //geom_ptr noded (segment->Union(mline.get())); - LineMerger merger; - //merger.add(noded.get()); - merger.add(mline.get()); - std::unique_ptr > merged(merger.getMergedLineStrings()); - - // Procces ways into lines or simple polygon list - std::vector polys; - polys.reserve(merged->size()); - - for (auto *line: *merged) { - // stuff into unique pointer to ensure auto-destruct - std::unique_ptr pline(line); - if (make_polygon && pline->getNumPoints() > 3 && pline->isClosed()) { - std::unique_ptr poly(gf.createPolygon(gf.createLinearRing(pline->getCoordinates()),0)); - double area = get_area(poly.get(), projection); - if (area > 0.0) { - polys.emplace_back(std::move(poly), - gf.createLinearRing(pline->getCoordinates()), - area); - } - } else { - double distance = 0; - std::unique_ptr segment; - segment = std::unique_ptr(gf.getCoordinateSequenceFactory()->create((size_t)0, (size_t)2)); - segment->add(pline->getCoordinateN(0)); - for(int j=1; j<(int)pline->getNumPoints(); ++j) { - segment->add(pline->getCoordinateN(j)); - distance += pline->getCoordinateN(j).distance(pline->getCoordinateN(j-1)); - if ((distance >= split_at) || (j == (int)pline->getNumPoints()-1)) { - geom_ptr geom = geom_ptr(gf.createLineString(segment.release())); - - wkbs.emplace_back(geom.get(), false); - - segment.reset(gf.getCoordinateSequenceFactory()->create((size_t)0, (size_t)2)); - distance=0; - segment->add(pline->getCoordinateN(j)); - } - } - } - } - - if (!polys.empty()) - { - std::sort(polys.begin(), polys.end(), polygondata_comparearea()); - - unsigned toplevelpolygons = 0; - int istoplevelafterall; - size_t totalpolys = polys.size(); - - geos::geom::prep::PreparedGeometryFactory pgf; - for (unsigned i=0 ;i < totalpolys; ++i) - { - if (polys[i].iscontained) continue; - toplevelpolygons++; - const geos::geom::prep::PreparedGeometry* preparedtoplevelpolygon = pgf.create(polys[i].polygon.get()); - - for (unsigned j=i+1; j < totalpolys; ++j) - { - // Does preparedtoplevelpolygon contain the smaller polygon[j]? - if (polys[j].containedbyid == 0 && preparedtoplevelpolygon->contains(polys[j].polygon.get())) - { - // are we in a [i] contains [k] contains [j] situation - // which would actually make j top level - istoplevelafterall = 0; - for (unsigned k=i+1; k < j; ++k) - { - if (polys[k].iscontained && polys[k].containedbyid == i && polys[k].polygon->contains(polys[j].polygon.get())) - { - istoplevelafterall = 1; - break; - } -#if 0 - else if (polys[k].polygon->intersects(polys[j].polygon) || polys[k].polygon->touches(polys[j].polygon)) - { - // FIXME: This code does not work as intended - // It should be setting the polys[k].ring in order to update this object - // but the value of polys[k].polygon calculated is normally NULL - - // Add polygon this polygon (j) to k since they intersect - // Mark ourselfs to be dropped (2), delete the original k - Geometry* polyunion = polys[k].polygon->Union(polys[j].polygon); - delete(polys[k].polygon); - polys[k].polygon = dynamic_cast(polyunion); - polys[j].iscontained = 2; // Drop - istoplevelafterall = 2; - break; - } -#endif - } - if (istoplevelafterall == 0) - { - polys[j].iscontained = true; - polys[j].containedbyid = i; - } - } - } - pgf.destroy(preparedtoplevelpolygon); - } - // polys now is a list of polygons tagged with which ones are inside each other - - // List of polygons for multipolygon - std::unique_ptr > polygons(new std::vector); - - // For each top level polygon create a new polygon including any holes - for (unsigned i=0 ;i < totalpolys; ++i) - { - if (polys[i].iscontained) continue; - - // List of holes for this top level polygon - std::unique_ptr > interior(new std::vector); - for (unsigned j=i+1; j < totalpolys; ++j) - { - if (polys[j].iscontained && polys[j].containedbyid == i) - { - interior->push_back(polys[j].ring.release()); - } - } - - Polygon* poly(gf.createPolygon(polys[i].ring.release(), interior.release())); - poly->normalize(); - polygons->push_back(poly); - } - - // Make a multipolygon if required - if ((toplevelpolygons > 1) && enable_multi) - { - geom_ptr multipoly(gf.createMultiPolygon(polygons.release())); - if (!multipoly->isValid() && !excludepoly) { - multipoly = geom_ptr(multipoly->buffer(0)); - } - multipoly->normalize(); - - if (!multipoly->isEmpty() && multipoly->isValid()) { - wkbs.emplace_back(multipoly.get(), true, projection); - } - } - else - { - for(unsigned i=0; iat(i)); - if (!poly->isValid() && !excludepoly) { - poly = geom_ptr(poly->buffer(0)); - poly->normalize(); - } - if (!poly->isEmpty() && poly->isValid()) { - wkbs.emplace_back(poly.get(), true, projection); - } - } - } - } - }//TODO: don't show in message id when osm_id == -1 - catch (const std::exception& e) - { - std::cerr << std::endl << "Standard exception processing relation id="<< osm_id << ": " << e.what() << std::endl; - } - catch (...) - { - std::cerr << std::endl << "Exception caught processing relation id=" << osm_id << std::endl; - } - - return wkbs; -} diff --git a/geometry-builder.hpp b/geometry-builder.hpp deleted file mode 100644 index 56457c173..000000000 --- a/geometry-builder.hpp +++ /dev/null @@ -1,114 +0,0 @@ -/* -#----------------------------------------------------------------------------- -# Part of osm2pgsql utility -#----------------------------------------------------------------------------- -# By Artem Pavlenko, Copyright 2007 -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License -# as published by the Free Software Foundation; either version 2 -# of the License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software -# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -#----------------------------------------------------------------------------- -*/ - -#ifndef GEOMETRY_BUILDER_H -#define GEOMETRY_BUILDER_H - -#include "osmtypes.hpp" - -#include -#include -#include - -namespace geos { namespace geom { -class Geometry; -class GeometryFactory; -class CoordinateSequence; -}} - -class reprojection; - -class geometry_builder -{ -public: - struct pg_geom_t - { - pg_geom_t(const std::string &geom_str, bool poly, double geom_area = 0) - : geom(geom_str), area(geom_area), polygon(poly) - {} - - /// Create an invalid geometry. - pg_geom_t() - : area(0), polygon(false) - {} - - pg_geom_t(const geos::geom::Geometry *g, bool poly, reprojection *p = nullptr) - { - set(g, poly, p); - } - - /** - * Set geometry from a Geos geometry. - */ - void set(const geos::geom::Geometry *geom, bool poly, reprojection *p = nullptr); - - - bool is_polygon() const - { - return polygon; - } - - bool valid() const - { - return !geom.empty(); - } - - std::string geom; - double area; - bool polygon; - }; - - typedef std::vector pg_geoms_t; - - static int parse_wkb(const char *wkb, multinodelist_t &nodes, bool *polygon); - pg_geom_t get_wkb_simple(const nodelist_t &nodes, int polygon) const; - pg_geoms_t get_wkb_split(const nodelist_t &nodes, int polygon, double split_at) const; - pg_geoms_t build_both(const multinodelist_t &xnodes, int make_polygon, - int enable_multi, double split_at, osmid_t osm_id = -1) const; - pg_geoms_t build_polygons(const multinodelist_t &xnodes, bool enable_multi, osmid_t osm_id = -1) const; - /** Output relation as a multiline. - * - * Used by gazetteer only. - */ - pg_geom_t build_multilines(const multinodelist_t &xnodes, osmid_t osm_id) const; - - void set_exclude_broken_polygon(bool exclude) - { - excludepoly = exclude; - } - - void set_reprojection(reprojection *r) - { - projection = r; - } - - -private: - std::unique_ptr - create_simple_poly(geos::geom::GeometryFactory &gf, - std::unique_ptr coords) const; - - bool excludepoly = false; - reprojection *projection = nullptr; -}; - -#endif diff --git a/output-multi.cpp b/output-multi.cpp index 03fd311fe..acad3fdfe 100644 --- a/output-multi.cpp +++ b/output-multi.cpp @@ -1,6 +1,5 @@ #include "output-multi.hpp" #include "expire-tiles.hpp" -#include "geometry-builder.hpp" #include "id-tracker.hpp" #include "middle.hpp" #include "options.hpp" From 3d5c53dfb16b77c5f7d4d14a98622fc3ab640358 Mon Sep 17 00:00:00 2001 From: Sarah Hoffmann Date: Sat, 4 Feb 2017 00:42:53 +0100 Subject: [PATCH 10/30] remove unused functions from expire --- expire-tiles.cpp | 43 ------------------------------------------- expire-tiles.hpp | 2 -- 2 files changed, 45 deletions(-) diff --git a/expire-tiles.cpp b/expire-tiles.cpp index f8336bac5..cd14c1f93 100644 --- a/expire-tiles.cpp +++ b/expire-tiles.cpp @@ -333,49 +333,6 @@ int expire_tiles::from_bbox(double min_lon, double min_lat, double max_lon, doub return 0; } -void expire_tiles::from_nodes_line(const nodelist_t &nodes) -{ - if (maxzoom < 0 || nodes.empty()) - return; - - if (nodes.size() == 1) { - from_bbox(nodes[0].x, nodes[0].y, nodes[0].x, nodes[0].y); - } else { - for (size_t i = 1; i < nodes.size(); ++i) - from_line(nodes[i - 1].x, nodes[i - 1].y, nodes[i].x, nodes[i].y); - } -} - -/* - * Calculate a bounding box from a list of nodes and expire all tiles within it - */ -void expire_tiles::from_nodes_poly(const nodelist_t &nodes, osmid_t osm_id) -{ - if (maxzoom < 0 || nodes.empty()) - return; - - double min_lon = nodes[0].x; - double min_lat = nodes[0].y; - double max_lon = nodes[0].x; - double max_lat = nodes[0].y; - - for (size_t i = 1; i < nodes.size(); ++i) { - if (nodes[i].x < min_lon) - min_lon = nodes[i].x; - if (nodes[i].y < min_lat) - min_lat = nodes[i].y; - if (nodes[i].x > max_lon) - max_lon = nodes[i].x; - if (nodes[i].y > max_lat) - max_lat = nodes[i].y; - } - - if (from_bbox(min_lon, min_lat, max_lon, max_lat)) { - /* Bounding box too big - just expire tiles on the line */ - fprintf(stderr, "\rLarge polygon (%.0f x %.0f metres, OSM ID %" PRIdOSMID ") - only expiring perimeter\n", max_lon - min_lon, max_lat - min_lat, osm_id); - from_nodes_line(nodes); - } -} void expire_tiles::from_wkb(const char *wkb, osmid_t osm_id) { diff --git a/expire-tiles.hpp b/expire-tiles.hpp index 448585778..1ef70395e 100644 --- a/expire-tiles.hpp +++ b/expire-tiles.hpp @@ -18,8 +18,6 @@ struct expire_tiles const std::shared_ptr &projection); int from_bbox(double min_lon, double min_lat, double max_lon, double max_lat); - void from_nodes_line(const nodelist_t &nodes); - void from_nodes_poly(const nodelist_t &nodes, osmid_t osm_id); void from_wkb(const char* wkb, osmid_t osm_id); int from_db(table_t* table, osmid_t osm_id); From dfd4a51c8f1e104930d42c5a68260bea8979035c Mon Sep 17 00:00:00 2001 From: Sarah Hoffmann Date: Sat, 4 Feb 2017 00:50:49 +0100 Subject: [PATCH 11/30] remove now unused write_row functions in table --- output-multi.cpp | 4 ++-- output-pgsql.cpp | 14 ++++++------ table.cpp | 57 ++---------------------------------------------- table.hpp | 7 ++---- 4 files changed, 13 insertions(+), 69 deletions(-) diff --git a/output-multi.cpp b/output-multi.cpp index acad3fdfe..1c7cc325f 100644 --- a/output-multi.cpp +++ b/output-multi.cpp @@ -421,7 +421,7 @@ int output_multi_t::process_relation(osmium::Relation const &rel, void output_multi_t::copy_node_to_table(osmid_t id, std::string const &geom, taglist_t &tags) { - m_table->write_row_wkb(id, tags, geom); + m_table->write_row(id, tags, geom); } /** @@ -449,7 +449,7 @@ void output_multi_t::copy_to_table(const osmid_t id, } m_expire.from_wkb(geom.c_str(), id); - m_table->write_row_wkb(id, tags, geom); + m_table->write_row(id, tags, geom); } void output_multi_t::delete_from_output(osmid_t id) { diff --git a/output-pgsql.cpp b/output-pgsql.cpp index 27d2f46e3..3bca70859 100644 --- a/output-pgsql.cpp +++ b/output-pgsql.cpp @@ -87,14 +87,14 @@ void output_pgsql_t::pgsql_out_way(osmium::Way const &way, taglist_t *tags, .get_area(); tags->push_override(tag_t("way_area", std::to_string(area))); } - m_tables[t_poly]->write_row_wkb(way.id(), *tags, wkb); + m_tables[t_poly]->write_row(way.id(), *tags, wkb); } } else { for (auto const &wkb : m_builder.get_wkb_line(way.nodes(), true)) { expire.from_wkb(wkb.c_str(), way.id()); - m_tables[t_line]->write_row_wkb(way.id(), *tags, wkb); + m_tables[t_line]->write_row(way.id(), *tags, wkb); if (roads) { - m_tables[t_roads]->write_row_wkb(way.id(), *tags, wkb); + m_tables[t_roads]->write_row(way.id(), *tags, wkb); } } @@ -274,7 +274,7 @@ int output_pgsql_t::node_add(osmium::Node const &node) auto wkb = m_builder.get_wkb_node(node.location()); expire.from_wkb(wkb.c_str(), node.id()); - m_tables[t_point]->write_row_wkb(node.id(), outtags, wkb); + m_tables[t_point]->write_row(node.id(), outtags, wkb); return 0; } @@ -390,7 +390,7 @@ int output_pgsql_t::pgsql_process_relation(osmium::Relation const &rel, .get_area(); outtags.push_override(tag_t("way_area", std::to_string(area))); } - m_tables[t_poly]->write_row_wkb(-rel.id(), outtags, wkb); + m_tables[t_poly]->write_row(-rel.id(), outtags, wkb); } /* Tagtransform will have marked those member ways of the relation that @@ -417,9 +417,9 @@ int output_pgsql_t::pgsql_process_relation(osmium::Relation const &rel, auto wkbs = m_builder.get_wkb_multiline(buffer, true); for (auto const &wkb : wkbs) { expire.from_wkb(wkb.c_str(), -rel.id()); - m_tables[t_line]->write_row_wkb(-rel.id(), outtags, wkb); + m_tables[t_line]->write_row(-rel.id(), outtags, wkb); if (roads) - m_tables[t_roads]->write_row_wkb(-rel.id(), outtags, wkb); + m_tables[t_roads]->write_row(-rel.id(), outtags, wkb); } } diff --git a/table.cpp b/table.cpp index f922de1d5..3ef4d7ea0 100644 --- a/table.cpp +++ b/table.cpp @@ -33,7 +33,6 @@ table_t::table_t(const string& conninfo, const string& name, const string& type, //we use these a lot, so instead of constantly allocating them we predefine these single_fmt = fmt("%1%"); - point_fmt = fmt("POINT(%.15g %.15g)"); del_fmt = fmt("DELETE FROM %1% WHERE osm_id = %2%"); } @@ -41,7 +40,7 @@ table_t::table_t(const table_t& other): conninfo(other.conninfo), name(other.name), type(other.type), sql_conn(nullptr), copyMode(false), buffer(), srid(other.srid), append(other.append), slim(other.slim), drop_temp(other.drop_temp), hstore_mode(other.hstore_mode), enable_hstore_index(other.enable_hstore_index), columns(other.columns), hstore_columns(other.hstore_columns), copystr(other.copystr), table_space(other.table_space), - table_space_index(other.table_space_index), single_fmt(other.single_fmt), point_fmt(other.point_fmt), del_fmt(other.del_fmt) + table_space_index(other.table_space_index), single_fmt(other.single_fmt), del_fmt(other.del_fmt) { // if the other table has already started, then we want to execute // the same stuff to get into the same state. but if it hasn't, then @@ -288,65 +287,13 @@ void table_t::stop_copy() copyMode = false; } -void table_t::write_node(const osmid_t id, const taglist_t &tags, double lat, double lon) -{ - write_row(id, tags, (point_fmt % lon % lat).str()); -} - void table_t::delete_row(const osmid_t id) { stop_copy(); pgsql_exec_simple(sql_conn, PGRES_COMMAND_OK, (del_fmt % name % id).str()); } -void table_t::write_row(const osmid_t id, const taglist_t &tags, const std::string &geom) -{ - //add the osm id - buffer.append((single_fmt % id).str()); - buffer.push_back('\t'); - - // used to remember which columns have been written out already. - std::vector used; - - if (hstore_mode != HSTORE_NONE) - used.assign(tags.size(), false); - - //get the regular columns' values - write_columns(tags, buffer, hstore_mode == HSTORE_NORM?&used:nullptr); - - //get the hstore columns' values - write_hstore_columns(tags, buffer); - - //get the key value pairs for the tags column - if (hstore_mode != HSTORE_NONE) - write_tags_column(tags, buffer, used); - - //give the geometry an srid - buffer.append("SRID="); - buffer.append(srid); - buffer.push_back(';'); - //add the geometry - buffer.append(geom); - //we need \n because we are copying from stdin - buffer.push_back('\n'); - - //tell the db we are copying if for some reason we arent already - if (!copyMode) - { - pgsql_exec_simple(sql_conn, PGRES_COPY_IN, copystr); - copyMode = true; - } - - //send all the data to postgres - if(buffer.length() > BUFFER_SEND_SIZE) - { - pgsql_CopyData(name.c_str(), sql_conn, buffer); - buffer.clear(); - } -} - -void table_t::write_row_wkb(osmid_t const id, taglist_t const &tags, - std::string const &geom) +void table_t::write_row(osmid_t id, taglist_t const &tags, std::string const &geom) { //add the osm id buffer.append((single_fmt % id).str()); diff --git a/table.hpp b/table.hpp index bc41044e7..497cd1b69 100644 --- a/table.hpp +++ b/table.hpp @@ -31,10 +31,7 @@ class table_t void begin(); void commit(); - void write_row(const osmid_t id, const taglist_t &tags, const std::string &geom); - void write_row_wkb(const osmid_t id, const taglist_t &tags, - const std::string &geom); - void write_node(const osmid_t id, const taglist_t &tags, double lat, double lon); + void write_row(osmid_t id, taglist_t const &tags, std::string const &geom); void delete_row(const osmid_t id); std::string const& get_name(); @@ -110,7 +107,7 @@ class table_t boost::optional table_space; boost::optional table_space_index; - boost::format single_fmt, point_fmt, del_fmt; + boost::format single_fmt, del_fmt; }; #endif From 29bd6cc2268a667ee664a3113ee9f712b9a4d806 Mon Sep 17 00:00:00 2001 From: Sarah Hoffmann Date: Sat, 4 Feb 2017 11:24:04 +0100 Subject: [PATCH 12/30] remove unused get_node_list() function for nodelist_t --- middle-pgsql.cpp | 95 --------------------------------------- middle-pgsql.hpp | 2 - middle-ram.cpp | 14 ------ middle-ram.hpp | 2 - middle.hpp | 8 ---- node-persistent-cache.cpp | 27 ----------- node-persistent-cache.hpp | 2 - tests/middle-tests.cpp | 56 +++++++++-------------- tests/mockups.hpp | 10 ----- 9 files changed, 22 insertions(+), 194 deletions(-) diff --git a/middle-pgsql.cpp b/middle-pgsql.cpp index c8766ce04..6191d206b 100644 --- a/middle-pgsql.cpp +++ b/middle-pgsql.cpp @@ -286,92 +286,6 @@ void middle_pgsql_t::local_nodes_set(osmium::Node const &node) } } -// This should be made more efficient by using an IN(ARRAY[]) construct */ -size_t middle_pgsql_t::local_nodes_get_list(nodelist_t &out, - osmium::WayNodeList const &nds, - reprojection const *proj) const -{ - assert(out.empty()); - - char tmp[16]; - - char *tmp2 = static_cast(malloc(sizeof(char) * nds.size() * 16)); - if (tmp2 == nullptr) { - return 0; // failed to allocate memory, return - } - - // create a list of ids in tmp2 to query the database - sprintf(tmp2, "{"); - int countDB = 0; - out.reserve(nds.size()); - for (auto const &n : nds) { - // Check cache first */ - auto loc = cache->get(n.ref()); - if (loc.valid()) { - out.push_back(proj->reproject(loc)); - continue; - } - - countDB++; - // Mark nodes as needing to be fetched from the DB - out.emplace_back(NAN, NAN); - - snprintf(tmp, sizeof(tmp), "%" PRIdOSMID ",", n.ref()); - strncat(tmp2, tmp, sizeof(char) * (nds.size() * 16 - 2)); - } - // replace last , with } to complete list of ids - tmp2[strlen(tmp2) - 1] = '}'; - - if (countDB == 0) { - free(tmp2); - return nds.size(); // All ids where in cache, so nothing more to do - } - - pgsql_endCopy(node_table); - - PGconn *sql_conn = node_table->sql_conn; - - char const *paramValues[1]; - paramValues[0] = tmp2; - PGresult *res = pgsql_execPrepared(sql_conn, "get_node_list", 1, - paramValues, PGRES_TUPLES_OK); - int countPG = PQntuples(res); - - // store the pg results in a hashmap and telling it how many we expect - std::unordered_map pg_nodes(countPG); - - for (int i = 0; i < countPG; i++) { - osmid_t id = strtoosmid(PQgetvalue(res, i, 0), nullptr, 10); - osmium::Location n((int)strtol(PQgetvalue(res, i, 2), nullptr, 10), - (int)strtol(PQgetvalue(res, i, 1), nullptr, 10)); - - pg_nodes.emplace(id, n); - } - - PQclear(res); - free(tmp2); - - // If some of the nodes in the way don't exist, the returning list has holes. - // Merge the two lists removing any holes. - size_t wrtidx = 0; - for (size_t i = 0; i < nds.size(); ++i) { - if (std::isnan(out[i].x)) { - auto found = pg_nodes.find(nds[i].ref()); - if (found != pg_nodes.end()) { - out[wrtidx] = proj->reproject(found->second); - ++wrtidx; - } - } else { - if (wrtidx < i) - out[wrtidx] = out[i]; - ++wrtidx; - } - } - out.resize(wrtidx, osmium::geom::Coordinates(NAN, NAN)); - - return wrtidx; -} - size_t middle_pgsql_t::local_nodes_get_list(osmium::WayNodeList *nodes) const { size_t count = 0; @@ -433,15 +347,6 @@ void middle_pgsql_t::nodes_set(osmium::Node const &node) } } -size_t middle_pgsql_t::nodes_get_list(nodelist_t &out, - osmium::WayNodeList const &nds, - reprojection const *proj) const -{ - return (out_options->flat_node_cache_enabled) - ? persistent_cache->get_list(out, nds, proj) - : local_nodes_get_list(out, nds, proj); -} - size_t middle_pgsql_t::nodes_get_list(osmium::WayNodeList *nodes) const { return (out_options->flat_node_cache_enabled) diff --git a/middle-pgsql.hpp b/middle-pgsql.hpp index 13f661447..5f1c24d71 100644 --- a/middle-pgsql.hpp +++ b/middle-pgsql.hpp @@ -27,8 +27,6 @@ struct middle_pgsql_t : public slim_middle_t { void commit(void) override; void nodes_set(osmium::Node const &node) override; - size_t nodes_get_list(nodelist_t &out, osmium::WayNodeList const &nds, - reprojection const *proj) const override; size_t nodes_get_list(osmium::WayNodeList *nodes) const override; void nodes_delete(osmid_t id) override; void node_changed(osmid_t id) override; diff --git a/middle-ram.cpp b/middle-ram.cpp index 8ee5dbbd1..02540d9fa 100644 --- a/middle-ram.cpp +++ b/middle-ram.cpp @@ -48,20 +48,6 @@ void middle_ram_t::relations_set(osmium::Relation const &rel) rels.set(rel.id(), new ramRel(rel, out_options->extra_attributes)); } -size_t middle_ram_t::nodes_get_list(nodelist_t &out, - osmium::WayNodeList const &nds, - reprojection const *proj) const -{ - for (auto const &in : nds) { - auto loc = cache->get(in.ref()); - if (loc.valid()) { - out.push_back(proj->reproject(loc)); - } - } - - return out.size(); -} - size_t middle_ram_t::nodes_get_list(osmium::WayNodeList *nodes) const { size_t count = 0; diff --git a/middle-ram.hpp b/middle-ram.hpp index 8cca7ded9..34cbf6d32 100644 --- a/middle-ram.hpp +++ b/middle-ram.hpp @@ -91,8 +91,6 @@ struct middle_ram_t : public middle_t { void commit(void) override; void nodes_set(osmium::Node const &node) override; - size_t nodes_get_list(nodelist_t &out, osmium::WayNodeList const &nds, - reprojection const *proj) const override; size_t nodes_get_list(osmium::WayNodeList *nodes) const override; int nodes_delete(osmid_t id); int node_changed(osmid_t id); diff --git a/middle.hpp b/middle.hpp index e757e6736..48a2d4dd8 100644 --- a/middle.hpp +++ b/middle.hpp @@ -23,14 +23,6 @@ struct options_t; struct middle_query_t { virtual ~middle_query_t() {} - /** - * Retrives node locations for the given node list and - * reprojects them. - */ - virtual size_t nodes_get_list(nodelist_t &out, - osmium::WayNodeList const &nds, - reprojection const *proj) const = 0; - /** * Retrives node locations for the given node list. * diff --git a/node-persistent-cache.cpp b/node-persistent-cache.cpp index 3659f5b27..45f96de58 100644 --- a/node-persistent-cache.cpp +++ b/node-persistent-cache.cpp @@ -2,7 +2,6 @@ #include "node-persistent-cache.hpp" #include "options.hpp" -#include "reprojection.hpp" void node_persistent_cache::set(osmid_t id, const osmium::Location &coord) { @@ -25,32 +24,6 @@ osmium::Location node_persistent_cache::get(osmid_t id) return osmium::Location(); } -size_t node_persistent_cache::get_list(nodelist_t &out, - osmium::WayNodeList const &nds, - reprojection const *proj) -{ - assert(nds.empty()); - out.reserve(nds.size()); - - for (auto const &n : nds) { - try { - /* Check cache first */ - auto loc = m_ram_cache->get(n.ref()); - if (!loc.valid() && n.ref() >= 0) { - loc = m_index->get( - static_cast(n.ref())); - } - if (loc.valid()) { - auto coord = proj->reproject(loc); - out.emplace_back(coord.x, coord.y); - } - } catch (osmium::not_found const &) { - } - } - - return out.size(); -} - size_t node_persistent_cache::get_list(osmium::WayNodeList *nodes) { size_t count = 0; diff --git a/node-persistent-cache.hpp b/node-persistent-cache.hpp index cb4f7d551..1902d963f 100644 --- a/node-persistent-cache.hpp +++ b/node-persistent-cache.hpp @@ -21,8 +21,6 @@ class node_persistent_cache void set(osmid_t id, osmium::Location const &coord); osmium::Location get(osmid_t id); - size_t get_list(nodelist_t &out, osmium::WayNodeList const &nds, - reprojection const *proj); size_t get_list(osmium::WayNodeList *nodes); private: diff --git a/tests/middle-tests.cpp b/tests/middle-tests.cpp index bb856435c..e764de548 100644 --- a/tests/middle-tests.cpp +++ b/tests/middle-tests.cpp @@ -23,18 +23,18 @@ std::shared_ptr proj(reprojection::create_projection(PROJ_LATLONG)); #define ALLOWED_ERROR 10e-9 -bool node_okay(osmium::geom::Coordinates node, osmium::Node const &expected) +bool node_okay(osmium::Location loc, osmium::Node const &expected) { - if ((node.y > expected.location().lat() + ALLOWED_ERROR) || - (node.y < expected.location().lat() - ALLOWED_ERROR)) { + if ((loc.lat() > expected.location().lat() + ALLOWED_ERROR) || + (loc.lat() < expected.location().lat() - ALLOWED_ERROR)) { std::cerr << "ERROR: Node should have lat=" << expected.location().lat() - << ", but got back " << node.y << " from middle.\n"; + << ", but got back " << loc.lat() << " from middle.\n"; return false; } - if ((node.x > expected.location().lon() + ALLOWED_ERROR) || - (node.x < expected.location().lon() - ALLOWED_ERROR)) { + if ((loc.lon() > expected.location().lon() + ALLOWED_ERROR) || + (loc.lon() < expected.location().lon() - ALLOWED_ERROR)) { std::cerr << "ERROR: Node should have lon=" << expected.location().lon() - << ", but got back " << node.x << " from middle.\n"; + << ", but got back " << loc.lon() << " from middle.\n"; return false; } return true; @@ -57,25 +57,19 @@ int test_node_set(middle_t *mid) buffer.clear(); auto const &node = buffer.get(add_node(1234, 12.3456789, 98.7654321)); - auto const &way = buffer.get(way_with_nodes({node.id()})); - nodelist_t nodes; + auto &way = buffer.get(way_with_nodes({node.id()})); // set the node mid->nodes_set(node); // get it back - if (mid->nodes_get_list(nodes, way.nodes(), proj.get()) != - way.nodes().size()) { + if (mid->nodes_get_list(&(way.nodes())) != way.nodes().size()) { std::cerr << "ERROR: Unable to get node list.\n"; return 1; } - if (nodes.size() != way.nodes().size()) { - std::cerr << "ERROR: Mismatch in returned node list size.\n"; - return 1; - } // check that it's the same - if (!node_okay(nodes[0], node)) { + if (!node_okay(way.nodes()[0].location(), node)) { return 1; } @@ -137,23 +131,16 @@ int test_nodes_comprehensive_set(middle_t *mid) ids.push_back(node.id()); } - auto const &way = buffer.get(way_with_nodes(ids)); + auto &way = buffer.get(way_with_nodes(ids)); - nodelist_t nodes; - if (mid->nodes_get_list(nodes, way.nodes(), proj.get()) != ids.size()) { + if (mid->nodes_get_list(&(way.nodes())) != ids.size()) { std::cerr << "ERROR: Unable to get node list.\n"; return 1; } - if (nodes.size() != ids.size()) { - std::cerr << "ERROR: Mismatch in returned node list size.\n"; - return 1; - } - - for (size_t i = 0; i < nodes.size(); ++i) - { + for (size_t i = 0; i < ids.size(); ++i) { auto const &node = buffer.get(expected_nodes[i]); - if (!node_okay(nodes[i], node)) { + if (!node_okay(way.nodes()[i].location(), node)) { return 1; } } @@ -220,7 +207,7 @@ int test_way_set(middle_t *mid) size_t way_count = mid->ways_get_list(ways, buffer); if (way_count != 1) { std::cerr << "ERROR: Unable to get way list.\n"; return 1; } - auto const &way = buffer.get(buf_pos); + auto &way = buffer.get(buf_pos); // check that it's the same if (way.nodes().size() != nds.size()) { std::cerr << "ERROR: Way should have " << nds.size() << " nodes, but got back " @@ -232,17 +219,18 @@ int test_way_set(middle_t *mid) << way.id() << " from middle.\n"; return 1; } - nodelist_t xnodes; - mid->nodes_get_list(xnodes, way.nodes(), proj.get()); + mid->nodes_get_list(&(way.nodes())); for (size_t i = 0; i < nds.size(); ++i) { - if (xnodes[i].x != lon) { + if (way.nodes()[i].location().lon() != lon) { std::cerr << "ERROR: Way node should have lon=" << lon - << ", but got back " << xnodes[i].x << " from middle.\n"; + << ", but got back " << way.nodes()[i].location().lon() + << " from middle.\n"; return 1; } - if (xnodes[i].y != lat) { + if (way.nodes()[i].location().lat() != lat) { std::cerr << "ERROR: Way node should have lat=" << lat - << ", but got back " << xnodes[i].y << " from middle.\n"; + << ", but got back " << way.nodes()[i].location().lat() + << " from middle.\n"; return 1; } } diff --git a/tests/mockups.hpp b/tests/mockups.hpp index 6583c7a55..d8184b310 100644 --- a/tests/mockups.hpp +++ b/tests/mockups.hpp @@ -15,11 +15,6 @@ struct dummy_middle_t : public middle_t { void commit(void) override { } void nodes_set(osmium::Node const &) override {} - size_t nodes_get_list(nodelist_t &, osmium::WayNodeList const &, - reprojection const *) const override - { - return 0; - } size_t nodes_get_list(osmium::WayNodeList *) const override { return 0; } void ways_set(osmium::Way const &) override { } @@ -53,11 +48,6 @@ struct dummy_slim_middle_t : public slim_middle_t { void commit(void) override { } void nodes_set(osmium::Node const &) override {} - size_t nodes_get_list(nodelist_t &, osmium::WayNodeList const &, - reprojection const *) const override - { - return 0; - } size_t nodes_get_list(osmium::WayNodeList *) const override { return 0; } void ways_set(osmium::Way const &) override { } From a73c5ee2bb5adf968a2add9c15f55ee3043e01f3 Mon Sep 17 00:00:00 2001 From: Sarah Hoffmann Date: Sat, 4 Feb 2017 12:12:36 +0100 Subject: [PATCH 13/30] fix area string conversion Go back to printf conversion because std::to_string() does not have sufficient precision. --- output-pgsql.cpp | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/output-pgsql.cpp b/output-pgsql.cpp index 3bca70859..dba17cff2 100644 --- a/output-pgsql.cpp +++ b/output-pgsql.cpp @@ -79,13 +79,15 @@ void output_pgsql_t::pgsql_out_way(osmium::Way const &way, taglist_t *tags, if (!wkb.empty()) { expire.from_wkb(wkb.c_str(), way.id()); if (m_enable_way_area) { + char tmp[32]; auto const area = m_options.reproject_area ? ewkb::parser_t(wkb).get_area( m_options.projection.get()) : ewkb::parser_t(wkb) .get_area(); - tags->push_override(tag_t("way_area", std::to_string(area))); + snprintf(tmp, sizeof(tmp), "%g", area); + tags->push_override(tag_t("way_area", tmp)); } m_tables[t_poly]->write_row(way.id(), *tags, wkb); } @@ -379,6 +381,7 @@ int output_pgsql_t::pgsql_process_relation(osmium::Relation const &rel, if (make_boundary || make_polygon) { auto wkbs = m_builder.get_wkb_multipolygon(rel, buffer); + char tmp[32]; for (auto const &wkb : wkbs) { expire.from_wkb(wkb.c_str(), -rel.id()); if (m_enable_way_area) { @@ -388,7 +391,8 @@ int output_pgsql_t::pgsql_process_relation(osmium::Relation const &rel, m_options.projection.get()) : ewkb::parser_t(wkb) .get_area(); - outtags.push_override(tag_t("way_area", std::to_string(area))); + snprintf(tmp, sizeof(tmp), "%g", area); + outtags.push_override(tag_t("way_area", tmp)); } m_tables[t_poly]->write_row(-rel.id(), outtags, wkb); } From f5ddbc61aae1f1b10da529dce3920535e5cfeeb1 Mon Sep 17 00:00:00 2001 From: Sarah Hoffmann Date: Sat, 4 Feb 2017 12:20:12 +0100 Subject: [PATCH 14/30] remove remaining traces of nodelist types No longer needed. Using libosmium types now. --- middle-pgsql.hpp | 2 -- osmtypes.hpp | 3 --- output-gazetteer.cpp | 1 - 3 files changed, 6 deletions(-) diff --git a/middle-pgsql.hpp b/middle-pgsql.hpp index 5f1c24d71..0756b1d5b 100644 --- a/middle-pgsql.hpp +++ b/middle-pgsql.hpp @@ -87,8 +87,6 @@ struct middle_pgsql_t : public slim_middle_t { */ void connect(table_desc& table); void local_nodes_set(osmium::Node const &node); - size_t local_nodes_get_list(nodelist_t &out, osmium::WayNodeList const &nds, - reprojection const *proj) const; size_t local_nodes_get_list(osmium::WayNodeList *nodes) const; void local_nodes_delete(osmid_t osm_id); diff --git a/osmtypes.hpp b/osmtypes.hpp index 6c830ef4e..14760dc6f 100644 --- a/osmtypes.hpp +++ b/osmtypes.hpp @@ -22,9 +22,6 @@ typedef int64_t osmid_t; #define PRIdOSMID PRId64 #define POSTGRES_OSMID_TYPE "int8" -typedef std::vector nodelist_t; -typedef std::vector multinodelist_t; - struct member { osmium::item_type type; osmid_t id; diff --git a/output-gazetteer.cpp b/output-gazetteer.cpp index 8225004b2..07e339321 100644 --- a/output-gazetteer.cpp +++ b/output-gazetteer.cpp @@ -690,7 +690,6 @@ int output_gazetteer_t::process_way(osmium::Way *way) /* Are we interested in this item? */ if (places.has_data()) { /* Fetch the node details */ - nodelist_t nodes; m_mid->nodes_get_list(&(way->nodes())); /* Get the geometry of the object */ From 9822c4049afae81220f2da0bd0e8be77094b83a3 Mon Sep 17 00:00:00 2001 From: Sarah Hoffmann Date: Sat, 4 Feb 2017 17:36:56 +0100 Subject: [PATCH 15/30] streamline retival of ways for relations Get roles and ids directly from the relation member list, avoiding intermediate id lists and additional loops through the members. --- geometry-processor.cpp | 33 ++++--------------------------- geometry-processor.hpp | 5 +---- middle-pgsql.cpp | 44 +++++++++++++++++++++++++----------------- middle-pgsql.hpp | 3 ++- middle-ram.cpp | 15 +++++++------- middle-ram.hpp | 3 ++- middle.hpp | 12 +++++++++++- output-gazetteer.cpp | 11 +++-------- output-multi.cpp | 2 +- output-pgsql.cpp | 11 ++--------- tests/middle-tests.cpp | 26 ++++++++++++++++++++++--- tests/mockups.hpp | 12 ++++++++++-- 12 files changed, 93 insertions(+), 84 deletions(-) diff --git a/geometry-processor.cpp b/geometry-processor.cpp index 76323d45c..729293d4a 100644 --- a/geometry-processor.cpp +++ b/geometry-processor.cpp @@ -79,41 +79,16 @@ relation_helper::relation_helper() : data(1024, osmium::memory::Buffer::auto_grow::yes) {} - -size_t relation_helper::set(osmium::RelationMemberList const &member_list, middle_t const *mid) +size_t relation_helper::set(osmium::Relation const &rel, middle_t const *mid) { // cleanup - input_way_ids.clear(); data.clear(); roles.clear(); - //grab the way members' ids - for (auto const &m : member_list) { - /* Need to handle more than just ways... */ - if (m.type() == osmium::item_type::way) { - input_way_ids.push_back(m.ref()); - } - } - - //if we didn't end up using any we'll bail - if (input_way_ids.empty()) - return 0; - - //get the nodes of the ways - auto num_ways = mid->ways_get_list(input_way_ids, data); - - //grab the roles of each way - for (auto const &w : data.select()) { - for (auto const &member : member_list) { - if (member.ref() == w.id() && - member.type() == osmium::item_type::way) { - roles.emplace_back(member.role()); - break; - } - } - } + // get the nodes and roles of the ways + auto num_ways = mid->rel_way_members_get(rel, &roles, data); - //mark the ends of each so whoever uses them will know where they end.. + // mark the ends of each so whoever uses them will know where they end.. superseded.resize(num_ways); return num_ways; diff --git a/geometry-processor.hpp b/geometry-processor.hpp index 0cf66793f..479b81914 100644 --- a/geometry-processor.hpp +++ b/geometry-processor.hpp @@ -84,16 +84,13 @@ class relation_helper public: relation_helper(); - size_t set(osmium::RelationMemberList const &member_list, middle_t const *mid); + size_t set(osmium::Relation const &rel, middle_t const *mid); multitaglist_t get_filtered_tags(tagtransform *transform, export_list const &el) const; void add_way_locations(middle_t const *mid); rolelist_t roles; std::vector superseded; osmium::memory::Buffer data; - -private: - idlist_t input_way_ids; }; #endif /* GEOMETRY_PROCESSOR_HPP */ diff --git a/middle-pgsql.cpp b/middle-pgsql.cpp index 6191d206b..b9a12eb4a 100644 --- a/middle-pgsql.cpp +++ b/middle-pgsql.cpp @@ -487,30 +487,33 @@ bool middle_pgsql_t::ways_get(osmid_t id, osmium::memory::Buffer &buffer) const return true; } -size_t middle_pgsql_t::ways_get_list(const idlist_t &ids, osmium::memory::Buffer &buffer) const +size_t middle_pgsql_t::rel_way_members_get(osmium::Relation const &rel, + rolelist_t *roles, + osmium::memory::Buffer &buffer) const { - if (ids.empty()) - return 0; - char tmp[16]; - std::unique_ptr tmp2(new (std::nothrow) char[ids.size() * 16]); char const *paramValues[1]; - if (tmp2 == nullptr) return 0; //failed to allocate memory, return */ + // create a list of ids in tmp2 to query the database + std::string tmp2("{"); + for (auto const &m : rel.members()) { + if (m.type() == osmium::item_type::way) { + snprintf(tmp, sizeof(tmp), "%" PRIdOSMID ",", m.ref()); + tmp2.append(tmp); + } + } - // create a list of ids in tmp2 to query the database */ - sprintf(tmp2.get(), "{"); - for (auto id : ids) { - snprintf(tmp, sizeof(tmp), "%" PRIdOSMID ",", id); - strncat(tmp2.get(), tmp, sizeof(char)*(ids.size()*16 - 2)); + if (tmp2.length() == 1) { + return 0; // no ways found } - tmp2[strlen(tmp2.get()) - 1] = '}'; // replace last , with } to complete list of ids*/ + // replace last , with } to complete list of ids + tmp2[tmp2.length() - 1] = '}'; pgsql_endCopy(way_table); PGconn *sql_conn = way_table->sql_conn; - paramValues[0] = tmp2.get(); + paramValues[0] = tmp2.c_str(); PGresult *res = pgsql_execPrepared(sql_conn, "get_way_list", 1, paramValues, PGRES_TUPLES_OK); int countPG = PQntuples(res); @@ -520,22 +523,27 @@ size_t middle_pgsql_t::ways_get_list(const idlist_t &ids, osmium::memory::Buffer wayidspg.push_back(strtoosmid(PQgetvalue(res, i, 0), nullptr, 10)); } - // Match the list of ways coming from postgres in a different order // back to the list of ways given by the caller */ - int outres = 0; - for (auto id : ids) { + size_t outres = 0; + for (auto const &m : rel.members()) { + if (m.type() != osmium::item_type::way) { + continue; + } for (int j = 0; j < countPG; j++) { - if (id == wayidspg[j]) { + if (m.ref() == wayidspg[j]) { { osmium::builder::WayBuilder builder(buffer); - builder.set_id(id); + builder.set_id(m.ref()); pgsql_parse_nodes(PQgetvalue(res, j, 1), buffer, builder); pgsql_parse_tags(PQgetvalue(res, j, 2), buffer, builder); } buffer.commit(); + if (roles) { + roles->emplace_back(m.role()); + } outres++; break; } diff --git a/middle-pgsql.hpp b/middle-pgsql.hpp index 0756b1d5b..40186df9c 100644 --- a/middle-pgsql.hpp +++ b/middle-pgsql.hpp @@ -33,7 +33,8 @@ struct middle_pgsql_t : public slim_middle_t { void ways_set(osmium::Way const &way) override; bool ways_get(osmid_t id, osmium::memory::Buffer &buffer) const override; - size_t ways_get_list(const idlist_t &ids, osmium::memory::Buffer &buffer) const override; + size_t rel_way_members_get(osmium::Relation const &rel, rolelist_t *roles, + osmium::memory::Buffer &buffer) const override; void ways_delete(osmid_t id) override; void way_changed(osmid_t id) override; diff --git a/middle-ram.cpp b/middle-ram.cpp index 02540d9fa..97ea1a88f 100644 --- a/middle-ram.cpp +++ b/middle-ram.cpp @@ -118,15 +118,16 @@ bool middle_ram_t::ways_get(osmid_t id, osmium::memory::Buffer &buffer) const return true; } -size_t middle_ram_t::ways_get_list(const idlist_t &ids, osmium::memory::Buffer &buffer) const +size_t middle_ram_t::rel_way_members_get(osmium::Relation const &rel, + rolelist_t *roles, + osmium::memory::Buffer &buffer) const { - if (ids.empty()) { - return 0; - } - size_t count = 0; - for (auto const id: ids) { - if (ways_get(id, buffer)) { + for (auto const &m : rel.members()) { + if (m.type() == osmium::item_type::way && ways_get(m.ref(), buffer)) { + if (roles) { + roles->emplace_back(m.role()); + } ++count; } } diff --git a/middle-ram.hpp b/middle-ram.hpp index 34cbf6d32..67982552a 100644 --- a/middle-ram.hpp +++ b/middle-ram.hpp @@ -97,7 +97,8 @@ struct middle_ram_t : public middle_t { void ways_set(osmium::Way const &way) override; bool ways_get(osmid_t id, osmium::memory::Buffer &buffer) const override; - size_t ways_get_list(idlist_t const &ids, osmium::memory::Buffer &buffer) const override; + size_t rel_way_members_get(osmium::Relation const &rel, rolelist_t *roles, + osmium::memory::Buffer &buffer) const override; int ways_delete(osmid_t id); int way_changed(osmid_t id); diff --git a/middle.hpp b/middle.hpp index 48a2d4dd8..0e6a6ab2b 100644 --- a/middle.hpp +++ b/middle.hpp @@ -43,7 +43,17 @@ struct middle_query_t { */ virtual bool ways_get(osmid_t id, osmium::memory::Buffer &buffer) const = 0; - virtual size_t ways_get_list(idlist_t const &ids, osmium::memory::Buffer &buffer) const = 0; + /** + * Retrives the way members of a relation and stores them in + * the given osmium buffer. + * + * \param rel Relation to get the members for. + * \param[out] roles Roles for the ways that where retrived. + * \param[out] buffer Buffer where to store the members in. + */ + virtual size_t + rel_way_members_get(osmium::Relation const &rel, rolelist_t *roles, + osmium::memory::Buffer &buffer) const = 0; /** * Retrives a single relation from the relation storage diff --git a/output-gazetteer.cpp b/output-gazetteer.cpp index 07e339321..4deb42d97 100644 --- a/output-gazetteer.cpp +++ b/output-gazetteer.cpp @@ -741,15 +741,10 @@ int output_gazetteer_t::process_relation(osmium::Relation const &rel) return 0; /* get the boundary path (ways) */ - idlist_t xid2; - for (const auto& member: rel.members()) { - /* only interested in ways */ - if (member.type() == osmium::item_type::way) - xid2.push_back(member.ref()); - } - osmium_buffer.clear(); - if (xid2.empty() || (m_mid->ways_get_list(xid2, osmium_buffer) == 0)) { + auto num_ways = m_mid->rel_way_members_get(rel, nullptr, osmium_buffer); + + if (num_ways == 0) { if (m_options.append) delete_unused_full('R', rel.id()); diff --git a/output-multi.cpp b/output-multi.cpp index 1c7cc325f..953aa442c 100644 --- a/output-multi.cpp +++ b/output-multi.cpp @@ -363,7 +363,7 @@ int output_multi_t::process_relation(osmium::Relation const &rel, if (!filter) { //TODO: move this into geometry processor, figure a way to come back for tag transform //grab ways/nodes of the members in the relation, bail if none were used - if (m_relation_helper.set(rel.members(), (middle_t*)m_mid) < 1) + if (m_relation_helper.set(rel, (middle_t *)m_mid) < 1) return 0; //filter the tags on each member because we got them from the middle diff --git a/output-pgsql.cpp b/output-pgsql.cpp index dba17cff2..ccd38dc90 100644 --- a/output-pgsql.cpp +++ b/output-pgsql.cpp @@ -326,13 +326,13 @@ int output_pgsql_t::pgsql_process_relation(osmium::Relation const &rel, } buffer.clear(); - auto num_ways = m_mid->ways_get_list(xid2, buffer); + rolelist_t xrole; + auto num_ways = m_mid->rel_way_members_get(rel, &xrole, buffer); if (num_ways == 0) return 0; multitaglist_t xtags(num_ways, taglist_t()); - rolelist_t xrole(num_ways); size_t i = 0; for (auto const &w : buffer.select()) { @@ -348,13 +348,6 @@ int output_pgsql_t::pgsql_process_relation(osmium::Relation const &rel, //TODO: if the filter says that this member is now not interesting we //should decrement the count and remove his nodes and tags etc. for //now we'll just keep him with no tags so he will get filtered later - for (auto const &member : rel.members()) { - if (member.ref() == w.id() && - member.type() == osmium::item_type::way) { - xrole[i] = member.role(); - break; - } - } ++i; } diff --git a/tests/middle-tests.cpp b/tests/middle-tests.cpp index e764de548..f495e4179 100644 --- a/tests/middle-tests.cpp +++ b/tests/middle-tests.cpp @@ -201,12 +201,32 @@ int test_way_set(middle_t *mid) mid->commit(); // get it back - idlist_t ways; - ways.push_back(way_id); + osmium::memory::Buffer relbuf(4096, osmium::memory::Buffer::auto_grow::yes); + { + using namespace osmium::builder::attr; + osmium::builder::add_relation( + relbuf, _id(123), _member(osmium::item_type::node, 132), + _member(osmium::item_type::way, way_id, "outer")); + } + + auto const &rel = relbuf.get(0); + auto buf_pos = buffer.committed(); - size_t way_count = mid->ways_get_list(ways, buffer); + rolelist_t roles; + size_t way_count = mid->rel_way_members_get(rel, &roles, buffer); if (way_count != 1) { std::cerr << "ERROR: Unable to get way list.\n"; return 1; } + if (roles.size() != 1) { + std::cerr << "Bad length of role ist. Expected 1, got " << roles.size() + << ".\n"; + return 1; + } + + if (strcmp(roles[0], "outer") != 0) { + std::cerr << "Bad role. Expected 'outer', got '" << roles[0] << "'.\n"; + return 1; + } + auto &way = buffer.get(buf_pos); // check that it's the same if (way.nodes().size() != nds.size()) { diff --git a/tests/mockups.hpp b/tests/mockups.hpp index d8184b310..f5ab58e1f 100644 --- a/tests/mockups.hpp +++ b/tests/mockups.hpp @@ -19,7 +19,11 @@ struct dummy_middle_t : public middle_t { void ways_set(osmium::Way const &) override { } bool ways_get(osmid_t, osmium::memory::Buffer &) const override { return true; } - size_t ways_get_list(idlist_t const &, osmium::memory::Buffer &) const override { return 0; } + size_t rel_way_members_get(osmium::Relation const &, rolelist_t *, + osmium::memory::Buffer &) const override + { + return 0; + } void relations_set(osmium::Relation const &) override { } bool relations_get(osmid_t, osmium::memory::Buffer &) const override { return 0; } @@ -52,7 +56,11 @@ struct dummy_slim_middle_t : public slim_middle_t { void ways_set(osmium::Way const &) override { } bool ways_get(osmid_t, osmium::memory::Buffer &) const override { return true; } - size_t ways_get_list(idlist_t const &, osmium::memory::Buffer &) const override { return 0; } + size_t rel_way_members_get(osmium::Relation const &, rolelist_t *, + osmium::memory::Buffer &) const override + { + return 0; + } void relations_set(osmium::Relation const &) override { } bool relations_get(osmid_t, osmium::memory::Buffer &) const override { return 0; } From b27b1b37bda56c690c792ba56c4f149eebc0070b Mon Sep 17 00:00:00 2001 From: Sarah Hoffmann Date: Sat, 4 Feb 2017 18:02:41 +0100 Subject: [PATCH 16/30] add converter from hex to binary wkb Postgres returns wkbs in hexform whereas the parser expects a pure binary form. So convert before usage. --- expire-tiles.cpp | 7 ++++--- wkb.hpp | 23 +++++++++++++++++++++++ 2 files changed, 27 insertions(+), 3 deletions(-) diff --git a/expire-tiles.cpp b/expire-tiles.cpp index cd14c1f93..11f7558a2 100644 --- a/expire-tiles.cpp +++ b/expire-tiles.cpp @@ -464,9 +464,10 @@ int expire_tiles::from_db(table_t* table, osmid_t osm_id) { //dirty the stuff const char* wkb = nullptr; - while((wkb = wkbs.get_next())) - // XXX need to convert from hex to binary - from_wkb(wkb, osm_id); + while ((wkb = wkbs.get_next())) { + auto binwkb = ewkb::parser_t::wkb_from_hex(wkb); + from_wkb(binwkb.c_str(), osm_id); + } //return how many rows were affected return wkbs.get_count(); diff --git a/wkb.hpp b/wkb.hpp index 4569226d9..c775c4a39 100644 --- a/wkb.hpp +++ b/wkb.hpp @@ -210,6 +210,29 @@ class writer_t class parser_t { public: + inline static std::string wkb_from_hex(std::string const &wkb) + { + std::string out; + + bool front = true; + char outc; + for (char c : wkb) { + c -= 48; + if (c > 9) { + c -= 7; + } + if (front) { + outc = char(c << 4); + front = false; + } else { + out += outc | c; + front = true; + } + } + + return out; + } + explicit parser_t(char const *wkb) : m_wkb(wkb), m_pos(0) {} explicit parser_t(std::string const &wkb) : m_wkb(wkb.c_str()), m_pos(0) {} From 0d491cfac122441fecb9a268b712b88ee869c287 Mon Sep 17 00:00:00 2001 From: Sarah Hoffmann Date: Sat, 4 Feb 2017 22:11:23 +0100 Subject: [PATCH 17/30] output-multi: compute way_area only when there is a column with the name --- output-multi.cpp | 23 +++++++++++------------ output-multi.hpp | 3 ++- taginfo.cpp | 16 ++++++++++++++++ taginfo_impl.hpp | 1 + 4 files changed, 30 insertions(+), 13 deletions(-) diff --git a/output-multi.cpp b/output-multi.cpp index 953aa442c..263b0f625 100644 --- a/output-multi.cpp +++ b/output-multi.cpp @@ -33,7 +33,8 @@ output_multi_t::output_multi_t(const std::string &name, ways_done_tracker(new id_tracker()), m_expire(m_options.expire_tiles_zoom, m_options.expire_tiles_max_bbox, m_options.projection), - buffer(1024, osmium::memory::Buffer::auto_grow::yes) + buffer(1024, osmium::memory::Buffer::auto_grow::yes), + m_way_area(m_export_list->has_column(m_osm_type, "way_area")) { } @@ -49,7 +50,8 @@ output_multi_t::output_multi_t(const output_multi_t &other) ways_done_tracker(other.ways_done_tracker), m_expire(m_options.expire_tiles_zoom, m_options.expire_tiles_max_bbox, m_options.projection), - buffer(1024, osmium::memory::Buffer::auto_grow::yes) + buffer(1024, osmium::memory::Buffer::auto_grow::yes), + m_way_area(other.m_way_area) { } @@ -306,15 +308,14 @@ int output_multi_t::reprocess_way(osmium::Way *way, bool exists) } //check if we are keeping this way - int polygon = 0; taglist_t outtags; unsigned int filter = m_tagtransform->filter_tags( - *way, &polygon, 0, *m_export_list.get(), outtags, true); + *way, 0, 0, *m_export_list.get(), outtags, true); if (!filter) { m_mid->nodes_get_list(&(way->nodes())); auto geom = m_processor->process_way(*way); if (!geom.empty()) { - copy_to_table(way->id(), geom, outtags, polygon); + copy_to_table(way->id(), geom, outtags); } } return 0; @@ -322,10 +323,8 @@ int output_multi_t::reprocess_way(osmium::Way *way, bool exists) int output_multi_t::process_way(osmium::Way *way) { //check if we are keeping this way - int polygon = 0, roads = 0; taglist_t outtags; - auto filter = m_tagtransform->filter_tags(*way, &polygon, &roads, - *m_export_list.get(), outtags, true); + auto filter = m_tagtransform->filter_tags(*way, 0, 0, *m_export_list.get(), outtags, true); if (!filter) { //get the geom from the middle if (m_mid->nodes_get_list(&(way->nodes())) < 1) @@ -341,7 +340,7 @@ int output_multi_t::process_way(osmium::Way *way) { } else { // We wouldn't be interested in this as a relation, so no need to mark it pending. // TODO: Does this imply anything for non-multipolygon relations? - copy_to_table(way->id(), geom, outtags, polygon); + copy_to_table(way->id(), geom, outtags); } } } @@ -394,7 +393,7 @@ int output_multi_t::process_relation(osmium::Relation const &rel, auto geoms = m_processor->process_relation(rel, m_relation_helper.data); for (const auto geom : geoms) { - copy_to_table(-rel.id(), geom, outtags, make_polygon); + copy_to_table(-rel.id(), geom, outtags); } //TODO: should this loop be inside the if above just in case? @@ -435,10 +434,10 @@ void output_multi_t::copy_node_to_table(osmid_t id, std::string const &geom, */ void output_multi_t::copy_to_table(const osmid_t id, geometry_processor::wkb_t const &geom, - taglist_t &tags, int polygon) + taglist_t &tags) { // XXX really should depend on expected output type - if (polygon) { + if (m_way_area) { // It's a polygon table (implied by it turning into a poly), // and it got formed into a polygon, so expire as a polygon and write the geom auto area = diff --git a/output-multi.hpp b/output-multi.hpp index 804ebe0e0..5ff89616a 100644 --- a/output-multi.hpp +++ b/output-multi.hpp @@ -70,7 +70,7 @@ class output_multi_t : public output_t { int process_relation(osmium::Relation const &rel, bool exists, bool pending=false); void copy_node_to_table(osmid_t id, const std::string &geom, taglist_t &tags); void copy_to_table(const osmid_t id, geometry_processor::wkb_t const &geom, - taglist_t &tags, int polygon); + taglist_t &tags); std::unique_ptr m_tagtransform; std::unique_ptr m_export_list; @@ -83,6 +83,7 @@ class output_multi_t : public output_t { expire_tiles m_expire; relation_helper m_relation_helper; osmium::memory::Buffer buffer; + bool m_way_area; }; #endif diff --git a/taginfo.cpp b/taginfo.cpp index 1a2ed8d05..f30a87f52 100644 --- a/taginfo.cpp +++ b/taginfo.cpp @@ -70,6 +70,22 @@ const std::vector &export_list::get(osmium::item_type id) const { } } +bool export_list::has_column(osmium::item_type id, char const *name) const +{ + auto idx = item_type_to_nwr_index(id); + if (idx >= exportList.size()) { + return false; + } + + for (auto const &info : exportList[idx]) { + if (info.name == name) { + return true; + } + } + + return false; +} + columns_t export_list::normal_columns(osmium::item_type id) const { columns_t columns; diff --git a/taginfo_impl.hpp b/taginfo_impl.hpp index cf2e554af..c6313bef7 100644 --- a/taginfo_impl.hpp +++ b/taginfo_impl.hpp @@ -42,6 +42,7 @@ struct export_list { const std::vector &get(osmium::item_type id) const; columns_t normal_columns(osmium::item_type id) const; + bool has_column(osmium::item_type id, char const *name) const; std::vector > exportList; /* Indexed osmium nwr index */ }; From eb9ec381a72621d2ac8074ad05769dea14b10916 Mon Sep 17 00:00:00 2001 From: Sarah Hoffmann Date: Sat, 4 Feb 2017 22:18:59 +0100 Subject: [PATCH 18/30] remove unused exclude-invalid-polygon option --- README.md | 2 +- options.cpp | 7 +------ options.hpp | 1 - tests/test-options-parse.cpp | 1 - 4 files changed, 2 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index f2b04f8db..5f78f726d 100644 --- a/README.md +++ b/README.md @@ -110,7 +110,7 @@ cmake .. -G "Unix Makefiles" -DCMAKE_BUILD_TYPE=Debug -DBUILD_TESTS=ON ## Usage ## -Osm2pgsql has one program, the executable itself, which has **44** command line +Osm2pgsql has one program, the executable itself, which has **43** command line options. Before loading into a database, the database must be created and the PostGIS diff --git a/options.cpp b/options.cpp index 9d032c761..7aef5189f 100644 --- a/options.cpp +++ b/options.cpp @@ -61,7 +61,6 @@ namespace {"drop", 0, 0, 206}, {"unlogged", 0, 0, 207}, {"flat-nodes",1,0,209}, - {"exclude-invalid-polygon",0,0,210}, {"tag-transform-script",1,0,212}, {"reproject-area",0,0,213}, {0, 0, 0, 0} @@ -204,7 +203,6 @@ namespace -K|--keep-coastlines Keep coastline data rather than filtering it out.\n\ By default natural=coastline tagged data will be discarded\n\ because renderers usually have shape files for them.\n\ - --exclude-invalid-polygon do not attempt to recover invalid geometries.\n\ --reproject-area compute area column using spherical mercator coordinates.\n\ -h|--help Help information.\n\ -v|--verbose Verbose output.\n"); @@ -278,7 +276,7 @@ options_t::options_t() alloc_chunkwise(ALLOC_SPARSE), #endif droptemp(false), unlogged(false), hstore_match_only(false), - flat_node_cache_enabled(false), excludepoly(false), reproject_area(false), + flat_node_cache_enabled(false), reproject_area(false), flat_node_file(boost::none), tag_transform_script(boost::none), tag_transform_node_func(boost::none), tag_transform_way_func(boost::none), tag_transform_rel_func(boost::none), tag_transform_rel_mem_func(boost::none), @@ -453,9 +451,6 @@ options_t::options_t(int argc, char *argv[]): options_t() flat_node_cache_enabled = true; flat_node_file = optarg; break; - case 210: - excludepoly = true; - break; case 211: enable_hstore_index = true; break; diff --git a/options.hpp b/options.hpp index 8b1490410..6865a0cc8 100644 --- a/options.hpp +++ b/options.hpp @@ -71,7 +71,6 @@ struct options_t { bool unlogged; ///< use unlogged tables where possible bool hstore_match_only; ///< only copy rows that match an explicitly listed key bool flat_node_cache_enabled; - bool excludepoly; bool reproject_area; boost::optional flat_node_file; /** diff --git a/tests/test-options-parse.cpp b/tests/test-options-parse.cpp index 3b280c560..b90ecff6f 100644 --- a/tests/test-options-parse.cpp +++ b/tests/test-options-parse.cpp @@ -265,7 +265,6 @@ void test_random_perms() add_arg_or_not("--extra-attributes", args, options.extra_attributes); add_arg_or_not("--multi-geometry", args, options.enable_multi); add_arg_or_not("--keep-coastlines", args, options.keep_coastlines); - add_arg_or_not("--exclude-invalid-polygon", args, options.excludepoly); //add the input file args.push_back("tests/liechtenstein-2013-08-03.osm.pbf"); From 7a354f2247ab74f73809319cc3f089182119accb Mon Sep 17 00:00:00 2001 From: Sarah Hoffmann Date: Sat, 4 Feb 2017 22:28:27 +0100 Subject: [PATCH 19/30] remove unused code --- osmium-builder.hpp | 2 +- reprojection.hpp | 11 ----------- 2 files changed, 1 insertion(+), 12 deletions(-) diff --git a/osmium-builder.hpp b/osmium-builder.hpp index 0e5759b38..ce6ba349a 100644 --- a/osmium-builder.hpp +++ b/osmium-builder.hpp @@ -22,7 +22,7 @@ class osmium_builder_t explicit osmium_builder_t(std::shared_ptr const &proj, bool build_multigeoms) : m_proj(proj), m_buffer(1024, osmium::memory::Buffer::auto_grow::yes), - m_writer(m_proj->epsg()), m_build_multigeoms(build_multigeoms) + m_writer(m_proj->target_srs()), m_build_multigeoms(build_multigeoms) { } diff --git a/reprojection.hpp b/reprojection.hpp index f37dd54d9..c0609dfd9 100644 --- a/reprojection.hpp +++ b/reprojection.hpp @@ -50,17 +50,6 @@ class reprojection : public boost::noncopyable return target_srs() == PROJ_LATLONG; } - // interface for osmium GeometryFactory - - osmium::geom::Coordinates operator()(osmium::Location location) const - { - return reproject(location); - } - - int epsg() const { return target_srs(); } - - std::string proj_string() const { return target_desc(); } - /** * Create a reprojection object with target srs `srs`. * From e5837d797be32d0c18c3a530eb20f71f919ed436 Mon Sep 17 00:00:00 2001 From: Paul Norman Date: Tue, 7 Feb 2017 14:38:13 -0800 Subject: [PATCH 20/30] Remove remaining geos references Geos is no longer required, or needed by CI --- .travis.yml | 3 --- README.md | 8 +++----- appveyor.yml | 1 - 3 files changed, 3 insertions(+), 9 deletions(-) diff --git a/.travis.yml b/.travis.yml index c5070e2d3..863ddae8f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,8 +8,6 @@ addons: packages: - g++-4.8 - libexpat1-dev - - libgeos-dev - - libgeos++-dev - libpq-dev - libbz2-dev - libproj-dev @@ -44,7 +42,6 @@ install: before_script: - $CXX --version - xml2-config --version - - geos-config --version - proj | head -n1 - lua -v script: diff --git a/README.md b/README.md index 5f78f726d..8c74b3282 100644 --- a/README.md +++ b/README.md @@ -35,7 +35,6 @@ to configure and build itself and requires Required libraries are * [expat](http://www.libexpat.org/) -* [geos](http://trac.osgeo.org/geos) * [proj](http://proj.osgeo.org/) * [bzip2](http://www.bzip.org/) * [zlib](http://www.zlib.net/) @@ -57,15 +56,14 @@ On a Debian or Ubuntu system, this can be done with: ```sh sudo apt-get install make cmake g++ libboost-dev libboost-system-dev \ libboost-filesystem-dev libexpat1-dev zlib1g-dev \ - libbz2-dev libpq-dev libgeos-dev libgeos++-dev libproj-dev lua5.2 \ - liblua5.2-dev + libbz2-dev libpq-dev libproj-dev lua5.2 liblua5.2-dev ``` On a Fedora system, use ```sh sudo yum install cmake gcc-c++ boost-devel expat-devel zlib-devel bzip2-devel \ - postgresql-devel geos-devel proj-devel proj-epsg lua-devel + postgresql-devel proj-devel proj-epsg lua-devel ``` On RedHat / CentOS first run `sudo yum install epel-release` then install @@ -75,7 +73,7 @@ On a FreeBSD system, use ```sh pkg install devel/cmake devel/boost-libs textproc/expat2 \ - databases/postgresql94-client graphics/geos graphics/proj lang/lua52 + databases/postgresql94-client graphics/proj lang/lua52 ``` Once dependencies are installed, use CMake to build the Makefiles in a separate folder diff --git a/appveyor.yml b/appveyor.yml index 60f64e7a0..55be325f8 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -47,7 +47,6 @@ build_script: - copy /y ..\*.style osm2pgsql-bin - copy /y ..\*.lua osm2pgsql-bin - copy /y %PREFIX%\bin\lua.dll osm2pgsql-bin - - copy /y %PREFIX%\bin\geos.dll osm2pgsql-bin - copy /y "%PSQL_ROOT%\bin\libpq.dll" osm2pgsql-bin - copy /y "%PSQL_ROOT%\bin\libintl-8.dll" osm2pgsql-bin - copy /y "%PSQL_ROOT%\bin\libeay32.dll" osm2pgsql-bin From 04895af55264e77f43ebc63d2d79820803305579 Mon Sep 17 00:00:00 2001 From: Paul Norman Date: Wed, 8 Feb 2017 09:59:30 -0800 Subject: [PATCH 21/30] Remove exclude invalid polygon references --- docs/usage.md | 3 --- 1 file changed, 3 deletions(-) diff --git a/docs/usage.md b/docs/usage.md index 50fd1af42..39ec8e937 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -144,6 +144,3 @@ reimporting the database, at the cost of a * ``--keep-coastlines`` disables a hard-coded rule that would otherwise discard ``natural=coastline`` ways. - -* ``--exclude-invalid-polygon`` prevents osm2pgsql from attempting to form - valid polygons from invalid ones and just rejects the invalid ones. From 7e880abf6e96c364504ce9cd940a14e92d4fad40 Mon Sep 17 00:00:00 2001 From: Sarah Hoffmann Date: Fri, 10 Feb 2017 21:25:54 +0100 Subject: [PATCH 22/30] update to newest libosmium version This includes the custom modifications in the official libosmium branch. --- contrib/libosmium/README.contrib | 2 +- contrib/libosmium/osmium/area/assembler.hpp | 1390 ++--------------- .../osmium/area/assembler_config.hpp | 152 ++ .../osmium/area/detail/basic_assembler.hpp | 1204 ++++++++++++++ .../osmium/area/detail/segment_list.hpp | 51 +- .../libosmium/osmium/area/detail/vector.hpp | 2 +- .../libosmium/osmium/area/geom_assembler.hpp | 131 ++ .../osmium/area/problem_reporter.hpp | 9 + .../area/problem_reporter_exception.hpp | 42 +- .../osmium/area/problem_reporter_ogr.hpp | 12 +- .../osmium/area/problem_reporter_stream.hpp | 5 + contrib/libosmium/osmium/area/stats.hpp | 5 +- contrib/libosmium/osmium/builder/builder.hpp | 18 +- .../osmium/builder/builder_helper.hpp | 6 +- .../osmium/builder/osm_object_builder.hpp | 20 +- contrib/libosmium/osmium/geom/factory.hpp | 2 +- .../osmium/geom/mercator_projection.hpp | 6 +- contrib/libosmium/osmium/geom/projection.hpp | 54 +- contrib/libosmium/osmium/index/id_set.hpp | 1 + .../osmium/index/map/sparse_mem_table.hpp | 2 +- contrib/libosmium/osmium/index/nwr_array.hpp | 59 + .../libosmium/osmium/index/relations_map.hpp | 280 +++- .../osmium/io/detail/pbf_input_format.hpp | 2 +- .../osmium/io/detail/pbf_output_format.hpp | 1 - .../osmium/io/detail/protobuf_tags.hpp | 2 +- .../osmium/io/detail/write_thread.hpp | 2 +- .../osmium/io/detail/xml_output_format.hpp | 1 - contrib/libosmium/osmium/osm/crc.hpp | 2 +- .../libosmium/osmium/relations/collector.hpp | 11 - contrib/libosmium/osmium/tags/filter.hpp | 33 +- .../libosmium/osmium/tags/regex_filter.hpp | 4 +- contrib/libosmium/osmium/tags/taglist.hpp | 13 +- .../osmium/thread/function_wrapper.hpp | 2 +- contrib/libosmium/osmium/thread/pool.hpp | 2 +- contrib/libosmium/osmium/thread/util.hpp | 2 +- contrib/libosmium/osmium/util/file.hpp | 10 +- osmium-builder.cpp | 10 +- 37 files changed, 2040 insertions(+), 1510 deletions(-) create mode 100644 contrib/libosmium/osmium/area/assembler_config.hpp create mode 100644 contrib/libosmium/osmium/area/detail/basic_assembler.hpp create mode 100644 contrib/libosmium/osmium/area/geom_assembler.hpp create mode 100644 contrib/libosmium/osmium/index/nwr_array.hpp diff --git a/contrib/libosmium/README.contrib b/contrib/libosmium/README.contrib index a97e3229d..98e88d6cd 100644 --- a/contrib/libosmium/README.contrib +++ b/contrib/libosmium/README.contrib @@ -1,2 +1,2 @@ Source: https://github.com/osmcode/libosmium -Revision: v2.11.0 +Revision: ab8236405330607932c8f6395956ec6d9bcdee5c diff --git a/contrib/libosmium/osmium/area/assembler.hpp b/contrib/libosmium/osmium/area/assembler.hpp index 7495503d5..8ec793579 100644 --- a/contrib/libosmium/osmium/area/assembler.hpp +++ b/contrib/libosmium/osmium/area/assembler.hpp @@ -38,253 +38,50 @@ DEALINGS IN THE SOFTWARE. #include #include #include +#include #include #include -#include #include #include #include -#include -#include #include #include +#include #include #include #include #include #include -#include #include #include #include #include #include #include -#include -#include - -#include -#include -#include -#include -#include namespace osmium { namespace area { - /** - * Configuration for osmium::area::Assembler objects. Create this - * once, set the options you want and then re-use it every time you - * create an Assembler object. - */ - struct AssemblerConfig { - - /** - * Optional pointer to problem reporter. - */ - osmium::area::ProblemReporter* problem_reporter = nullptr; - - /** - * Debug level. If this is greater than zero, debug messages will - * be printed to stderr. Available levels are 1 to 3. Note that - * level 2 and above will generate a lot of messages! - */ - int debug_level = 0; - - /** - * The roles of multipolygon members are ignored when assembling - * multipolygons, because they are often missing or wrong. If this - * is set, the roles are checked after the multipolygons are built - * against what the assembly process decided where the inner and - * outer rings are. This slows down the processing, so it only - * makes sense if you want to get the problem reports. - */ - bool check_roles = false; - - /** - * When the assembler can't create an area, usually because its - * geometry would be invalid, it will create an "empty" area object - * without rings. This allows you to detect where an area was - * invalid. - * - * If this is set to false, invalid areas will simply be discarded. - */ - bool create_empty_areas = true; - - /** - * Create areas for (multi)polygons where the tags are on the - * relation. - * - * If this is set to false, those areas will simply be discarded. - */ - bool create_new_style_polygons = true; - - /** - * Create areas for (multi)polygons where the tags are on the - * outer way(s). - * - * If this is set to false, those areas will simply be discarded. - */ - bool create_old_style_polygons = true; - - /** - * Create areas for polygons created from ways. - * - * If this is set to false, those areas will simply be discarded. - */ - bool create_way_polygons = true; - - /** - * Keep the type tag from multipolygon relations on the area - * object. By default this is false, and the type tag will be - * removed. - */ - bool keep_type_tag = false; - - AssemblerConfig() noexcept = default; - - /** - * Constructor - * @deprecated Use default constructor and set values afterwards. - */ - explicit AssemblerConfig(osmium::area::ProblemReporter* pr, bool d = false) : - problem_reporter(pr), - debug_level(d) { - } - - /** - * Enable or disable debug output to stderr. This is for Osmium - * developers only. - * - * @deprecated Set debug_level directly. - */ - OSMIUM_DEPRECATED void enable_debug_output(bool d = true) { - debug_level = d; - } - - }; // struct AssemblerConfig - - namespace detail { - - using open_ring_its_type = std::list::iterator>; - - struct location_to_ring_map { - osmium::Location location; - open_ring_its_type::iterator ring_it; - bool start; - - location_to_ring_map(const osmium::Location& l, const open_ring_its_type::iterator& r, bool s) noexcept : - location(l), - ring_it(r), - start(s) { - } - - explicit location_to_ring_map(const osmium::Location& l) noexcept : - location(l), - ring_it(), - start(false) { - } - - const detail::ProtoRing& ring() const noexcept { - return **ring_it; - } - - }; // struct location_to_ring_map - - inline bool operator==(const location_to_ring_map& lhs, const location_to_ring_map& rhs) noexcept { - return lhs.location == rhs.location; - } - - inline bool operator<(const location_to_ring_map& lhs, const location_to_ring_map& rhs) noexcept { - return lhs.location < rhs.location; - } - - } // namespace detail - /** * Assembles area objects from closed ways or multipolygon relations * and their members. */ - class Assembler { - - using open_ring_its_type = detail::open_ring_its_type; - using location_to_ring_map = detail::location_to_ring_map; - - struct slocation { - - static constexpr const uint32_t invalid_item = 1 << 30; - - uint32_t item : 31; - uint32_t reverse : 1; - - slocation() noexcept : - item(invalid_item), - reverse(false) { - } - - explicit slocation(uint32_t n, bool r = false) noexcept : - item(n), - reverse(r) { - } - - osmium::Location location(const detail::SegmentList& segment_list) const noexcept { - const auto& segment = segment_list[item]; - return reverse ? segment.second().location() : segment.first().location(); - } - - const osmium::NodeRef& node_ref(const detail::SegmentList& segment_list) const noexcept { - const auto& segment = segment_list[item]; - return reverse ? segment.second() : segment.first(); - } - - osmium::Location location(const detail::SegmentList& segment_list, const osmium::Location& default_location) const noexcept { - if (item == invalid_item) { - return default_location; - } - return location(segment_list); - } - - }; // struct slocation - - // Configuration settings for this Assembler - const AssemblerConfig& m_config; - - // List of segments (connection between two nodes) - osmium::area::detail::SegmentList m_segment_list; - - // The rings we are building from the segments - std::list m_rings; - - // All node locations - std::vector m_locations; - - // All locations where more than two segments start/end - std::vector m_split_locations; - - // Statistics - area_stats m_stats; - - // The number of members the multipolygon relation has - size_t m_num_members = 0; - - bool debug() const noexcept { - return m_config.debug_level > 1; - } + class Assembler : public detail::BasicAssembler { bool report_ways() const noexcept { - if (!m_config.problem_reporter) { + if (!config().problem_reporter) { return false; } - return m_stats.duplicate_nodes || - m_stats.duplicate_segments || - m_stats.intersections || - m_stats.open_rings || - m_stats.short_ways || - m_stats.touching_rings || - m_stats.ways_in_multiple_rings || - m_stats.wrong_role; + return stats().duplicate_nodes || + stats().duplicate_segments || + stats().intersections || + stats().open_rings || + stats().short_ways || + stats().touching_rings || + stats().ways_in_multiple_rings || + stats().wrong_role; } void add_tags_to_area(osmium::builder::AreaBuilder& builder, const osmium::Way& way) const { @@ -295,7 +92,7 @@ namespace osmium { std::map counter; for (const osmium::Way* way : ways) { for (const auto& tag : way->tags()) { - std::string kv {tag.key()}; + std::string kv{tag.key()}; kv.append(1, '\0'); kv.append(tag.value()); ++counter[kv]; @@ -342,7 +139,7 @@ namespace osmium { } void add_tags_to_area(osmium::builder::AreaBuilder& builder, const osmium::Relation& relation) { - const auto count = std::count_if(relation.tags().cbegin(), relation.tags().cend(), filter()); + const auto count = std::count_if(relation.tags().cbegin(), relation.tags().cend(), std::cref(filter())); if (debug()) { std::cerr << " found " << count << " tags on relation (without ignored ones)\n"; @@ -353,18 +150,18 @@ namespace osmium { std::cerr << " use tags from relation\n"; } - if (m_config.keep_type_tag) { + if (config().keep_type_tag) { builder.add_item(relation.tags()); } else { copy_tags_without_type(builder, relation.tags()); } } else { - ++m_stats.no_tags_on_relation; + ++stats().no_tags_on_relation; if (debug()) { std::cerr << " use tags from outer ways\n"; } std::set ways; - for (const auto& ring : m_rings) { + for (const auto& ring : rings()) { if (ring.is_outer()) { ring.get_ways(ways); } @@ -384,989 +181,12 @@ namespace osmium { } } - template - static void build_ring_from_proto_ring(osmium::builder::AreaBuilder& builder, const detail::ProtoRing& ring) { - TBuilder ring_builder{builder}; - ring_builder.add_node_ref(ring.get_node_ref_start()); - for (const auto& segment : ring.segments()) { - ring_builder.add_node_ref(segment->stop()); - } - } - - /** - * Append each outer ring together with its inner rings to the - * area in the buffer. - */ - void add_rings_to_area(osmium::builder::AreaBuilder& builder) const { - for (const detail::ProtoRing& ring : m_rings) { - if (ring.is_outer()) { - build_ring_from_proto_ring(builder, ring); - for (const detail::ProtoRing* inner : ring.inner_rings()) { - build_ring_from_proto_ring(builder, *inner); - } - } - } - } - - void check_inner_outer_roles() { - if (debug()) { - std::cerr << " Checking inner/outer roles\n"; - } - - std::unordered_map way_rings; - std::unordered_set ways_in_multiple_rings; - - for (const detail::ProtoRing& ring : m_rings) { - for (const auto& segment : ring.segments()) { - assert(segment->way()); - - if (!segment->role_empty() && (ring.is_outer() ? !segment->role_outer() : !segment->role_inner())) { - ++m_stats.wrong_role; - if (debug()) { - std::cerr << " Segment " << *segment << " from way " << segment->way()->id() << " has role '" << segment->role_name() - << "', but should have role '" << (ring.is_outer() ? "outer" : "inner") << "'\n"; - } - if (m_config.problem_reporter) { - if (ring.is_outer()) { - m_config.problem_reporter->report_role_should_be_outer(segment->way()->id(), segment->first().location(), segment->second().location()); - } else { - m_config.problem_reporter->report_role_should_be_inner(segment->way()->id(), segment->first().location(), segment->second().location()); - } - } - } - - auto& r = way_rings[segment->way()]; - if (!r) { - r = ˚ - } else if (r != &ring) { - ways_in_multiple_rings.insert(segment->way()); - } - - } - } - - for (const osmium::Way* way : ways_in_multiple_rings) { - ++m_stats.ways_in_multiple_rings; - if (debug()) { - std::cerr << " Way " << way->id() << " is in multiple rings\n"; - } - if (m_config.problem_reporter) { - m_config.problem_reporter->report_way_in_multiple_rings(*way); - } - } - - } - - detail::NodeRefSegment* get_next_segment(const osmium::Location& location) { - auto it = std::lower_bound(m_locations.begin(), m_locations.end(), slocation{}, [this, &location](const slocation& lhs, const slocation& rhs) { - return lhs.location(m_segment_list, location) < rhs.location(m_segment_list, location); - }); - - assert(it != m_locations.end()); - if (m_segment_list[it->item].is_done()) { - ++it; - } - assert(it != m_locations.end()); - - assert(!m_segment_list[it->item].is_done()); - return &m_segment_list[it->item]; - } - - class rings_stack_element { - - int32_t m_y; - detail::ProtoRing* m_ring_ptr; - - public: - - rings_stack_element(int32_t y, detail::ProtoRing* ring_ptr) : - m_y(y), - m_ring_ptr(ring_ptr) { - } - - int32_t y() const noexcept { - return m_y; - } - - const detail::ProtoRing& ring() const noexcept { - return *m_ring_ptr; - } - - detail::ProtoRing* ring_ptr() noexcept { - return m_ring_ptr; - } - - bool operator==(const rings_stack_element& rhs) const noexcept { - return m_ring_ptr == rhs.m_ring_ptr; - } - - bool operator<(const rings_stack_element& rhs) const noexcept { - return m_y < rhs.m_y; - } - - }; // class ring_stack_element - - using rings_stack = std::vector; - - void remove_duplicates(rings_stack& outer_rings) { - while (true) { - const auto it = std::adjacent_find(outer_rings.begin(), outer_rings.end()); - if (it == outer_rings.end()) { - return; - } - outer_rings.erase(it, std::next(it, 2)); - } - } - - detail::ProtoRing* find_enclosing_ring(detail::NodeRefSegment* segment) { - if (debug()) { - std::cerr << " Looking for ring enclosing " << *segment << "\n"; - } - - const auto location = segment->first().location(); - const auto end_location = segment->second().location(); - - while (segment->first().location() == location) { - if (segment == &m_segment_list.back()) { - break; - } - ++segment; - } - - int nesting = 0; - - rings_stack outer_rings; - while (segment >= &m_segment_list.front()) { - if (!segment->is_direction_done()) { - --segment; - continue; - } - if (debug()) { - std::cerr << " Checking against " << *segment << "\n"; - } - const osmium::Location& a = segment->first().location(); - const osmium::Location& b = segment->second().location(); - - if (segment->first().location() == location) { - const int64_t ax = a.x(); - const int64_t bx = b.x(); - const int64_t lx = end_location.x(); - const int64_t ay = a.y(); - const int64_t by = b.y(); - const int64_t ly = end_location.y(); - const auto z = (bx - ax)*(ly - ay) - (by - ay)*(lx - ax); - if (debug()) { - std::cerr << " Segment XXXX z=" << z << "\n"; - } - if (z > 0) { - nesting += segment->is_reverse() ? -1 : 1; - if (debug()) { - std::cerr << " Segment is below (nesting=" << nesting << ")\n"; - } - if (segment->ring()->is_outer()) { - if (debug()) { - std::cerr << " Segment belongs to outer ring\n"; - } - outer_rings.emplace_back(a.y(), segment->ring()); - } - } - } else if (a.x() <= location.x() && location.x() < b.x()) { - if (debug()) { - std::cerr << " Is in x range\n"; - } - - const int64_t ax = a.x(); - const int64_t bx = b.x(); - const int64_t lx = location.x(); - const int64_t ay = a.y(); - const int64_t by = b.y(); - const int64_t ly = location.y(); - const auto z = (bx - ax)*(ly - ay) - (by - ay)*(lx - ax); - - if (z >= 0) { - nesting += segment->is_reverse() ? -1 : 1; - if (debug()) { - std::cerr << " Segment is below (nesting=" << nesting << ")\n"; - } - if (segment->ring()->is_outer()) { - if (debug()) { - std::cerr << " Segment belongs to outer ring\n"; - } - const int32_t y = int32_t(ay + (by - ay) * (lx - ax) / (bx - ax)); - outer_rings.emplace_back(y, segment->ring()); - } - } - } - --segment; - } - - if (nesting % 2 == 0) { - if (debug()) { - std::cerr << " Decided that this is an outer ring\n"; - } - return nullptr; - } else { - if (debug()) { - std::cerr << " Decided that this is an inner ring\n"; - } - assert(!outer_rings.empty()); - - std::sort(outer_rings.rbegin(), outer_rings.rend()); - if (debug()) { - for (const auto& o : outer_rings) { - std::cerr << " y=" << o.y() << " " << o.ring() << "\n"; - } - } - - remove_duplicates(outer_rings); - if (debug()) { - std::cerr << " after remove duplicates:\n"; - for (const auto& o : outer_rings) { - std::cerr << " y=" << o.y() << " " << o.ring() << "\n"; - } - } - - assert(!outer_rings.empty()); - return outer_rings.front().ring_ptr(); - } - } - - bool is_split_location(const osmium::Location& location) const noexcept { - return std::find(m_split_locations.cbegin(), m_split_locations.cend(), location) != m_split_locations.cend(); - } - - uint32_t add_new_ring(slocation& node) { - detail::NodeRefSegment* segment = &m_segment_list[node.item]; - assert(!segment->is_done()); - - if (debug()) { - std::cerr << " Starting new ring at location " << node.location(m_segment_list) << " with segment " << *segment << "\n"; - } - - if (node.reverse) { - segment->reverse(); - } - - detail::ProtoRing* outer_ring = nullptr; - - if (segment != &m_segment_list.front()) { - outer_ring = find_enclosing_ring(segment); - } - segment->mark_direction_done(); - - m_rings.emplace_back(segment); - detail::ProtoRing* ring = &m_rings.back(); - if (outer_ring) { - if (debug()) { - std::cerr << " This is an inner ring. Outer ring is " << *outer_ring << "\n"; - } - outer_ring->add_inner_ring(ring); - ring->set_outer_ring(outer_ring); - } else if (debug()) { - std::cerr << " This is an outer ring\n"; - } - - const osmium::Location& first_location = node.location(m_segment_list); - osmium::Location last_location = segment->stop().location(); - - uint32_t nodes = 1; - while (first_location != last_location) { - ++nodes; - detail::NodeRefSegment* next_segment = get_next_segment(last_location); - next_segment->mark_direction_done(); - if (next_segment->start().location() != last_location) { - next_segment->reverse(); - } - ring->add_segment_back(next_segment); - if (debug()) { - std::cerr << " Next segment is " << *next_segment << "\n"; - } - last_location = next_segment->stop().location(); - } - - ring->fix_direction(); - - if (debug()) { - std::cerr << " Completed ring: " << *ring << "\n"; - } - - return nodes; - } - - uint32_t add_new_ring_complex(slocation& node) { - detail::NodeRefSegment* segment = &m_segment_list[node.item]; - assert(!segment->is_done()); - - if (debug()) { - std::cerr << " Starting new ring at location " << node.location(m_segment_list) << " with segment " << *segment << "\n"; - } - - if (node.reverse) { - segment->reverse(); - } - - m_rings.emplace_back(segment); - detail::ProtoRing* ring = &m_rings.back(); - - const osmium::Location& first_location = node.location(m_segment_list); - osmium::Location last_location = segment->stop().location(); - - uint32_t nodes = 1; - while (first_location != last_location && !is_split_location(last_location)) { - ++nodes; - detail::NodeRefSegment* next_segment = get_next_segment(last_location); - if (next_segment->start().location() != last_location) { - next_segment->reverse(); - } - ring->add_segment_back(next_segment); - if (debug()) { - std::cerr << " Next segment is " << *next_segment << "\n"; - } - last_location = next_segment->stop().location(); - } - - if (debug()) { - if (first_location == last_location) { - std::cerr << " Completed ring: " << *ring << "\n"; - } else { - std::cerr << " Completed partial ring: " << *ring << "\n"; - } - } - - return nodes; - } - - void create_locations_list() { - m_locations.reserve(m_segment_list.size() * 2); - - for (uint32_t n = 0; n < m_segment_list.size(); ++n) { - m_locations.emplace_back(n, false); - m_locations.emplace_back(n, true); - } - - std::stable_sort(m_locations.begin(), m_locations.end(), [this](const slocation& lhs, const slocation& rhs) { - return lhs.location(m_segment_list) < rhs.location(m_segment_list); - }); - } - - void find_inner_outer_complex(detail::ProtoRing* ring) { - detail::ProtoRing* outer_ring = find_enclosing_ring(ring->min_segment()); - if (outer_ring) { - outer_ring->add_inner_ring(ring); - ring->set_outer_ring(outer_ring); - } - ring->fix_direction(); - ring->mark_direction_done(); - } - - void find_inner_outer_complex() { - if (debug()) { - std::cerr << " Finding inner/outer rings\n"; - } - std::vector rings; - rings.reserve(m_rings.size()); - for (auto& ring : m_rings) { - if (ring.closed()) { - rings.push_back(&ring); - } - } - - if (rings.empty()) { - return; - } - - std::sort(rings.begin(), rings.end(), [](detail::ProtoRing* a, detail::ProtoRing* b) { - return a->min_segment() < b->min_segment(); - }); - - rings.front()->fix_direction(); - rings.front()->mark_direction_done(); - if (debug()) { - std::cerr << " First ring is outer: " << *rings.front() << "\n"; - } - for (auto it = std::next(rings.begin()); it != rings.end(); ++it) { - if (debug()) { - std::cerr << " Checking (at min segment " << *((*it)->min_segment()) << ") ring " << **it << "\n"; - } - find_inner_outer_complex(*it); - if (debug()) { - std::cerr << " Ring is " << ((*it)->is_outer() ? "OUTER: " : "INNER: ") << **it << "\n"; - } - } - } - - /** - * Finds all locations where more than two segments meet. If there - * are any open rings found along the way, they are reported - * and the function returns false. - */ - bool find_split_locations() { - osmium::Location previous_location; - for (auto it = m_locations.cbegin(); it != m_locations.cend(); ++it) { - const osmium::NodeRef& nr = it->node_ref(m_segment_list); - const osmium::Location& loc = nr.location(); - if (std::next(it) == m_locations.cend() || loc != std::next(it)->location(m_segment_list)) { - if (debug()) { - std::cerr << " Found open ring at " << nr << "\n"; - } - if (m_config.problem_reporter) { - const auto& segment = m_segment_list[it->item]; - m_config.problem_reporter->report_ring_not_closed(nr, segment.way()); - } - ++m_stats.open_rings; - } else { - if (loc == previous_location && (m_split_locations.empty() || m_split_locations.back() != previous_location )) { - m_split_locations.push_back(previous_location); - } - ++it; - if (it == m_locations.end()) { - break; - } - } - previous_location = loc; - } - return m_stats.open_rings == 0; - } - - void create_rings_simple_case() { - auto count_remaining = m_segment_list.size(); - for (slocation& sl : m_locations) { - const detail::NodeRefSegment& segment = m_segment_list[sl.item]; - if (!segment.is_done()) { - count_remaining -= add_new_ring(sl); - if (count_remaining == 0) { - return; - } - } - } - } - - std::vector create_location_to_ring_map(open_ring_its_type& open_ring_its) { - std::vector xrings; - xrings.reserve(open_ring_its.size() * 2); - - for (auto it = open_ring_its.begin(); it != open_ring_its.end(); ++it) { - if (debug()) { - std::cerr << " Ring: " << **it << "\n"; - } - xrings.emplace_back((*it)->get_node_ref_start().location(), it, true); - xrings.emplace_back((*it)->get_node_ref_stop().location(), it, false); - } - - std::sort(xrings.begin(), xrings.end()); - - return xrings; - } - - void merge_two_rings(open_ring_its_type& open_ring_its, const location_to_ring_map& m1, const location_to_ring_map& m2) { - auto& r1 = *m1.ring_it; - auto& r2 = *m2.ring_it; - - if (r1->get_node_ref_stop().location() == r2->get_node_ref_start().location()) { - r1->join_forward(*r2); - } else if (r1->get_node_ref_stop().location() == r2->get_node_ref_stop().location()) { - r1->join_backward(*r2); - } else if (r1->get_node_ref_start().location() == r2->get_node_ref_start().location()) { - r1->reverse(); - r1->join_forward(*r2); - } else if (r1->get_node_ref_start().location() == r2->get_node_ref_stop().location()) { - r1->reverse(); - r1->join_backward(*r2); - } else { - assert(false); - } - - m_rings.erase(r2); - open_ring_its.remove(r2); - - if (r1->closed()) { - open_ring_its.remove(r1); - } - } - - bool try_to_merge(open_ring_its_type& open_ring_its) { - if (open_ring_its.empty()) { - return false; - } - - if (debug()) { - std::cerr << " Trying to merge " << open_ring_its.size() << " open rings\n"; - } - - std::vector xrings = create_location_to_ring_map(open_ring_its); - - auto it = xrings.cbegin(); - while (it != xrings.cend()) { - it = std::adjacent_find(it, xrings.cend()); - if (it == xrings.cend()) { - return false; - } - auto after = std::next(it, 2); - if (after == xrings.cend() || after->location != it->location) { - if (debug()) { - std::cerr << " Merging two rings\n"; - } - merge_two_rings(open_ring_its, *it, *std::next(it)); - return true; - } - while (it != xrings.cend() && it->location == after->location) { - ++it; - } - } - - return false; - } - - bool there_are_open_rings() const noexcept { - return std::any_of(m_rings.cbegin(), m_rings.cend(), [](const detail::ProtoRing& ring){ - return !ring.closed(); - }); - } - - struct candidate { - int64_t sum; - std::vector> rings; - osmium::Location start_location; - osmium::Location stop_location; - - explicit candidate(location_to_ring_map& ring, bool reverse) : - sum(ring.ring().sum()), - rings(), - start_location(ring.ring().get_node_ref_start().location()), - stop_location(ring.ring().get_node_ref_stop().location()) { - rings.emplace_back(ring, reverse); - } - - bool closed() const noexcept { - return start_location == stop_location; - } - - }; - - void find_candidates(std::vector& candidates, std::unordered_set& loc_done, const std::vector& xrings, candidate& cand) { - if (debug()) { - std::cerr << " find_candidates sum=" << cand.sum << " start=" << cand.start_location << " stop=" << cand.stop_location << "\n"; - for (const auto& ring : cand.rings) { - std::cerr << " " << ring.first.ring() << (ring.second ? " reverse" : "") << "\n"; - } - } - - const auto connections = make_range(std::equal_range(xrings.cbegin(), - xrings.cend(), - location_to_ring_map{cand.stop_location})); - - assert(connections.begin() != connections.end()); - - assert(!cand.rings.empty()); - const detail::ProtoRing* ring_leading_here = &cand.rings.back().first.ring(); - for (const location_to_ring_map& m : connections) { - const detail::ProtoRing& ring = m.ring(); - - if (&ring != ring_leading_here) { - if (debug()) { - std::cerr << " next possible connection: " << ring << (m.start ? "" : " reverse") << "\n"; - } - - candidate c = cand; - if (m.start) { - c.rings.emplace_back(m, false); - c.stop_location = ring.get_node_ref_stop().location(); - c.sum += ring.sum(); - } else { - c.rings.emplace_back(m, true); - c.stop_location = ring.get_node_ref_start().location(); - c.sum -= ring.sum(); - } - if (c.closed()) { - if (debug()) { - std::cerr << " found candidate\n"; - } - candidates.push_back(c); - } else if (loc_done.count(c.stop_location) == 0) { - if (debug()) { - std::cerr << " recurse...\n"; - } - loc_done.insert(c.stop_location); - find_candidates(candidates, loc_done, xrings, c); - if (debug()) { - std::cerr << " ...back\n"; - } - } else if (debug()) { - std::cerr << " loop found\n"; - } - } - } - } - - /** - * If there are multiple open rings and mltiple ways to join them, - * this function is called. It will take the first open ring and - * try recursively all ways of closing it. Of all the candidates - * the one with the smallest area is chosen and closed. If it - * can't close this ring, an error is reported and the function - * returns false. - */ - bool join_connected_rings(open_ring_its_type& open_ring_its) { - assert(!open_ring_its.empty()); - - if (debug()) { - std::cerr << " Trying to merge " << open_ring_its.size() << " open rings\n"; - } - - std::vector xrings = create_location_to_ring_map(open_ring_its); - - const auto ring_min = std::min_element(xrings.begin(), xrings.end(), [](const location_to_ring_map& lhs, const location_to_ring_map& rhs) { - return lhs.ring().min_segment() < rhs.ring().min_segment(); - }); - - find_inner_outer_complex(); - detail::ProtoRing* outer_ring = find_enclosing_ring(ring_min->ring().min_segment()); - bool ring_min_is_outer = !outer_ring; - if (debug()) { - std::cerr << " Open ring is " << (ring_min_is_outer ? "outer" : "inner") << " ring\n"; - } - for (auto& ring : m_rings) { - ring.reset(); - } - - candidate cand{*ring_min, false}; - - // Locations we have visited while finding candidates, used - // to detect loops. - std::unordered_set loc_done; - - loc_done.insert(cand.stop_location); - - std::vector candidates; - find_candidates(candidates, loc_done, xrings, cand); - - if (candidates.empty()) { - if (debug()) { - std::cerr << " Found no candidates\n"; - } - if (!open_ring_its.empty()) { - ++m_stats.open_rings; - if (m_config.problem_reporter) { - for (auto& it : open_ring_its) { - m_config.problem_reporter->report_ring_not_closed(it->get_node_ref_start()); - m_config.problem_reporter->report_ring_not_closed(it->get_node_ref_stop()); - } - } - } - return false; - } - - if (debug()) { - std::cerr << " Found candidates:\n"; - for (const auto& cand : candidates) { - std::cerr << " sum=" << cand.sum << "\n"; - for (const auto& ring : cand.rings) { - std::cerr << " " << ring.first.ring() << (ring.second ? " reverse" : "") << "\n"; - } - } - } - - // Find the candidate with the smallest/largest area - const auto chosen_cand = ring_min_is_outer ? - std::min_element(candidates.cbegin(), candidates.cend(), [](const candidate& lhs, const candidate& rhs) { - return std::abs(lhs.sum) < std::abs(rhs.sum); - }) : - std::max_element(candidates.cbegin(), candidates.cend(), [](const candidate& lhs, const candidate& rhs) { - return std::abs(lhs.sum) < std::abs(rhs.sum); - }); - - if (debug()) { - std::cerr << " Decided on: sum=" << chosen_cand->sum << "\n"; - for (const auto& ring : chosen_cand->rings) { - std::cerr << " " << ring.first.ring() << (ring.second ? " reverse" : "") << "\n"; - } - } - - // Join all (open) rings in the candidate to get one closed ring. - assert(chosen_cand->rings.size() > 1); - const auto& first_ring = chosen_cand->rings.front().first; - for (auto it = chosen_cand->rings.begin() + 1; it != chosen_cand->rings.end(); ++it) { - merge_two_rings(open_ring_its, first_ring, it->first); - } - - if (debug()) { - std::cerr << " Merged to " << first_ring.ring() << "\n"; - } - - return true; - } - - bool create_rings_complex_case() { - // First create all the (partial) rings starting at the split locations - auto count_remaining = m_segment_list.size(); - for (const osmium::Location& location : m_split_locations) { - const auto locs = make_range(std::equal_range(m_locations.begin(), - m_locations.end(), - slocation{}, - [this, &location](const slocation& lhs, const slocation& rhs) { - return lhs.location(m_segment_list, location) < rhs.location(m_segment_list, location); - })); - for (auto& loc : locs) { - if (!m_segment_list[loc.item].is_done()) { - count_remaining -= add_new_ring_complex(loc); - if (count_remaining == 0) { - break; - } - } - } - } - - // Now find all the rest of the rings (ie not starting at split locations) - if (count_remaining > 0) { - for (slocation& sl : m_locations) { - const detail::NodeRefSegment& segment = m_segment_list[sl.item]; - if (!segment.is_done()) { - count_remaining -= add_new_ring_complex(sl); - if (count_remaining == 0) { - break; - } - } - } - } - - // Now all segments are in exactly one (partial) ring. - - // If there are open rings, try to join them to create closed - // rings. - if (there_are_open_rings()) { - ++m_stats.area_really_complex_case; - - open_ring_its_type open_ring_its; - for (auto it = m_rings.begin(); it != m_rings.end(); ++it) { - if (!it->closed()) { - open_ring_its.push_back(it); - } - } - - while (!open_ring_its.empty()) { - if (debug()) { - std::cerr << " There are " << open_ring_its.size() << " open rings\n"; - } - while (try_to_merge(open_ring_its)); - - if (!open_ring_its.empty()) { - if (debug()) { - std::cerr << " After joining obvious cases there are still " << open_ring_its.size() << " open rings\n"; - } - if (!join_connected_rings(open_ring_its)) { - return false; - } - } - } - - if (debug()) { - std::cerr << " Joined all open rings\n"; - } - } - - // Now all rings are complete. - - find_inner_outer_complex(); - - return true; - } - - /** - * Checks if any ways were completely removed in the - * erase_duplicate_segments step. - */ - bool ways_were_lost() { - std::unordered_set ways_in_segments; - - for (const auto& segment : m_segment_list) { - ways_in_segments.insert(segment.way()); - } - - return ways_in_segments.size() < m_num_members; - } - - /** - * Create rings from segments. - */ - bool create_rings() { - m_stats.nodes += m_segment_list.size(); - - // Sort the list of segments (from left to right and bottom - // to top). - osmium::Timer timer_sort; - m_segment_list.sort(); - timer_sort.stop(); - - // Remove duplicate segments. Removal is in pairs, so if there - // are two identical segments, they will both be removed. If - // there are three, two will be removed and one remains. - osmium::Timer timer_dupl; - m_stats.duplicate_segments = m_segment_list.erase_duplicate_segments(m_config.problem_reporter); - timer_dupl.stop(); - - // If there are no segments left at this point, this isn't - // a valid area. - if (m_segment_list.empty()) { - if (debug()) { - std::cerr << " No segments left\n"; - } - return false; - } - - // If one or more complete ways was removed because of - // duplicate segments, this isn't a valid area. - if (ways_were_lost()) { - if (debug()) { - std::cerr << " Complete ways removed because of duplicate segments\n"; - } - return false; - } - - if (m_config.debug_level >= 3) { - std::cerr << "Sorted de-duplicated segment list:\n"; - for (const auto& s : m_segment_list) { - std::cerr << " " << s << "\n"; - } - } - - // Now we look for segments crossing each other. If there are - // any, the multipolygon is invalid. - // In the future this could be improved by trying to fix those - // cases. - osmium::Timer timer_intersection; - m_stats.intersections = m_segment_list.find_intersections(m_config.problem_reporter); - timer_intersection.stop(); - - if (m_stats.intersections) { - return false; - } - - // This creates an ordered list of locations of both endpoints - // of all segments with pointers back to the segments. We will - // use this list later to quickly find which segment(s) fits - // onto a known segment. - osmium::Timer timer_locations_list; - create_locations_list(); - timer_locations_list.stop(); - - // Find all locations where more than two segments start or - // end. We call those "split" locations. If there are any - // "spike" segments found while doing this, we know the area - // geometry isn't valid and return. - osmium::Timer timer_split; - if (!find_split_locations()) { - return false; - } - timer_split.stop(); - - // Now report all split locations to the problem reporter. - m_stats.touching_rings += m_split_locations.size(); - if (!m_split_locations.empty()) { - if (debug()) { - std::cerr << " Found split locations:\n"; - } - for (const auto& location : m_split_locations) { - if (m_config.problem_reporter) { - auto it = std::lower_bound(m_locations.cbegin(), m_locations.cend(), slocation{}, [this, &location](const slocation& lhs, const slocation& rhs) { - return lhs.location(m_segment_list, location) < rhs.location(m_segment_list, location); - }); - assert(it != m_locations.cend()); - const osmium::object_id_type id = it->node_ref(m_segment_list).ref(); - m_config.problem_reporter->report_touching_ring(id, location); - } - if (debug()) { - std::cerr << " " << location << "\n"; - } - } - } - - // From here on we use two different algorithms depending on - // whether there were any split locations or not. If there - // are no splits, we use the faster "simple algorithm", if - // there are, we use the slower "complex algorithm". - osmium::Timer timer_simple_case; - osmium::Timer timer_complex_case; - if (m_split_locations.empty()) { - if (debug()) { - std::cerr << " No split locations -> using simple algorithm\n"; - } - ++m_stats.area_simple_case; - - timer_simple_case.start(); - create_rings_simple_case(); - timer_simple_case.stop(); - } else { - if (debug()) { - std::cerr << " Found split locations -> using complex algorithm\n"; - } - ++m_stats.area_touching_rings_case; - - timer_complex_case.start(); - if (!create_rings_complex_case()) { - return false; - } - timer_complex_case.stop(); - } - - // If the assembler was so configured, now check whether the - // member roles are correctly tagged. - if (m_config.check_roles && m_stats.from_relations) { - osmium::Timer timer_roles; - check_inner_outer_roles(); - timer_roles.stop(); - } - - m_stats.outer_rings = std::count_if(m_rings.cbegin(), m_rings.cend(), [](const detail::ProtoRing& ring){ - return ring.is_outer(); - }); - m_stats.inner_rings = m_rings.size() - m_stats.outer_rings; - -#ifdef OSMIUM_WITH_TIMER - std::cout << m_stats.nodes << ' ' << m_stats.outer_rings << ' ' << m_stats.inner_rings << - ' ' << timer_sort.elapsed_microseconds() << - ' ' << timer_dupl.elapsed_microseconds() << - ' ' << timer_intersection.elapsed_microseconds() << - ' ' << timer_locations_list.elapsed_microseconds() << - ' ' << timer_split.elapsed_microseconds(); - - if (m_split_locations.empty()) { - std::cout << ' ' << timer_simple_case.elapsed_microseconds() << - " 0"; - } else { - std::cout << " 0" << - ' ' << timer_complex_case.elapsed_microseconds(); - } - - std::cout << -# ifdef OSMIUM_AREA_CHECK_INNER_OUTER_ROLES - ' ' << timer_roles.elapsed_microseconds() << -# else - " 0" << -# endif - '\n'; -#endif - - return true; - } - -#ifdef OSMIUM_WITH_TIMER - static bool print_header() { - std::cout << "nodes outer_rings inner_rings sort dupl intersection locations split simple_case complex_case roles_check\n"; - return true; - } - - static bool init_header() { - static bool printed_print_header = print_header(); - return printed_print_header; - } -#endif - bool create_area(osmium::memory::Buffer& out_buffer, const osmium::Way& way) { osmium::builder::AreaBuilder builder{out_buffer}; builder.initialize_from_object(way); const bool area_okay = create_rings(); - if (area_okay || m_config.create_empty_areas) { + if (area_okay || config().create_empty_areas) { add_tags_to_area(builder, way); } if (area_okay) { @@ -1374,19 +194,19 @@ namespace osmium { } if (report_ways()) { - m_config.problem_reporter->report_way(way); + config().problem_reporter->report_way(way); } - return area_okay || m_config.create_empty_areas; + return area_okay || config().create_empty_areas; } bool create_area(osmium::memory::Buffer& out_buffer, const osmium::Relation& relation, const std::vector& members) { - m_num_members = members.size(); + set_num_members(members.size()); osmium::builder::AreaBuilder builder{out_buffer}; builder.initialize_from_object(relation); const bool area_okay = create_rings(); - if (area_okay || m_config.create_empty_areas) { + if (area_okay || config().create_empty_areas) { add_tags_to_area(builder, relation); } if (area_okay) { @@ -1395,11 +215,11 @@ namespace osmium { if (report_ways()) { for (const osmium::Way* way : members) { - m_config.problem_reporter->report_way(*way); + config().problem_reporter->report_way(*way); } } - return area_okay || m_config.create_empty_areas; + return area_okay || config().create_empty_areas; } public: @@ -1407,11 +227,7 @@ namespace osmium { using config_type = osmium::area::AssemblerConfig; explicit Assembler(const config_type& config) : - m_config(config), - m_segment_list(config.debug_level > 1) { -#ifdef OSMIUM_WITH_TIMER - init_header(); -#endif + detail::BasicAssembler(config) { } ~Assembler() noexcept = default; @@ -1419,89 +235,63 @@ namespace osmium { /** * Assemble an area from the given way. * The resulting area is put into the out_buffer. + * + * @returns false if there was some kind of error building the + * area, true otherwise. */ - void operator()(const osmium::Way& way, osmium::memory::Buffer& out_buffer) { - if (!m_config.create_way_polygons) { - return; + bool operator()(const osmium::Way& way, osmium::memory::Buffer& out_buffer) { + if (!config().create_way_polygons) { + return true; } if (way.tags().has_tag("area", "no")) { - return; + return true; } - if (m_config.problem_reporter) { - m_config.problem_reporter->set_object(osmium::item_type::way, way.id()); - m_config.problem_reporter->set_nodes(way.nodes().size()); + if (config().problem_reporter) { + config().problem_reporter->set_object(osmium::item_type::way, way.id()); + config().problem_reporter->set_nodes(way.nodes().size()); } // Ignore (but count) ways without segments. if (way.nodes().size() < 2) { - ++m_stats.short_ways; - return; + ++stats().short_ways; + return false; } if (!way.ends_have_same_id()) { - ++m_stats.duplicate_nodes; - if (m_config.problem_reporter) { - m_config.problem_reporter->report_duplicate_node(way.nodes().front().ref(), way.nodes().back().ref(), way.nodes().front().location()); + ++stats().duplicate_nodes; + if (config().problem_reporter) { + config().problem_reporter->report_duplicate_node(way.nodes().front().ref(), way.nodes().back().ref(), way.nodes().front().location()); } } - ++m_stats.from_ways; - m_stats.duplicate_nodes += m_segment_list.extract_segments_from_way(m_config.problem_reporter, way); + ++stats().from_ways; + stats().invalid_locations = segment_list().extract_segments_from_way(config().problem_reporter, + stats().duplicate_nodes, + way); + if (!config().ignore_invalid_locations && stats().invalid_locations > 0) { + return false; + } - if (m_config.debug_level > 0) { - std::cerr << "\nAssembling way " << way.id() << " containing " << m_segment_list.size() << " nodes\n"; + if (config().debug_level > 0) { + std::cerr << "\nAssembling way " << way.id() << " containing " << segment_list().size() << " nodes\n"; } // Now create the Area object and add the attributes and tags // from the way. - if (create_area(out_buffer, way)) { + const bool okay = create_area(out_buffer, way); + if (okay) { out_buffer.commit(); } else { out_buffer.rollback(); } if (debug()) { - std::cerr << "Done: " << m_stats << "\n"; - } - } - - bool make_area(const osmium::Way &way, - osmium::memory::Buffer &out_buffer) - { - m_segment_list.extract_segments_from_way( - m_config.problem_reporter, way, true); - - if (!create_rings()) { - return false; - } - - osmium::builder::AreaBuilder builder{out_buffer}; - builder.initialize_from_object(way); - add_rings_to_area(builder); - out_buffer.commit(); - return true; - } - - bool make_area(const osmium::Relation &relation, - osmium::memory::Buffer const &ways, - osmium::memory::Buffer &out_buffer) - { - for (auto const &w : ways.select()) { - m_segment_list.extract_segments_from_way( - m_config.problem_reporter, w, true); - } - - if (!create_rings()) { - return false; + std::cerr << "Done: " << stats() << "\n"; } - osmium::builder::AreaBuilder builder{out_buffer}; - builder.initialize_from_object(relation); - add_rings_to_area(builder); - out_buffer.commit(); - return true; + return okay; } /** @@ -1526,38 +316,48 @@ namespace osmium { /** * Assemble an area from the given relation and its members. * The resulting area is put into the out_buffer. + * + * @returns false if there was some kind of error building the + * area(s), true otherwise. */ - void operator()(const osmium::Relation& relation, const std::vector& members, osmium::memory::Buffer& out_buffer) { + bool operator()(const osmium::Relation& relation, const std::vector& members, osmium::memory::Buffer& out_buffer) { assert(relation.members().size() >= members.size()); - if (m_config.problem_reporter) { - m_config.problem_reporter->set_object(osmium::item_type::relation, relation.id()); + if (config().problem_reporter) { + config().problem_reporter->set_object(osmium::item_type::relation, relation.id()); } if (relation.members().empty()) { - ++m_stats.no_way_in_mp_relation; - return; + ++stats().no_way_in_mp_relation; + return false; } - ++m_stats.from_relations; - m_stats.duplicate_nodes += m_segment_list.extract_segments_from_ways(m_config.problem_reporter, relation, members); - m_stats.member_ways = members.size(); + ++stats().from_relations; + stats().invalid_locations = segment_list().extract_segments_from_ways(config().problem_reporter, + stats().duplicate_nodes, + relation, + members); + if (!config().ignore_invalid_locations && stats().invalid_locations > 0) { + return false; + } + stats().member_ways = members.size(); - if (m_stats.member_ways == 1) { - ++m_stats.single_way_in_mp_relation; + if (stats().member_ways == 1) { + ++stats().single_way_in_mp_relation; } - if (m_config.debug_level > 0) { - std::cerr << "\nAssembling relation " << relation.id() << " containing " << members.size() << " way members with " << m_segment_list.size() << " nodes\n"; + if (config().debug_level > 0) { + std::cerr << "\nAssembling relation " << relation.id() << " containing " << members.size() << " way members with " << segment_list().size() << " nodes\n"; } const size_t area_offset = out_buffer.committed(); // Now create the Area object and add the attributes and tags // from the relation. - if (create_area(out_buffer, relation, members)) { - if ((m_config.create_new_style_polygons && m_stats.no_tags_on_relation == 0) || - (m_config.create_old_style_polygons && m_stats.no_tags_on_relation != 0)) { + bool okay = create_area(out_buffer, relation, members); + if (okay) { + if ((config().create_new_style_polygons && stats().no_tags_on_relation == 0) || + (config().create_old_style_polygons && stats().no_tags_on_relation != 0)) { out_buffer.commit(); } else { out_buffer.rollback(); @@ -1573,23 +373,23 @@ namespace osmium { // just built, add them to a list and later build areas for // them, too. std::vector ways_that_should_be_areas; - if (m_stats.wrong_role == 0) { + if (stats().wrong_role == 0) { detail::for_each_member(relation, members, [this, &ways_that_should_be_areas, &area_tags](const osmium::RelationMember& member, const osmium::Way& way) { if (!std::strcmp(member.role(), "inner")) { if (!way.nodes().empty() && way.is_closed() && way.tags().size() > 0) { - const auto d = std::count_if(way.tags().cbegin(), way.tags().cend(), filter()); + const auto d = std::count_if(way.tags().cbegin(), way.tags().cend(), std::cref(filter())); if (d > 0) { - osmium::tags::KeyFilter::iterator way_fi_begin(filter(), way.tags().cbegin(), way.tags().cend()); - osmium::tags::KeyFilter::iterator way_fi_end(filter(), way.tags().cend(), way.tags().cend()); - osmium::tags::KeyFilter::iterator area_fi_begin(filter(), area_tags.cbegin(), area_tags.cend()); - osmium::tags::KeyFilter::iterator area_fi_end(filter(), area_tags.cend(), area_tags.cend()); + osmium::tags::KeyFilter::iterator way_fi_begin(std::cref(filter()), way.tags().cbegin(), way.tags().cend()); + osmium::tags::KeyFilter::iterator way_fi_end(std::cref(filter()), way.tags().cend(), way.tags().cend()); + osmium::tags::KeyFilter::iterator area_fi_begin(std::cref(filter()), area_tags.cbegin(), area_tags.cend()); + osmium::tags::KeyFilter::iterator area_fi_end(std::cref(filter()), area_tags.cend(), area_tags.cend()); if (!std::equal(way_fi_begin, way_fi_end, area_fi_begin) || d != std::distance(area_fi_begin, area_fi_end)) { ways_that_should_be_areas.push_back(&way); } else { - ++m_stats.inner_with_same_tags; - if (m_config.problem_reporter) { - m_config.problem_reporter->report_inner_with_same_tags(way); + ++stats().inner_with_same_tags; + if (config().problem_reporter) { + config().problem_reporter->report_inner_with_same_tags(way); } } } @@ -1599,22 +399,18 @@ namespace osmium { } if (debug()) { - std::cerr << "Done: " << m_stats << "\n"; + std::cerr << "Done: " << stats() << "\n"; } // Now build areas for all ways found in the last step. for (const osmium::Way* way : ways_that_should_be_areas) { - Assembler assembler(m_config); - assembler(*way, out_buffer); + Assembler assembler{config()}; + if (!assembler(*way, out_buffer)) { + okay = false; + } } - } - /** - * Get statistics from assembler. Call this after running the - * assembler to get statistics and data about errors. - */ - const osmium::area::area_stats& stats() const noexcept { - return m_stats; + return okay; } }; // class Assembler diff --git a/contrib/libosmium/osmium/area/assembler_config.hpp b/contrib/libosmium/osmium/area/assembler_config.hpp new file mode 100644 index 000000000..2e49080a4 --- /dev/null +++ b/contrib/libosmium/osmium/area/assembler_config.hpp @@ -0,0 +1,152 @@ +#ifndef OSMIUM_AREA_ASSEMBLER_CONFIG_HPP +#define OSMIUM_AREA_ASSEMBLER_CONFIG_HPP + +/* + +This file is part of Osmium (http://osmcode.org/libosmium). + +Copyright 2013-2017 Jochen Topf and others (see README). + +Boost Software License - Version 1.0 - August 17th, 2003 + +Permission is hereby granted, free of charge, to any person or organization +obtaining a copy of the software and accompanying documentation covered by +this license (the "Software") to use, reproduce, display, distribute, +execute, and transmit the Software, and to prepare derivative works of the +Software, and to permit third-parties to whom the Software is furnished to +do so, all subject to the following: + +The copyright notices in the Software and this entire statement, including +the above license grant, this restriction and the following disclaimer, +must be included in all copies of the Software, in whole or in part, and +all derivative works of the Software, unless such copies or derivative +works are solely in the form of machine-executable object code generated by +a source language processor. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT +SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE +FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, +ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. + +*/ + +#include + +namespace osmium { + + namespace area { + + class ProblemReporter; + + /** + * Configuration for osmium::area::Assembler objects. Create this + * once, set the options you want and then re-use it every time you + * create an Assembler object. + */ + struct AssemblerConfig { + + /** + * Optional pointer to problem reporter. + */ + osmium::area::ProblemReporter* problem_reporter = nullptr; + + /** + * Debug level. If this is greater than zero, debug messages will + * be printed to stderr. Available levels are 1 to 3. Note that + * level 2 and above will generate a lot of messages! + */ + int debug_level = 0; + + /** + * The roles of multipolygon members are ignored when assembling + * multipolygons, because they are often missing or wrong. If this + * is set, the roles are checked after the multipolygons are built + * against what the assembly process decided where the inner and + * outer rings are. This slows down the processing, so it only + * makes sense if you want to get the problem reports. + */ + bool check_roles = false; + + /** + * When the assembler can't create an area, usually because its + * geometry would be invalid, it will create an "empty" area object + * without rings. This allows you to detect where an area was + * invalid. + * + * If this is set to false, invalid areas will simply be discarded. + */ + bool create_empty_areas = true; + + /** + * Create areas for (multi)polygons where the tags are on the + * relation. + * + * If this is set to false, those areas will simply be discarded. + */ + bool create_new_style_polygons = true; + + /** + * Create areas for (multi)polygons where the tags are on the + * outer way(s). + * + * If this is set to false, those areas will simply be discarded. + */ + bool create_old_style_polygons = true; + + /** + * Create areas for polygons created from ways. + * + * If this is set to false, those areas will simply be discarded. + */ + bool create_way_polygons = true; + + /** + * Keep the type tag from multipolygon relations on the area + * object. By default this is false, and the type tag will be + * removed. + */ + bool keep_type_tag = false; + + /** + * If there is an invalid location in any of the ways needed for + * assembling the multipolygon, the assembler will normally fail. + * If this is set, the assembler will silently ignore the invalid + * locations pretending them to be not referenced from the ways. + * This will allow some areas to be built, others will now be + * incorrect. This can sometimes be useful to assemble areas + * crossing the boundary of an extract, but you will also get + * geometrically valid but wrong (multi)polygons. + */ + bool ignore_invalid_locations = false; + + AssemblerConfig() noexcept = default; + + /** + * Constructor + * @deprecated Use default constructor and set values afterwards. + */ + explicit AssemblerConfig(osmium::area::ProblemReporter* pr, bool d = false) : + problem_reporter(pr), + debug_level(d) { + } + + /** + * Enable or disable debug output to stderr. This is for Osmium + * developers only. + * + * @deprecated Set debug_level directly. + */ + OSMIUM_DEPRECATED void enable_debug_output(bool d = true) { + debug_level = d; + } + + }; // struct AssemblerConfig + + } // namespace area + +} // namespace osmium + +#endif // OSMIUM_AREA_ASSEMBLER_CONFIG_HPP diff --git a/contrib/libosmium/osmium/area/detail/basic_assembler.hpp b/contrib/libosmium/osmium/area/detail/basic_assembler.hpp new file mode 100644 index 000000000..13bd976c2 --- /dev/null +++ b/contrib/libosmium/osmium/area/detail/basic_assembler.hpp @@ -0,0 +1,1204 @@ +#ifndef OSMIUM_AREA_DETAIL_BASIC_ASSEMBLER_HPP +#define OSMIUM_AREA_DETAIL_BASIC_ASSEMBLER_HPP + +/* + +This file is part of Osmium (http://osmcode.org/libosmium). + +Copyright 2013-2017 Jochen Topf and others (see README). + +Boost Software License - Version 1.0 - August 17th, 2003 + +Permission is hereby granted, free of charge, to any person or organization +obtaining a copy of the software and accompanying documentation covered by +this license (the "Software") to use, reproduce, display, distribute, +execute, and transmit the Software, and to prepare derivative works of the +Software, and to permit third-parties to whom the Software is furnished to +do so, all subject to the following: + +The copyright notices in the Software and this entire statement, including +the above license grant, this restriction and the following disclaimer, +must be included in all copies of the Software, in whole or in part, and +all derivative works of the Software, unless such copies or derivative +works are solely in the form of machine-executable object code generated by +a source language processor. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT +SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE +FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, +ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. + +*/ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +namespace osmium { + + namespace area { + + namespace detail { + + using open_ring_its_type = std::list::iterator>; + + struct location_to_ring_map { + osmium::Location location; + open_ring_its_type::iterator ring_it; + bool start; + + location_to_ring_map(const osmium::Location& l, const open_ring_its_type::iterator& r, bool s) noexcept : + location(l), + ring_it(r), + start(s) { + } + + explicit location_to_ring_map(const osmium::Location& l) noexcept : + location(l), + ring_it(), + start(false) { + } + + const detail::ProtoRing& ring() const noexcept { + return **ring_it; + } + + }; // struct location_to_ring_map + + inline bool operator==(const location_to_ring_map& lhs, const location_to_ring_map& rhs) noexcept { + return lhs.location == rhs.location; + } + + inline bool operator<(const location_to_ring_map& lhs, const location_to_ring_map& rhs) noexcept { + return lhs.location < rhs.location; + } + + /** + * Class for assembling ways and relations into multipolygons + * (areas). Contains the basic functionality needed but is not + * used directly. Use the osmium::area::Assembler class instead. + */ + class BasicAssembler { + + struct slocation { + + static constexpr const uint32_t invalid_item = 1 << 30; + + uint32_t item : 31; + uint32_t reverse : 1; + + slocation() noexcept : + item(invalid_item), + reverse(false) { + } + + explicit slocation(uint32_t n, bool r = false) noexcept : + item(n), + reverse(r) { + } + + osmium::Location location(const detail::SegmentList& segment_list) const noexcept { + const auto& segment = segment_list[item]; + return reverse ? segment.second().location() : segment.first().location(); + } + + const osmium::NodeRef& node_ref(const detail::SegmentList& segment_list) const noexcept { + const auto& segment = segment_list[item]; + return reverse ? segment.second() : segment.first(); + } + + osmium::Location location(const detail::SegmentList& segment_list, const osmium::Location& default_location) const noexcept { + if (item == invalid_item) { + return default_location; + } + return location(segment_list); + } + + }; // struct slocation + + // Configuration settings for this Assembler + const AssemblerConfig& m_config; + + // List of segments (connection between two nodes) + osmium::area::detail::SegmentList m_segment_list; + + // The rings we are building from the segments + std::list m_rings; + + // All node locations + std::vector m_locations; + + // All locations where more than two segments start/end + std::vector m_split_locations; + + // Statistics + area_stats m_stats; + + // The number of members the multipolygon relation has + size_t m_num_members = 0; + + template + static void build_ring_from_proto_ring(osmium::builder::AreaBuilder& builder, const detail::ProtoRing& ring) { + TBuilder ring_builder{builder}; + ring_builder.add_node_ref(ring.get_node_ref_start()); + for (const auto& segment : ring.segments()) { + ring_builder.add_node_ref(segment->stop()); + } + } + + void check_inner_outer_roles() { + if (debug()) { + std::cerr << " Checking inner/outer roles\n"; + } + + std::unordered_map way_rings; + std::unordered_set ways_in_multiple_rings; + + for (const detail::ProtoRing& ring : m_rings) { + for (const auto& segment : ring.segments()) { + assert(segment->way()); + + if (!segment->role_empty() && (ring.is_outer() ? !segment->role_outer() : !segment->role_inner())) { + ++m_stats.wrong_role; + if (debug()) { + std::cerr << " Segment " << *segment << " from way " << segment->way()->id() << " has role '" << segment->role_name() + << "', but should have role '" << (ring.is_outer() ? "outer" : "inner") << "'\n"; + } + if (m_config.problem_reporter) { + if (ring.is_outer()) { + m_config.problem_reporter->report_role_should_be_outer(segment->way()->id(), segment->first().location(), segment->second().location()); + } else { + m_config.problem_reporter->report_role_should_be_inner(segment->way()->id(), segment->first().location(), segment->second().location()); + } + } + } + + auto& r = way_rings[segment->way()]; + if (!r) { + r = ˚ + } else if (r != &ring) { + ways_in_multiple_rings.insert(segment->way()); + } + + } + } + + for (const osmium::Way* way : ways_in_multiple_rings) { + ++m_stats.ways_in_multiple_rings; + if (debug()) { + std::cerr << " Way " << way->id() << " is in multiple rings\n"; + } + if (m_config.problem_reporter) { + m_config.problem_reporter->report_way_in_multiple_rings(*way); + } + } + + } + + detail::NodeRefSegment* get_next_segment(const osmium::Location& location) { + auto it = std::lower_bound(m_locations.begin(), m_locations.end(), slocation{}, [this, &location](const slocation& lhs, const slocation& rhs) { + return lhs.location(m_segment_list, location) < rhs.location(m_segment_list, location); + }); + + assert(it != m_locations.end()); + if (m_segment_list[it->item].is_done()) { + ++it; + } + assert(it != m_locations.end()); + + assert(!m_segment_list[it->item].is_done()); + return &m_segment_list[it->item]; + } + + class rings_stack_element { + + int32_t m_y; + detail::ProtoRing* m_ring_ptr; + + public: + + rings_stack_element(int32_t y, detail::ProtoRing* ring_ptr) : + m_y(y), + m_ring_ptr(ring_ptr) { + } + + int32_t y() const noexcept { + return m_y; + } + + const detail::ProtoRing& ring() const noexcept { + return *m_ring_ptr; + } + + detail::ProtoRing* ring_ptr() noexcept { + return m_ring_ptr; + } + + bool operator==(const rings_stack_element& rhs) const noexcept { + return m_ring_ptr == rhs.m_ring_ptr; + } + + bool operator<(const rings_stack_element& rhs) const noexcept { + return m_y < rhs.m_y; + } + + }; // class ring_stack_element + + using rings_stack = std::vector; + + void remove_duplicates(rings_stack& outer_rings) { + while (true) { + const auto it = std::adjacent_find(outer_rings.begin(), outer_rings.end()); + if (it == outer_rings.end()) { + return; + } + outer_rings.erase(it, std::next(it, 2)); + } + } + + detail::ProtoRing* find_enclosing_ring(detail::NodeRefSegment* segment) { + if (debug()) { + std::cerr << " Looking for ring enclosing " << *segment << "\n"; + } + + const auto location = segment->first().location(); + const auto end_location = segment->second().location(); + + while (segment->first().location() == location) { + if (segment == &m_segment_list.back()) { + break; + } + ++segment; + } + + int nesting = 0; + + rings_stack outer_rings; + while (segment >= &m_segment_list.front()) { + if (!segment->is_direction_done()) { + --segment; + continue; + } + if (debug()) { + std::cerr << " Checking against " << *segment << "\n"; + } + const osmium::Location& a = segment->first().location(); + const osmium::Location& b = segment->second().location(); + + if (segment->first().location() == location) { + const int64_t ax = a.x(); + const int64_t bx = b.x(); + const int64_t lx = end_location.x(); + const int64_t ay = a.y(); + const int64_t by = b.y(); + const int64_t ly = end_location.y(); + const auto z = (bx - ax)*(ly - ay) - (by - ay)*(lx - ax); + if (debug()) { + std::cerr << " Segment XXXX z=" << z << "\n"; + } + if (z > 0) { + nesting += segment->is_reverse() ? -1 : 1; + if (debug()) { + std::cerr << " Segment is below (nesting=" << nesting << ")\n"; + } + if (segment->ring()->is_outer()) { + if (debug()) { + std::cerr << " Segment belongs to outer ring\n"; + } + outer_rings.emplace_back(a.y(), segment->ring()); + } + } + } else if (a.x() <= location.x() && location.x() < b.x()) { + if (debug()) { + std::cerr << " Is in x range\n"; + } + + const int64_t ax = a.x(); + const int64_t bx = b.x(); + const int64_t lx = location.x(); + const int64_t ay = a.y(); + const int64_t by = b.y(); + const int64_t ly = location.y(); + const auto z = (bx - ax)*(ly - ay) - (by - ay)*(lx - ax); + + if (z >= 0) { + nesting += segment->is_reverse() ? -1 : 1; + if (debug()) { + std::cerr << " Segment is below (nesting=" << nesting << ")\n"; + } + if (segment->ring()->is_outer()) { + if (debug()) { + std::cerr << " Segment belongs to outer ring\n"; + } + const int32_t y = int32_t(ay + (by - ay) * (lx - ax) / (bx - ax)); + outer_rings.emplace_back(y, segment->ring()); + } + } + } + --segment; + } + + if (nesting % 2 == 0) { + if (debug()) { + std::cerr << " Decided that this is an outer ring\n"; + } + return nullptr; + } else { + if (debug()) { + std::cerr << " Decided that this is an inner ring\n"; + } + assert(!outer_rings.empty()); + + std::sort(outer_rings.rbegin(), outer_rings.rend()); + if (debug()) { + for (const auto& o : outer_rings) { + std::cerr << " y=" << o.y() << " " << o.ring() << "\n"; + } + } + + remove_duplicates(outer_rings); + if (debug()) { + std::cerr << " after remove duplicates:\n"; + for (const auto& o : outer_rings) { + std::cerr << " y=" << o.y() << " " << o.ring() << "\n"; + } + } + + assert(!outer_rings.empty()); + return outer_rings.front().ring_ptr(); + } + } + + bool is_split_location(const osmium::Location& location) const noexcept { + return std::find(m_split_locations.cbegin(), m_split_locations.cend(), location) != m_split_locations.cend(); + } + + uint32_t add_new_ring(slocation& node) { + detail::NodeRefSegment* segment = &m_segment_list[node.item]; + assert(!segment->is_done()); + + if (debug()) { + std::cerr << " Starting new ring at location " << node.location(m_segment_list) << " with segment " << *segment << "\n"; + } + + if (node.reverse) { + segment->reverse(); + } + + detail::ProtoRing* outer_ring = nullptr; + + if (segment != &m_segment_list.front()) { + outer_ring = find_enclosing_ring(segment); + } + segment->mark_direction_done(); + + m_rings.emplace_back(segment); + detail::ProtoRing* ring = &m_rings.back(); + if (outer_ring) { + if (debug()) { + std::cerr << " This is an inner ring. Outer ring is " << *outer_ring << "\n"; + } + outer_ring->add_inner_ring(ring); + ring->set_outer_ring(outer_ring); + } else if (debug()) { + std::cerr << " This is an outer ring\n"; + } + + const osmium::Location& first_location = node.location(m_segment_list); + osmium::Location last_location = segment->stop().location(); + + uint32_t nodes = 1; + while (first_location != last_location) { + ++nodes; + detail::NodeRefSegment* next_segment = get_next_segment(last_location); + next_segment->mark_direction_done(); + if (next_segment->start().location() != last_location) { + next_segment->reverse(); + } + ring->add_segment_back(next_segment); + if (debug()) { + std::cerr << " Next segment is " << *next_segment << "\n"; + } + last_location = next_segment->stop().location(); + } + + ring->fix_direction(); + + if (debug()) { + std::cerr << " Completed ring: " << *ring << "\n"; + } + + return nodes; + } + + uint32_t add_new_ring_complex(slocation& node) { + detail::NodeRefSegment* segment = &m_segment_list[node.item]; + assert(!segment->is_done()); + + if (debug()) { + std::cerr << " Starting new ring at location " << node.location(m_segment_list) << " with segment " << *segment << "\n"; + } + + if (node.reverse) { + segment->reverse(); + } + + m_rings.emplace_back(segment); + detail::ProtoRing* ring = &m_rings.back(); + + const osmium::Location& first_location = node.location(m_segment_list); + osmium::Location last_location = segment->stop().location(); + + uint32_t nodes = 1; + while (first_location != last_location && !is_split_location(last_location)) { + ++nodes; + detail::NodeRefSegment* next_segment = get_next_segment(last_location); + if (next_segment->start().location() != last_location) { + next_segment->reverse(); + } + ring->add_segment_back(next_segment); + if (debug()) { + std::cerr << " Next segment is " << *next_segment << "\n"; + } + last_location = next_segment->stop().location(); + } + + if (debug()) { + if (first_location == last_location) { + std::cerr << " Completed ring: " << *ring << "\n"; + } else { + std::cerr << " Completed partial ring: " << *ring << "\n"; + } + } + + return nodes; + } + + void create_locations_list() { + m_locations.reserve(m_segment_list.size() * 2); + + for (uint32_t n = 0; n < m_segment_list.size(); ++n) { + m_locations.emplace_back(n, false); + m_locations.emplace_back(n, true); + } + + std::stable_sort(m_locations.begin(), m_locations.end(), [this](const slocation& lhs, const slocation& rhs) { + return lhs.location(m_segment_list) < rhs.location(m_segment_list); + }); + } + + void find_inner_outer_complex(detail::ProtoRing* ring) { + detail::ProtoRing* outer_ring = find_enclosing_ring(ring->min_segment()); + if (outer_ring) { + outer_ring->add_inner_ring(ring); + ring->set_outer_ring(outer_ring); + } + ring->fix_direction(); + ring->mark_direction_done(); + } + + void find_inner_outer_complex() { + if (debug()) { + std::cerr << " Finding inner/outer rings\n"; + } + std::vector rings; + rings.reserve(m_rings.size()); + for (auto& ring : m_rings) { + if (ring.closed()) { + rings.push_back(&ring); + } + } + + if (rings.empty()) { + return; + } + + std::sort(rings.begin(), rings.end(), [](detail::ProtoRing* a, detail::ProtoRing* b) { + return a->min_segment() < b->min_segment(); + }); + + rings.front()->fix_direction(); + rings.front()->mark_direction_done(); + if (debug()) { + std::cerr << " First ring is outer: " << *rings.front() << "\n"; + } + for (auto it = std::next(rings.begin()); it != rings.end(); ++it) { + if (debug()) { + std::cerr << " Checking (at min segment " << *((*it)->min_segment()) << ") ring " << **it << "\n"; + } + find_inner_outer_complex(*it); + if (debug()) { + std::cerr << " Ring is " << ((*it)->is_outer() ? "OUTER: " : "INNER: ") << **it << "\n"; + } + } + } + + /** + * Finds all locations where more than two segments meet. If there + * are any open rings found along the way, they are reported + * and the function returns false. + */ + bool find_split_locations() { + osmium::Location previous_location; + for (auto it = m_locations.cbegin(); it != m_locations.cend(); ++it) { + const osmium::NodeRef& nr = it->node_ref(m_segment_list); + const osmium::Location& loc = nr.location(); + if (std::next(it) == m_locations.cend() || loc != std::next(it)->location(m_segment_list)) { + if (debug()) { + std::cerr << " Found open ring at " << nr << "\n"; + } + if (m_config.problem_reporter) { + const auto& segment = m_segment_list[it->item]; + m_config.problem_reporter->report_ring_not_closed(nr, segment.way()); + } + ++m_stats.open_rings; + } else { + if (loc == previous_location && (m_split_locations.empty() || m_split_locations.back() != previous_location )) { + m_split_locations.push_back(previous_location); + } + ++it; + if (it == m_locations.end()) { + break; + } + } + previous_location = loc; + } + return m_stats.open_rings == 0; + } + + void create_rings_simple_case() { + auto count_remaining = m_segment_list.size(); + for (slocation& sl : m_locations) { + const detail::NodeRefSegment& segment = m_segment_list[sl.item]; + if (!segment.is_done()) { + count_remaining -= add_new_ring(sl); + if (count_remaining == 0) { + return; + } + } + } + } + + std::vector create_location_to_ring_map(open_ring_its_type& open_ring_its) { + std::vector xrings; + xrings.reserve(open_ring_its.size() * 2); + + for (auto it = open_ring_its.begin(); it != open_ring_its.end(); ++it) { + if (debug()) { + std::cerr << " Ring: " << **it << "\n"; + } + xrings.emplace_back((*it)->get_node_ref_start().location(), it, true); + xrings.emplace_back((*it)->get_node_ref_stop().location(), it, false); + } + + std::sort(xrings.begin(), xrings.end()); + + return xrings; + } + + void merge_two_rings(open_ring_its_type& open_ring_its, const location_to_ring_map& m1, const location_to_ring_map& m2) { + auto& r1 = *m1.ring_it; + auto& r2 = *m2.ring_it; + + if (r1->get_node_ref_stop().location() == r2->get_node_ref_start().location()) { + r1->join_forward(*r2); + } else if (r1->get_node_ref_stop().location() == r2->get_node_ref_stop().location()) { + r1->join_backward(*r2); + } else if (r1->get_node_ref_start().location() == r2->get_node_ref_start().location()) { + r1->reverse(); + r1->join_forward(*r2); + } else if (r1->get_node_ref_start().location() == r2->get_node_ref_stop().location()) { + r1->reverse(); + r1->join_backward(*r2); + } else { + assert(false); + } + + m_rings.erase(r2); + open_ring_its.remove(r2); + + if (r1->closed()) { + open_ring_its.remove(r1); + } + } + + bool try_to_merge(open_ring_its_type& open_ring_its) { + if (open_ring_its.empty()) { + return false; + } + + if (debug()) { + std::cerr << " Trying to merge " << open_ring_its.size() << " open rings\n"; + } + + std::vector xrings = create_location_to_ring_map(open_ring_its); + + auto it = xrings.cbegin(); + while (it != xrings.cend()) { + it = std::adjacent_find(it, xrings.cend()); + if (it == xrings.cend()) { + return false; + } + auto after = std::next(it, 2); + if (after == xrings.cend() || after->location != it->location) { + if (debug()) { + std::cerr << " Merging two rings\n"; + } + merge_two_rings(open_ring_its, *it, *std::next(it)); + return true; + } + while (it != xrings.cend() && it->location == after->location) { + ++it; + } + } + + return false; + } + + bool there_are_open_rings() const noexcept { + return std::any_of(m_rings.cbegin(), m_rings.cend(), [](const detail::ProtoRing& ring){ + return !ring.closed(); + }); + } + + struct candidate { + int64_t sum; + std::vector> rings; + osmium::Location start_location; + osmium::Location stop_location; + + explicit candidate(location_to_ring_map& ring, bool reverse) : + sum(ring.ring().sum()), + rings(), + start_location(ring.ring().get_node_ref_start().location()), + stop_location(ring.ring().get_node_ref_stop().location()) { + rings.emplace_back(ring, reverse); + } + + bool closed() const noexcept { + return start_location == stop_location; + } + + }; + + void find_candidates(std::vector& candidates, std::unordered_set& loc_done, const std::vector& xrings, candidate& cand) { + if (debug()) { + std::cerr << " find_candidates sum=" << cand.sum << " start=" << cand.start_location << " stop=" << cand.stop_location << "\n"; + for (const auto& ring : cand.rings) { + std::cerr << " " << ring.first.ring() << (ring.second ? " reverse" : "") << "\n"; + } + } + + const auto connections = make_range(std::equal_range(xrings.cbegin(), + xrings.cend(), + location_to_ring_map{cand.stop_location})); + + assert(connections.begin() != connections.end()); + + assert(!cand.rings.empty()); + const detail::ProtoRing* ring_leading_here = &cand.rings.back().first.ring(); + for (const location_to_ring_map& m : connections) { + const detail::ProtoRing& ring = m.ring(); + + if (&ring != ring_leading_here) { + if (debug()) { + std::cerr << " next possible connection: " << ring << (m.start ? "" : " reverse") << "\n"; + } + + candidate c = cand; + if (m.start) { + c.rings.emplace_back(m, false); + c.stop_location = ring.get_node_ref_stop().location(); + c.sum += ring.sum(); + } else { + c.rings.emplace_back(m, true); + c.stop_location = ring.get_node_ref_start().location(); + c.sum -= ring.sum(); + } + if (c.closed()) { + if (debug()) { + std::cerr << " found candidate\n"; + } + candidates.push_back(c); + } else if (loc_done.count(c.stop_location) == 0) { + if (debug()) { + std::cerr << " recurse...\n"; + } + loc_done.insert(c.stop_location); + find_candidates(candidates, loc_done, xrings, c); + if (debug()) { + std::cerr << " ...back\n"; + } + } else if (debug()) { + std::cerr << " loop found\n"; + } + } + } + } + + /** + * If there are multiple open rings and multiple ways to join them, + * this function is called. It will take the first open ring and + * try recursively all ways of closing it. Of all the candidates + * the one with the smallest area is chosen and closed. If it + * can't close this ring, an error is reported and the function + * returns false. + */ + bool join_connected_rings(open_ring_its_type& open_ring_its) { + assert(!open_ring_its.empty()); + + if (debug()) { + std::cerr << " Trying to merge " << open_ring_its.size() << " open rings\n"; + } + + std::vector xrings = create_location_to_ring_map(open_ring_its); + + const auto ring_min = std::min_element(xrings.begin(), xrings.end(), [](const location_to_ring_map& lhs, const location_to_ring_map& rhs) { + return lhs.ring().min_segment() < rhs.ring().min_segment(); + }); + + find_inner_outer_complex(); + detail::ProtoRing* outer_ring = find_enclosing_ring(ring_min->ring().min_segment()); + bool ring_min_is_outer = !outer_ring; + if (debug()) { + std::cerr << " Open ring is " << (ring_min_is_outer ? "outer" : "inner") << " ring\n"; + } + for (auto& ring : m_rings) { + ring.reset(); + } + + candidate cand{*ring_min, false}; + + // Locations we have visited while finding candidates, used + // to detect loops. + std::unordered_set loc_done; + + loc_done.insert(cand.stop_location); + + std::vector candidates; + find_candidates(candidates, loc_done, xrings, cand); + + if (candidates.empty()) { + if (debug()) { + std::cerr << " Found no candidates\n"; + } + if (!open_ring_its.empty()) { + ++m_stats.open_rings; + if (m_config.problem_reporter) { + for (auto& it : open_ring_its) { + m_config.problem_reporter->report_ring_not_closed(it->get_node_ref_start()); + m_config.problem_reporter->report_ring_not_closed(it->get_node_ref_stop()); + } + } + } + return false; + } + + if (debug()) { + std::cerr << " Found candidates:\n"; + for (const auto& cand : candidates) { + std::cerr << " sum=" << cand.sum << "\n"; + for (const auto& ring : cand.rings) { + std::cerr << " " << ring.first.ring() << (ring.second ? " reverse" : "") << "\n"; + } + } + } + + // Find the candidate with the smallest/largest area + const auto chosen_cand = ring_min_is_outer ? + std::min_element(candidates.cbegin(), candidates.cend(), [](const candidate& lhs, const candidate& rhs) { + return std::abs(lhs.sum) < std::abs(rhs.sum); + }) : + std::max_element(candidates.cbegin(), candidates.cend(), [](const candidate& lhs, const candidate& rhs) { + return std::abs(lhs.sum) < std::abs(rhs.sum); + }); + + if (debug()) { + std::cerr << " Decided on: sum=" << chosen_cand->sum << "\n"; + for (const auto& ring : chosen_cand->rings) { + std::cerr << " " << ring.first.ring() << (ring.second ? " reverse" : "") << "\n"; + } + } + + // Join all (open) rings in the candidate to get one closed ring. + assert(chosen_cand->rings.size() > 1); + const auto& first_ring = chosen_cand->rings.front().first; + for (auto it = chosen_cand->rings.begin() + 1; it != chosen_cand->rings.end(); ++it) { + merge_two_rings(open_ring_its, first_ring, it->first); + } + + if (debug()) { + std::cerr << " Merged to " << first_ring.ring() << "\n"; + } + + return true; + } + + bool create_rings_complex_case() { + // First create all the (partial) rings starting at the split locations + auto count_remaining = m_segment_list.size(); + for (const osmium::Location& location : m_split_locations) { + const auto locs = make_range(std::equal_range(m_locations.begin(), + m_locations.end(), + slocation{}, + [this, &location](const slocation& lhs, const slocation& rhs) { + return lhs.location(m_segment_list, location) < rhs.location(m_segment_list, location); + })); + for (auto& loc : locs) { + if (!m_segment_list[loc.item].is_done()) { + count_remaining -= add_new_ring_complex(loc); + if (count_remaining == 0) { + break; + } + } + } + } + + // Now find all the rest of the rings (ie not starting at split locations) + if (count_remaining > 0) { + for (slocation& sl : m_locations) { + const detail::NodeRefSegment& segment = m_segment_list[sl.item]; + if (!segment.is_done()) { + count_remaining -= add_new_ring_complex(sl); + if (count_remaining == 0) { + break; + } + } + } + } + + // Now all segments are in exactly one (partial) ring. + + // If there are open rings, try to join them to create closed + // rings. + if (there_are_open_rings()) { + ++m_stats.area_really_complex_case; + + open_ring_its_type open_ring_its; + for (auto it = m_rings.begin(); it != m_rings.end(); ++it) { + if (!it->closed()) { + open_ring_its.push_back(it); + } + } + + while (!open_ring_its.empty()) { + if (debug()) { + std::cerr << " There are " << open_ring_its.size() << " open rings\n"; + } + while (try_to_merge(open_ring_its)); + + if (!open_ring_its.empty()) { + if (debug()) { + std::cerr << " After joining obvious cases there are still " << open_ring_its.size() << " open rings\n"; + } + if (!join_connected_rings(open_ring_its)) { + return false; + } + } + } + + if (debug()) { + std::cerr << " Joined all open rings\n"; + } + } + + // Now all rings are complete. + + find_inner_outer_complex(); + + return true; + } + + /** + * Checks if any ways were completely removed in the + * erase_duplicate_segments step. + */ + bool ways_were_lost() { + std::unordered_set ways_in_segments; + + for (const auto& segment : m_segment_list) { + ways_in_segments.insert(segment.way()); + } + + return ways_in_segments.size() < m_num_members; + } + +#ifdef OSMIUM_WITH_TIMER + static bool print_header() { + std::cout << "nodes outer_rings inner_rings sort dupl intersection locations split simple_case complex_case roles_check\n"; + return true; + } + + static bool init_header() { + static bool printed_print_header = print_header(); + return printed_print_header; + } +#endif + + protected: + + const std::list& rings() const noexcept { + return m_rings; + } + + void set_num_members(size_t size) noexcept { + m_num_members = size; + } + + osmium::area::detail::SegmentList& segment_list() noexcept { + return m_segment_list; + } + + /** + * Append each outer ring together with its inner rings to the + * area in the buffer. + */ + void add_rings_to_area(osmium::builder::AreaBuilder& builder) const { + for (const detail::ProtoRing& ring : m_rings) { + if (ring.is_outer()) { + build_ring_from_proto_ring(builder, ring); + for (const detail::ProtoRing* inner : ring.inner_rings()) { + build_ring_from_proto_ring(builder, *inner); + } + } + } + } + + /** + * Create rings from segments. + */ + bool create_rings() { + m_stats.nodes += m_segment_list.size(); + + // Sort the list of segments (from left to right and bottom + // to top). + osmium::Timer timer_sort; + m_segment_list.sort(); + timer_sort.stop(); + + // Remove duplicate segments. Removal is in pairs, so if there + // are two identical segments, they will both be removed. If + // there are three, two will be removed and one remains. + osmium::Timer timer_dupl; + m_stats.duplicate_segments = m_segment_list.erase_duplicate_segments(m_config.problem_reporter); + timer_dupl.stop(); + + // If there are no segments left at this point, this isn't + // a valid area. + if (m_segment_list.empty()) { + if (debug()) { + std::cerr << " No segments left\n"; + } + return false; + } + + // If one or more complete ways was removed because of + // duplicate segments, this isn't a valid area. + if (ways_were_lost()) { + if (debug()) { + std::cerr << " Complete ways removed because of duplicate segments\n"; + } + return false; + } + + if (m_config.debug_level >= 3) { + std::cerr << "Sorted de-duplicated segment list:\n"; + for (const auto& s : m_segment_list) { + std::cerr << " " << s << "\n"; + } + } + + // Now we look for segments crossing each other. If there are + // any, the multipolygon is invalid. + // In the future this could be improved by trying to fix those + // cases. + osmium::Timer timer_intersection; + m_stats.intersections = m_segment_list.find_intersections(m_config.problem_reporter); + timer_intersection.stop(); + + if (m_stats.intersections) { + return false; + } + + // This creates an ordered list of locations of both endpoints + // of all segments with pointers back to the segments. We will + // use this list later to quickly find which segment(s) fits + // onto a known segment. + osmium::Timer timer_locations_list; + create_locations_list(); + timer_locations_list.stop(); + + // Find all locations where more than two segments start or + // end. We call those "split" locations. If there are any + // "spike" segments found while doing this, we know the area + // geometry isn't valid and return. + osmium::Timer timer_split; + if (!find_split_locations()) { + return false; + } + timer_split.stop(); + + // Now report all split locations to the problem reporter. + m_stats.touching_rings += m_split_locations.size(); + if (!m_split_locations.empty()) { + if (debug()) { + std::cerr << " Found split locations:\n"; + } + for (const auto& location : m_split_locations) { + if (m_config.problem_reporter) { + auto it = std::lower_bound(m_locations.cbegin(), m_locations.cend(), slocation{}, [this, &location](const slocation& lhs, const slocation& rhs) { + return lhs.location(m_segment_list, location) < rhs.location(m_segment_list, location); + }); + assert(it != m_locations.cend()); + const osmium::object_id_type id = it->node_ref(m_segment_list).ref(); + m_config.problem_reporter->report_touching_ring(id, location); + } + if (debug()) { + std::cerr << " " << location << "\n"; + } + } + } + + // From here on we use two different algorithms depending on + // whether there were any split locations or not. If there + // are no splits, we use the faster "simple algorithm", if + // there are, we use the slower "complex algorithm". + osmium::Timer timer_simple_case; + osmium::Timer timer_complex_case; + if (m_split_locations.empty()) { + if (debug()) { + std::cerr << " No split locations -> using simple algorithm\n"; + } + ++m_stats.area_simple_case; + + timer_simple_case.start(); + create_rings_simple_case(); + timer_simple_case.stop(); + } else { + if (debug()) { + std::cerr << " Found split locations -> using complex algorithm\n"; + } + ++m_stats.area_touching_rings_case; + + timer_complex_case.start(); + if (!create_rings_complex_case()) { + return false; + } + timer_complex_case.stop(); + } + + // If the assembler was so configured, now check whether the + // member roles are correctly tagged. + if (m_config.check_roles && m_stats.from_relations) { + osmium::Timer timer_roles; + check_inner_outer_roles(); + timer_roles.stop(); + } + + m_stats.outer_rings = std::count_if(m_rings.cbegin(), m_rings.cend(), [](const detail::ProtoRing& ring){ + return ring.is_outer(); + }); + m_stats.inner_rings = m_rings.size() - m_stats.outer_rings; + +#ifdef OSMIUM_WITH_TIMER + std::cout << m_stats.nodes << ' ' << m_stats.outer_rings << ' ' << m_stats.inner_rings << + ' ' << timer_sort.elapsed_microseconds() << + ' ' << timer_dupl.elapsed_microseconds() << + ' ' << timer_intersection.elapsed_microseconds() << + ' ' << timer_locations_list.elapsed_microseconds() << + ' ' << timer_split.elapsed_microseconds(); + + if (m_split_locations.empty()) { + std::cout << ' ' << timer_simple_case.elapsed_microseconds() << + " 0"; + } else { + std::cout << " 0" << + ' ' << timer_complex_case.elapsed_microseconds(); + } + + std::cout << +# ifdef OSMIUM_AREA_CHECK_INNER_OUTER_ROLES + ' ' << timer_roles.elapsed_microseconds() << +# else + " 0" << +# endif + '\n'; +#endif + + return true; + } + + public: + + using config_type = osmium::area::AssemblerConfig; + + explicit BasicAssembler(const config_type& config) : + m_config(config), + m_segment_list(config.debug_level > 1) { +#ifdef OSMIUM_WITH_TIMER + init_header(); +#endif + } + + ~BasicAssembler() noexcept = default; + + const AssemblerConfig& config() const noexcept { + return m_config; + } + + bool debug() const noexcept { + return m_config.debug_level > 1; + } + + /** + * Get statistics from assembler. Call this after running the + * assembler to get statistics and data about errors. + */ + const osmium::area::area_stats& stats() const noexcept { + return m_stats; + } + + osmium::area::area_stats& stats() noexcept { + return m_stats; + } + + }; // class BasicAssembler + + } // namespace detail + + } // namespace area + +} // namespace osmium + +#endif // OSMIUM_AREA_DETAIL_BASIC_ASSEMBLER_HPP diff --git a/contrib/libosmium/osmium/area/detail/segment_list.hpp b/contrib/libosmium/osmium/area/detail/segment_list.hpp index 45a2a0ac0..2dc58a355 100644 --- a/contrib/libosmium/osmium/area/detail/segment_list.hpp +++ b/contrib/libosmium/osmium/area/detail/segment_list.hpp @@ -110,16 +110,16 @@ namespace osmium { }); } - uint32_t extract_segments_from_way_impl( - osmium::area::ProblemReporter *problem_reporter, - const osmium::Way &way, role_type role, - bool ignore_invalid = false) - { - uint32_t duplicate_nodes = 0; + uint32_t extract_segments_from_way_impl(osmium::area::ProblemReporter* problem_reporter, uint64_t& duplicate_nodes, const osmium::Way& way, role_type role) { + uint32_t invalid_locations = 0; osmium::NodeRef previous_nr; for (const osmium::NodeRef& nr : way.nodes()) { - if (ignore_invalid && !nr.location().valid()) { + if (!nr.location().valid()) { + ++invalid_locations; + if (problem_reporter) { + problem_reporter->report_invalid_location(way.id(), nr.ref()); + } continue; } if (previous_nr.location()) { @@ -135,7 +135,7 @@ namespace osmium { previous_nr = nr; } - return duplicate_nodes; + return invalid_locations; } public: @@ -220,29 +220,19 @@ namespace osmium { * same node or different nodes with same location) are * removed after reporting the duplicate node. */ - uint32_t extract_segments_from_way( - osmium::area::ProblemReporter *problem_reporter, - const osmium::Way &way, bool ignore_invalid = false) - { + uint32_t extract_segments_from_way(osmium::area::ProblemReporter* problem_reporter, uint64_t& duplicate_nodes, const osmium::Way& way) { if (way.nodes().empty()) { return 0; } m_segments.reserve(way.nodes().size() - 1); - return extract_segments_from_way_impl(problem_reporter, way, - role_type::outer, - ignore_invalid); + return extract_segments_from_way_impl(problem_reporter, duplicate_nodes, way, role_type::outer); } /** * Extract all segments from all ways that make up this * multipolygon relation and add them to the list. */ - uint32_t extract_segments_from_ways( - osmium::area::ProblemReporter *problem_reporter, - const osmium::Relation &relation, - const std::vector &members, - bool ignore_invalid = false) - { + uint32_t extract_segments_from_ways(osmium::area::ProblemReporter* problem_reporter, uint64_t& duplicate_nodes, const osmium::Relation& relation, const std::vector& members) { assert(relation.members().size() >= members.size()); const size_t num_segments = get_num_segments(members); @@ -251,18 +241,13 @@ namespace osmium { } m_segments.reserve(num_segments); - uint32_t duplicate_nodes = 0; - for_each_member( - relation, members, - [this, &problem_reporter, &duplicate_nodes, - ignore_invalid](const osmium::RelationMember &member, - const osmium::Way &way) { - duplicate_nodes += extract_segments_from_way_impl( - problem_reporter, way, - parse_role(member.role()), ignore_invalid); - }); - - return duplicate_nodes; + uint32_t invalid_locations = 0; + for_each_member(relation, members, [&](const osmium::RelationMember& member, const osmium::Way& way) { + const auto role = parse_role(member.role()); + invalid_locations += extract_segments_from_way_impl(problem_reporter, duplicate_nodes, way, role); + }); + + return invalid_locations; } /** diff --git a/contrib/libosmium/osmium/area/detail/vector.hpp b/contrib/libosmium/osmium/area/detail/vector.hpp index 14bf0cb7b..e8ff0a501 100644 --- a/contrib/libosmium/osmium/area/detail/vector.hpp +++ b/contrib/libosmium/osmium/area/detail/vector.hpp @@ -118,4 +118,4 @@ namespace osmium { } // namespace osmium -#endif // OSMIUM_AREA_DETAIL_VECTOR_HPP +#endif // OSMIUM_AREA_DETAIL_VECTOR_HPP diff --git a/contrib/libosmium/osmium/area/geom_assembler.hpp b/contrib/libosmium/osmium/area/geom_assembler.hpp new file mode 100644 index 000000000..84870ae9f --- /dev/null +++ b/contrib/libosmium/osmium/area/geom_assembler.hpp @@ -0,0 +1,131 @@ +#ifndef OSMIUM_AREA_GEOM_ASSEMBLER_HPP +#define OSMIUM_AREA_GEOM_ASSEMBLER_HPP + +/* + +This file is part of Osmium (http://osmcode.org/libosmium). + +Copyright 2013-2017 Jochen Topf and others (see README). + +Boost Software License - Version 1.0 - August 17th, 2003 + +Permission is hereby granted, free of charge, to any person or organization +obtaining a copy of the software and accompanying documentation covered by +this license (the "Software") to use, reproduce, display, distribute, +execute, and transmit the Software, and to prepare derivative works of the +Software, and to permit third-parties to whom the Software is furnished to +do so, all subject to the following: + +The copyright notices in the Software and this entire statement, including +the above license grant, this restriction and the following disclaimer, +must be included in all copies of the Software, in whole or in part, and +all derivative works of the Software, unless such copies or derivative +works are solely in the form of machine-executable object code generated by +a source language processor. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT +SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE +FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, +ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. + +*/ + +#include +#include +#include +#include +#include +#include +#include + +namespace osmium { + + namespace area { + + /** + * Assembles area objects from closed ways or multipolygon relations + * and their members. Unlike the Assembler, this one doesn't take + * tags into account at all. And it doesn't interpret all the config + * settings and doesn't do all the checks and error reporting the + * Assembler does. + * + * This class was developed specifically for the need of osm2pgsql. + * Unless you know what you are doing, use the Assembler class instead + * of this class. Contact the Libosmium developers if you want to use + * this class. + */ + class GeomAssembler : public detail::BasicAssembler { + + public: + + using config_type = osmium::area::AssemblerConfig; + + explicit GeomAssembler(const config_type& config) : + detail::BasicAssembler(config) { + } + + ~GeomAssembler() noexcept = default; + + /** + * Assemble an area from the given way. + * + * The resulting area is put into the out_buffer. + * + * @returns false if there was some kind of error building the + * area, true otherwise. + */ + bool operator()(const osmium::Way& way, osmium::memory::Buffer& out_buffer) { + segment_list().extract_segments_from_way(config().problem_reporter, stats().duplicate_nodes, way); + + if (!create_rings()) { + return false; + } + + { + osmium::builder::AreaBuilder builder{out_buffer}; + builder.initialize_from_object(way); + add_rings_to_area(builder); + } + out_buffer.commit(); + + return true; + } + + /** + * Assemble an area from the given relation and its member ways + * which are in the ways_buffer. + * + * The resulting area is put into the out_buffer. + * + * @returns false if there was some kind of error building the + * area, true otherwise. + */ + bool operator()(const osmium::Relation& relation, const osmium::memory::Buffer& ways_buffer, osmium::memory::Buffer& out_buffer) { + for (const auto& way : ways_buffer.select()) { + segment_list().extract_segments_from_way(config().problem_reporter, stats().duplicate_nodes, way); + } + + if (!create_rings()) { + return false; + } + + { + osmium::builder::AreaBuilder builder{out_buffer}; + builder.initialize_from_object(relation); + add_rings_to_area(builder); + } + out_buffer.commit(); + + return true; + } + + }; // class GeomAssembler + + } // namespace area + +} // namespace osmium + +#endif // OSMIUM_AREA_GEOM_ASSEMBLER_HPP diff --git a/contrib/libosmium/osmium/area/problem_reporter.hpp b/contrib/libosmium/osmium/area/problem_reporter.hpp index 8a72c1ad9..373403b93 100644 --- a/contrib/libosmium/osmium/area/problem_reporter.hpp +++ b/contrib/libosmium/osmium/area/problem_reporter.hpp @@ -189,6 +189,15 @@ namespace osmium { virtual void report_inner_with_same_tags(const osmium::Way& way) { } + /** + * Report an invalid location in a way. + * + * @param way_id ID of the way the node is in. + * @param node_id ID of the node with the invalid location. + */ + virtual void report_invalid_location(osmium::object_id_type way_id, osmium::object_id_type node_id) { + } + /** * In addition to reporting specific problems, this is used to * report all ways belonging to areas having problems. diff --git a/contrib/libosmium/osmium/area/problem_reporter_exception.hpp b/contrib/libosmium/osmium/area/problem_reporter_exception.hpp index 152be40a9..cc5eb98bd 100644 --- a/contrib/libosmium/osmium/area/problem_reporter_exception.hpp +++ b/contrib/libosmium/osmium/area/problem_reporter_exception.hpp @@ -60,58 +60,64 @@ namespace osmium { ~ProblemReporterException() override = default; void report_duplicate_node(osmium::object_id_type node_id1, osmium::object_id_type node_id2, osmium::Location location) override { - m_sstream.str(); + m_sstream.str(""); ProblemReporterStream::report_duplicate_node(node_id1, node_id2, location); - throw std::runtime_error(m_sstream.str()); + throw std::runtime_error{m_sstream.str()}; } void report_touching_ring(osmium::object_id_type node_id, osmium::Location location) override { - m_sstream.str(); + m_sstream.str(""); ProblemReporterStream::report_touching_ring(node_id, location); - throw std::runtime_error(m_sstream.str()); + throw std::runtime_error{m_sstream.str()}; } void report_intersection(osmium::object_id_type way1_id, osmium::Location way1_seg_start, osmium::Location way1_seg_end, osmium::object_id_type way2_id, osmium::Location way2_seg_start, osmium::Location way2_seg_end, osmium::Location intersection) override { - m_sstream.str(); + m_sstream.str(""); ProblemReporterStream::report_intersection(way1_id, way1_seg_start, way1_seg_end, way2_id, way2_seg_start, way2_seg_end, intersection); - throw std::runtime_error(m_sstream.str()); + throw std::runtime_error{m_sstream.str()}; } void report_duplicate_segment(const osmium::NodeRef& nr1, const osmium::NodeRef& nr2) override { - m_sstream.str(); + m_sstream.str(""); ProblemReporterStream::report_duplicate_segment(nr1, nr2); - throw std::runtime_error(m_sstream.str()); + throw std::runtime_error{m_sstream.str()}; } void report_ring_not_closed(const osmium::NodeRef& nr, const osmium::Way* way = nullptr) override { - m_sstream.str(); + m_sstream.str(""); ProblemReporterStream::report_ring_not_closed(nr, way); - throw std::runtime_error(m_sstream.str()); + throw std::runtime_error{m_sstream.str()}; } void report_role_should_be_outer(osmium::object_id_type way_id, osmium::Location seg_start, osmium::Location seg_end) override { - m_sstream.str(); + m_sstream.str(""); ProblemReporterStream::report_role_should_be_outer(way_id, seg_start, seg_end); - throw std::runtime_error(m_sstream.str()); + throw std::runtime_error{m_sstream.str()}; } void report_role_should_be_inner(osmium::object_id_type way_id, osmium::Location seg_start, osmium::Location seg_end) override { - m_sstream.str(); + m_sstream.str(""); ProblemReporterStream::report_role_should_be_inner(way_id, seg_start, seg_end); - throw std::runtime_error(m_sstream.str()); + throw std::runtime_error{m_sstream.str()}; } void report_way_in_multiple_rings(const osmium::Way& way) override { - m_sstream.str(); + m_sstream.str(""); ProblemReporterStream::report_way_in_multiple_rings(way); - throw std::runtime_error(m_sstream.str()); + throw std::runtime_error{m_sstream.str()}; } void report_inner_with_same_tags(const osmium::Way& way) override { - m_sstream.str(); + m_sstream.str(""); ProblemReporterStream::report_inner_with_same_tags(way); - throw std::runtime_error(m_sstream.str()); + throw std::runtime_error{m_sstream.str()}; + } + + void report_invalid_location(osmium::object_id_type way_id, osmium::object_id_type node_id) override { + m_sstream.str(""); + ProblemReporterStream::report_invalid_location(way_id, node_id); + throw std::runtime_error{m_sstream.str()}; } }; // class ProblemReporterException diff --git a/contrib/libosmium/osmium/area/problem_reporter_ogr.hpp b/contrib/libosmium/osmium/area/problem_reporter_ogr.hpp index 914c8bad8..2d8026a9f 100644 --- a/contrib/libosmium/osmium/area/problem_reporter_ogr.hpp +++ b/contrib/libosmium/osmium/area/problem_reporter_ogr.hpp @@ -73,14 +73,14 @@ namespace osmium { gdalcpp::Layer m_layer_ways; void set_object(gdalcpp::Feature& feature) { - const char t[2] = { osmium::item_type_to_char(m_object_type), '\0' }; + const char t[2] = {osmium::item_type_to_char(m_object_type), '\0'}; feature.set_field("obj_type", t); feature.set_field("obj_id", int32_t(m_object_id)); feature.set_field("nodes", int32_t(m_nodes)); } void write_point(const char* problem_type, osmium::object_id_type id1, osmium::object_id_type id2, osmium::Location location) { - gdalcpp::Feature feature(m_layer_perror, m_ogr_factory.create_point(location)); + gdalcpp::Feature feature{m_layer_perror, m_ogr_factory.create_point(location)}; set_object(feature); feature.set_field("id1", double(id1)); feature.set_field("id2", double(id2)); @@ -93,7 +93,7 @@ namespace osmium { ogr_linestring->addPoint(loc1.lon(), loc1.lat()); ogr_linestring->addPoint(loc2.lon(), loc2.lat()); - gdalcpp::Feature feature(m_layer_lerror, std::move(ogr_linestring)); + gdalcpp::Feature feature{m_layer_lerror, std::move(ogr_linestring)}; set_object(feature); feature.set_field("id1", static_cast(id1)); feature.set_field("id2", static_cast(id2)); @@ -175,7 +175,7 @@ namespace osmium { return; } try { - gdalcpp::Feature feature(m_layer_lerror, m_ogr_factory.create_linestring(way)); + gdalcpp::Feature feature{m_layer_lerror, m_ogr_factory.create_linestring(way)}; set_object(feature); feature.set_field("id1", int32_t(way.id())); feature.set_field("id2", 0); @@ -191,7 +191,7 @@ namespace osmium { return; } try { - gdalcpp::Feature feature(m_layer_lerror, m_ogr_factory.create_linestring(way)); + gdalcpp::Feature feature{m_layer_lerror, m_ogr_factory.create_linestring(way)}; set_object(feature); feature.set_field("id1", int32_t(way.id())); feature.set_field("id2", 0); @@ -212,7 +212,7 @@ namespace osmium { return; } try { - gdalcpp::Feature feature(m_layer_ways, m_ogr_factory.create_linestring(way)); + gdalcpp::Feature feature{m_layer_ways, m_ogr_factory.create_linestring(way)}; set_object(feature); feature.set_field("way_id", int32_t(way.id())); feature.add_to_layer(); diff --git a/contrib/libosmium/osmium/area/problem_reporter_stream.hpp b/contrib/libosmium/osmium/area/problem_reporter_stream.hpp index 0c008f131..910eb4ec0 100644 --- a/contrib/libosmium/osmium/area/problem_reporter_stream.hpp +++ b/contrib/libosmium/osmium/area/problem_reporter_stream.hpp @@ -114,6 +114,11 @@ namespace osmium { *m_out << "way_id=" << way.id() << '\n'; } + void report_invalid_location(osmium::object_id_type way_id, osmium::object_id_type node_id) override { + header("invalid location"); + *m_out << "way_id=" << way_id << " node_id=" << node_id << '\n'; + } + }; // class ProblemReporterStream } // namespace area diff --git a/contrib/libosmium/osmium/area/stats.hpp b/contrib/libosmium/osmium/area/stats.hpp index 07be78376..48b40882b 100644 --- a/contrib/libosmium/osmium/area/stats.hpp +++ b/contrib/libosmium/osmium/area/stats.hpp @@ -68,6 +68,7 @@ namespace osmium { uint64_t touching_rings = 0; ///< Rings touching in a node uint64_t ways_in_multiple_rings = 0; ///< Different segments of a way ended up in different rings uint64_t wrong_role = 0; ///< Member has wrong role (not "outer", "inner", or empty) + uint64_t invalid_locations = 0; ///< Invalid location found area_stats& operator+=(const area_stats& other) noexcept { area_really_complex_case += other.area_really_complex_case; @@ -91,6 +92,7 @@ namespace osmium { touching_rings += other.touching_rings; ways_in_multiple_rings += other.ways_in_multiple_rings; wrong_role += other.wrong_role; + invalid_locations += invalid_locations; return *this; } @@ -118,7 +120,8 @@ namespace osmium { << " single_way_in_mp_relation=" << s.single_way_in_mp_relation << " touching_rings=" << s.touching_rings << " ways_in_multiple_rings=" << s.ways_in_multiple_rings - << " wrong_role=" << s.wrong_role; + << " wrong_role=" << s.wrong_role + << " invalid_locations=" << s.invalid_locations; } } // namespace area diff --git a/contrib/libosmium/osmium/builder/builder.hpp b/contrib/libosmium/osmium/builder/builder.hpp index 107f4a9d0..d4358843b 100644 --- a/contrib/libosmium/osmium/builder/builder.hpp +++ b/contrib/libosmium/osmium/builder/builder.hpp @@ -167,6 +167,20 @@ namespace osmium { return length; } + /** + * Append data to buffer and append an additional \0. + * + * @param data Pointer to data. + * @param length Length of data in bytes. + * @returns The number of bytes appended (length + 1). + */ + osmium::memory::item_size_type append_with_zero(const char* data, const osmium::memory::item_size_type length) { + unsigned char* target = reserve_space(length + 1); + std::copy_n(reinterpret_cast(data), length, target); + target[length] = '\0'; + return length + 1; + } + /** * Append \0-terminated string to buffer. * @@ -180,9 +194,11 @@ namespace osmium { /** * Append '\0' to the buffer. * + * @deprecated Use append_with_zero() instead. + * * @returns The number of bytes appended (always 1). */ - osmium::memory::item_size_type append_zero() { + OSMIUM_DEPRECATED osmium::memory::item_size_type append_zero() { *reserve_space(1) = '\0'; return 1; } diff --git a/contrib/libosmium/osmium/builder/builder_helper.hpp b/contrib/libosmium/osmium/builder/builder_helper.hpp index 8bfa21e24..a273e5041 100644 --- a/contrib/libosmium/osmium/builder/builder_helper.hpp +++ b/contrib/libosmium/osmium/builder/builder_helper.hpp @@ -71,7 +71,7 @@ namespace osmium { * @deprecated * Use osmium::builder::add_tag_list() instead. */ - inline const osmium::TagList& build_tag_list(osmium::memory::Buffer& buffer, const std::initializer_list>& tags) { + OSMIUM_DEPRECATED inline const osmium::TagList& build_tag_list(osmium::memory::Buffer& buffer, const std::initializer_list>& tags) { const size_t pos = buffer.committed(); { osmium::builder::TagListBuilder tl_builder(buffer); @@ -87,7 +87,7 @@ namespace osmium { * @deprecated * Use osmium::builder::add_tag_list() instead. */ - inline const osmium::TagList& build_tag_list_from_map(osmium::memory::Buffer& buffer, const std::map& tags) { + OSMIUM_DEPRECATED inline const osmium::TagList& build_tag_list_from_map(osmium::memory::Buffer& buffer, const std::map& tags) { const size_t pos = buffer.committed(); { osmium::builder::TagListBuilder tl_builder(buffer); @@ -103,7 +103,7 @@ namespace osmium { * @deprecated * Use osmium::builder::add_tag_list() instead. */ - inline const osmium::TagList& build_tag_list_from_func(osmium::memory::Buffer& buffer, std::function func) { + OSMIUM_DEPRECATED inline const osmium::TagList& build_tag_list_from_func(osmium::memory::Buffer& buffer, std::function func) { const size_t pos = buffer.committed(); { osmium::builder::TagListBuilder tl_builder(buffer); diff --git a/contrib/libosmium/osmium/builder/osm_object_builder.hpp b/contrib/libosmium/osmium/builder/osm_object_builder.hpp index 1dd5f972f..d4483b192 100644 --- a/contrib/libosmium/osmium/builder/osm_object_builder.hpp +++ b/contrib/libosmium/osmium/builder/osm_object_builder.hpp @@ -99,7 +99,8 @@ namespace osmium { if (std::strlen(value) > osmium::max_osm_string_length) { throw std::length_error("OSM tag value is too long"); } - add_size(append(key) + append(value)); + add_size(append(key)); + add_size(append(value)); } /** @@ -117,8 +118,8 @@ namespace osmium { if (value_length > osmium::max_osm_string_length) { throw std::length_error("OSM tag value is too long"); } - add_size(append(key, osmium::memory::item_size_type(key_length)) + append_zero() + - append(value, osmium::memory::item_size_type(value_length)) + append_zero()); + add_size(append_with_zero(key, osmium::memory::item_size_type(key_length))); + add_size(append_with_zero(value, osmium::memory::item_size_type(value_length))); } /** @@ -134,8 +135,8 @@ namespace osmium { if (value.size() > osmium::max_osm_string_length) { throw std::length_error("OSM tag value is too long"); } - add_size(append(key.data(), osmium::memory::item_size_type(key.size()) + 1) + - append(value.data(), osmium::memory::item_size_type(value.size()) + 1)); + add_size(append(key.data(), osmium::memory::item_size_type(key.size()) + 1)); + add_size(append(value.data(), osmium::memory::item_size_type(value.size()) + 1)); } /** @@ -144,7 +145,8 @@ namespace osmium { * @param tag Tag. */ void add_tag(const osmium::Tag& tag) { - add_size(append(tag.key()) + append(tag.value())); + add_size(append(tag.key())); + add_size(append(tag.value())); } /** @@ -226,7 +228,7 @@ namespace osmium { throw std::length_error("OSM relation member role is too long"); } member.set_role_size(osmium::string_size_type(length) + 1); - add_size(append(role, osmium::memory::item_size_type(length)) + append_zero()); + add_size(append_with_zero(role, osmium::memory::item_size_type(length))); add_padding(true); } @@ -310,7 +312,7 @@ namespace osmium { throw std::length_error("OSM user name is too long"); } comment.set_user_size(osmium::string_size_type(length) + 1); - add_size(append(user, osmium::memory::item_size_type(length)) + append_zero()); + add_size(append_with_zero(user, osmium::memory::item_size_type(length))); } void add_text(osmium::ChangesetComment& comment, const char* text, const size_t length) { @@ -322,7 +324,7 @@ namespace osmium { throw std::length_error("OSM changeset comment is too long"); } comment.set_text_size(osmium::string_size_type(length) + 1); - add_size(append(text, osmium::memory::item_size_type(length)) + append_zero()); + add_size(append_with_zero(text, osmium::memory::item_size_type(length))); add_padding(true); } diff --git a/contrib/libosmium/osmium/geom/factory.hpp b/contrib/libosmium/osmium/geom/factory.hpp index 6f8d954b0..39d1eacea 100644 --- a/contrib/libosmium/osmium/geom/factory.hpp +++ b/contrib/libosmium/osmium/geom/factory.hpp @@ -192,7 +192,7 @@ namespace osmium { using multipolygon_type = typename TGeomImpl::multipolygon_type; using ring_type = typename TGeomImpl::ring_type; - int epsg() const { + int epsg() const noexcept { return m_projection.epsg(); } diff --git a/contrib/libosmium/osmium/geom/mercator_projection.hpp b/contrib/libosmium/osmium/geom/mercator_projection.hpp index 814bce632..a6b798aa2 100644 --- a/contrib/libosmium/osmium/geom/mercator_projection.hpp +++ b/contrib/libosmium/osmium/geom/mercator_projection.hpp @@ -118,7 +118,7 @@ namespace osmium { * @pre @code c.valid() @endcode */ inline Coordinates lonlat_to_mercator(const Coordinates& c) { - return Coordinates(detail::lon_to_x(c.x), detail::lat_to_y(c.y)); + return Coordinates{detail::lon_to_x(c.x), detail::lat_to_y(c.y)}; } /** @@ -127,7 +127,7 @@ namespace osmium { * @pre @code c.valid() @endcode */ inline Coordinates mercator_to_lonlat(const Coordinates& c) { - return Coordinates(detail::x_to_lon(c.x), detail::y_to_lat(c.y)); + return Coordinates{detail::x_to_lon(c.x), detail::y_to_lat(c.y)}; } /** @@ -139,7 +139,7 @@ namespace osmium { public: Coordinates operator()(osmium::Location location) const { - return Coordinates {detail::lon_to_x(location.lon()), detail::lat_to_y(location.lat())}; + return Coordinates{detail::lon_to_x(location.lon()), detail::lat_to_y(location.lat())}; } int epsg() const noexcept { diff --git a/contrib/libosmium/osmium/geom/projection.hpp b/contrib/libosmium/osmium/geom/projection.hpp index 60e0ecf3d..132b0680e 100644 --- a/contrib/libosmium/osmium/geom/projection.hpp +++ b/contrib/libosmium/osmium/geom/projection.hpp @@ -48,6 +48,7 @@ DEALINGS IN THE SOFTWARE. #include #include +#include #include #include @@ -70,15 +71,15 @@ namespace osmium { public: - explicit CRS(const std::string& crs) : - m_crs(pj_init_plus(crs.c_str()), ProjCRSDeleter()) { + explicit CRS(const char* crs) : + m_crs(pj_init_plus(crs), ProjCRSDeleter()) { if (!m_crs) { - throw osmium::projection_error(std::string{"creation of CRS failed: "} + pj_strerrno(*pj_get_errno_ref())); + throw osmium::projection_error{std::string{"creation of CRS failed: "} + pj_strerrno(*pj_get_errno_ref())}; } } - explicit CRS(const char* crs) : - CRS(std::string{crs}) { + explicit CRS(const std::string& crs) : + CRS(crs.c_str()) { } explicit CRS(int epsg) : @@ -88,15 +89,15 @@ namespace osmium { /** * Get underlying projPJ handle from proj library. */ - projPJ get() const { + projPJ get() const noexcept { return m_crs.get(); } - bool is_latlong() const { + bool is_latlong() const noexcept { return pj_is_latlong(m_crs.get()) != 0; } - bool is_geocent() const { + bool is_geocent() const noexcept { return pj_is_geocent(m_crs.get()) != 0; } @@ -108,12 +109,12 @@ namespace osmium { * * Coordinates have to be in radians and are produced in radians. * - * @throws osmmium::projection_error if the projection fails + * @throws osmium::projection_error if the projection fails */ inline Coordinates transform(const CRS& src, const CRS& dest, Coordinates c) { - int result = pj_transform(src.get(), dest.get(), 1, 1, &c.x, &c.y, nullptr); + const int result = pj_transform(src.get(), dest.get(), 1, 1, &c.x, &c.y, nullptr); if (result != 0) { - throw osmium::projection_error(std::string("projection failed: ") + pj_strerrno(result)); + throw osmium::projection_error{std::string("projection failed: ") + pj_strerrno(result)}; } return c; } @@ -121,12 +122,19 @@ namespace osmium { /** * Functor that does projection from WGS84 (EPSG:4326) to the given * CRS. + * + * If this Projection is initialized with the constructor taking + * an integer with the epsg code 4326, no projection is done. If it + * is initialized with epsg code 3857 the Osmium-internal + * implementation of the Mercator projection is used, otherwise this + * falls back to using the proj.4 library. Note that this "magic" does + * not work if you use any of the constructors taking a string. */ class Projection { int m_epsg; std::string m_proj_string; - CRS m_crs_wgs84 {4326}; + CRS m_crs_wgs84{4326}; CRS m_crs_user; public: @@ -145,19 +153,23 @@ namespace osmium { explicit Projection(int epsg) : m_epsg(epsg), - m_proj_string(std::string("+init=epsg:") + std::to_string(epsg)), + m_proj_string(std::string{"+init=epsg:"} + std::to_string(epsg)), m_crs_user(epsg) { } Coordinates operator()(osmium::Location location) const { - Coordinates c {location.lon(), location.lat()}; - - if (m_epsg != 4326) { - c = transform(m_crs_wgs84, m_crs_user, Coordinates(deg_to_rad(location.lon()), deg_to_rad(location.lat()))); - if (m_crs_user.is_latlong()) { - c.x = rad_to_deg(c.x); - c.y = rad_to_deg(c.y); - } + if (m_epsg == 4326) { + return Coordinates{location.lon(), location.lat()}; + } else if (m_epsg == 3857) { + return Coordinates{detail::lon_to_x(location.lon()), + detail::lat_to_y(location.lat())}; + } + + Coordinates c{transform(m_crs_wgs84, m_crs_user, Coordinates{deg_to_rad(location.lon()), + deg_to_rad(location.lat())})}; + if (m_crs_user.is_latlong()) { + c.x = rad_to_deg(c.x); + c.y = rad_to_deg(c.y); } return c; diff --git a/contrib/libosmium/osmium/index/id_set.hpp b/contrib/libosmium/osmium/index/id_set.hpp index 50426b006..9f3631439 100644 --- a/contrib/libosmium/osmium/index/id_set.hpp +++ b/contrib/libosmium/osmium/index/id_set.hpp @@ -408,6 +408,7 @@ namespace osmium { }; // class IdSetSmall + /// @deprecated Use nrw_array helper class instead. template class IdSetType> class NWRIdSet { diff --git a/contrib/libosmium/osmium/index/map/sparse_mem_table.hpp b/contrib/libosmium/osmium/index/map/sparse_mem_table.hpp index f6a5fa34f..e2414624c 100644 --- a/contrib/libosmium/osmium/index/map/sparse_mem_table.hpp +++ b/contrib/libosmium/osmium/index/map/sparse_mem_table.hpp @@ -152,4 +152,4 @@ namespace osmium { #endif // OSMIUM_WITH_SPARSEHASH -#endif // OSMIUM_INDEX_BYID_SPARSE_MEM_TABLE_HPP +#endif // OSMIUM_INDEX_MAP_SPARSE_MEM_TABLE_HPP diff --git a/contrib/libosmium/osmium/index/nwr_array.hpp b/contrib/libosmium/osmium/index/nwr_array.hpp new file mode 100644 index 000000000..2c52c7e8f --- /dev/null +++ b/contrib/libosmium/osmium/index/nwr_array.hpp @@ -0,0 +1,59 @@ +#ifndef OSMIUM_INDEX_NWR_ARRAY_HPP +#define OSMIUM_INDEX_NWR_ARRAY_HPP + +/* + +This file is part of Osmium (http://osmcode.org/libosmium). + +Copyright 2013-2017 Jochen Topf and others (see README). + +Boost Software License - Version 1.0 - August 17th, 2003 + +Permission is hereby granted, free of charge, to any person or organization +obtaining a copy of the software and accompanying documentation covered by +this license (the "Software") to use, reproduce, display, distribute, +execute, and transmit the Software, and to prepare derivative works of the +Software, and to permit third-parties to whom the Software is furnished to +do so, all subject to the following: + +The copyright notices in the Software and this entire statement, including +the above license grant, this restriction and the following disclaimer, +must be included in all copies of the Software, in whole or in part, and +all derivative works of the Software, unless such copies or derivative +works are solely in the form of machine-executable object code generated by +a source language processor. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT +SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE +FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, +ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. + +*/ + +#include + +namespace osmium { + + template + class nwr_array { + + T m_sets[3]; + + public: + + T& operator()(osmium::item_type type) noexcept { + return m_sets[osmium::item_type_to_nwr_index(type)]; + } + + const T& operator()(osmium::item_type type) const noexcept { + return m_sets[osmium::item_type_to_nwr_index(type)]; + } + + }; // class nwr_array + +} // namespace osmium + +#endif // OSMIUM_INDEX_NWR_ARRAY_HPP diff --git a/contrib/libosmium/osmium/index/relations_map.hpp b/contrib/libosmium/osmium/index/relations_map.hpp index 5f46e6609..bb9f5a7ce 100644 --- a/contrib/libosmium/osmium/index/relations_map.hpp +++ b/contrib/libosmium/osmium/index/relations_map.hpp @@ -91,6 +91,24 @@ namespace osmium { m_map.emplace_back(key, value); } + typename std::enable_if::value>::type flip_in_place() { + for (auto& p : m_map) { + using std::swap; + swap(p.key, p.value); + } + } + + flat_map flip_copy() { + flat_map map; + map.reserve(m_map.size()); + + for (const auto& p : m_map) { + map.set(p.value, p.key); + } + + return map; + } + void sort_unique() { std::sort(m_map.begin(), m_map.end()); const auto last = std::unique(m_map.begin(), m_map.end()); @@ -111,39 +129,46 @@ namespace osmium { return m_map.size(); } + void reserve(size_t size) { + m_map.reserve(size); + } + }; // class flat_map } // namespace detail /** - * Index for looking up parent relation IDs given a member relation ID. - * You can not instantiate such an index yourself, instead you need to - * instantiate a RelationsMapStash, fill it and then create an index from - * it: - * - * @code - * RelationsMapStash stash; - * ... - * for_each_relation(const osmium::Relation& relation) { - * stash.add_members(relation); - * } - * ... - * const auto index = stash.build_index(); - * ... - * osmium::unsigned_object_id_type member_id = ...; - * index.for_each_parent(member_id, [](osmium::unsigned_object_id_type id) { - * ... - * }); - * ... - * @endcode - * - */ + * Index for looking up parent relation IDs given a member relation ID + * or the other way around. + * + * You can not instantiate such an index yourself, instead you need to + * instantiate a RelationsMapStash, fill it and then create an index + * from it: + * + * @code + * RelationsMapStash stash; + * ... + * for_each_relation(const osmium::Relation& relation) { + * stash.add_members(relation); + * } + * ... + * const auto index = stash.build_member_to_parent_index(); + * ... + * osmium::unsigned_object_id_type member_id = ...; + * index.for_each(member_id, [](osmium::unsigned_object_id_type parent_id) { + * ... + * }); + * ... + * @endcode + * + */ class RelationsMapIndex { friend class RelationsMapStash; + friend class RelationsMapIndexes; using map_type = detail::flat_map; + osmium::unsigned_object_id_type, uint32_t>; map_type m_map; @@ -162,56 +187,122 @@ namespace osmium { RelationsMapIndex& operator=(RelationsMapIndex&&) = default; /** - * Find the given relation id in the index and call the given function - * with all parent relation ids. - * - * @code - * osmium::unsigned_object_id_type member_id = 17; - * index.for_each_parent(member_id, [](osmium::unsigned_object_id_type id) { - * ... - * }); - * @endcode - * - * Complexity: Logarithmic in the number of elements in the index. - * (Lookup uses binary search.) - */ - template - void for_each_parent(osmium::unsigned_object_id_type member_id, Func&& func) const { + * Find the given relation id in the index and call the given + * function with all parent relation ids. + * + * @code + * osmium::unsigned_object_id_type member_id = 17; + * index.for_each_parent(member_id, [](osmium::unsigned_object_id_type id) { + * ... + * }); + * @endcode + * + * @deprecated Use for_each() instead. + * + * Complexity: Logarithmic in the number of elements in the index. + * (Lookup uses binary search.) + */ + template + void for_each_parent(osmium::unsigned_object_id_type member_id, TFunc&& func) const { const auto parents = m_map.get(member_id); for (auto it = parents.first; it != parents.second; ++it) { - std::forward(func)(it->value); + std::forward(func)(it->value); + } + } + + /** + * Find the given relation id in the index and call the given + * function with all related relation ids. + * + * @code + * osmium::unsigned_object_id_type id = 17; + * index.for_each(id, [](osmium::unsigned_object_id_type rid) { + * ... + * }); + * @endcode + * + * Complexity: Logarithmic in the number of elements in the index. + * (Lookup uses binary search.) + */ + template + void for_each(osmium::unsigned_object_id_type id, TFunc&& func) const { + const auto parents = m_map.get(id); + for (auto it = parents.first; it != parents.second; ++it) { + std::forward(func)(it->value); } } /** - * Is this index empty? - * - * Complexity: Constant. - */ + * Is this index empty? + * + * Complexity: Constant. + */ bool empty() const noexcept { return m_map.empty(); } /** - * How many entries are in this index? - * - * Complexity: Constant. - */ + * How many entries are in this index? + * + * Complexity: Constant. + */ size_t size() const noexcept { return m_map.size(); } - }; // RelationsMapIndex + }; // class RelationsMapIndex + + class RelationsMapIndexes { + + friend class RelationsMapStash; + + RelationsMapIndex m_member_to_parent; + RelationsMapIndex m_parent_to_member; + + RelationsMapIndexes(RelationsMapIndex::map_type&& map1, RelationsMapIndex::map_type&& map2) : + m_member_to_parent(std::move(map1)), + m_parent_to_member(std::move(map2)) { + } + + public: + + const RelationsMapIndex& member_to_parent() const noexcept { + return m_member_to_parent; + } + + const RelationsMapIndex& parent_to_member() const noexcept { + return m_parent_to_member; + } + + /** + * Is this index empty? + * + * Complexity: Constant. + */ + bool empty() const noexcept { + return m_member_to_parent.empty(); + } + + /** + * How many entries are in this index? + * + * Complexity: Constant. + */ + size_t size() const noexcept { + return m_member_to_parent.size(); + } + + }; // class RelationsMapIndexes /** - * The RelationsMapStash is used to build up the data needed to create - * an index of member relation ID to parent relation ID. See the - * RelationsMapIndex class for more. - */ + * The RelationsMapStash is used to build up the data needed to create + * an index of member relation ID to parent relation ID or the other + * way around. See the RelationsMapIndex class for more. + */ class RelationsMapStash { using map_type = detail::flat_map; + osmium::unsigned_object_id_type, uint32_t>; map_type m_map; @@ -230,16 +321,16 @@ namespace osmium { RelationsMapStash& operator=(RelationsMapStash&&) = default; /** - * Add mapping from member to parent relation in the stash. - */ + * Add mapping from member to parent relation in the stash. + */ void add(osmium::unsigned_object_id_type member_id, osmium::unsigned_object_id_type relation_id) { assert(m_valid && "You can't use the RelationsMap any more after calling build_index()"); m_map.set(member_id, relation_id); } /** - * Add mapping from all members to given parent relation in the stash. - */ + * Add mapping from all members to given parent relation in the stash. + */ void add_members(const osmium::Relation& relation) { assert(m_valid && "You can't use the RelationsMap any more after calling build_index()"); for (const auto& member : relation.members()) { @@ -250,30 +341,33 @@ namespace osmium { } /** - * Is this stash empty? - * - * Complexity: Constant. - */ + * Is this stash empty? + * + * Complexity: Constant. + */ bool empty() const noexcept { assert(m_valid && "You can't use the RelationsMap any more after calling build_index()"); return m_map.empty(); } /** - * How many entries are in this stash? - * - * Complexity: Constant. - */ + * How many entries are in this stash? + * + * Complexity: Constant. + */ size_t size() const noexcept { assert(m_valid && "You can't use the RelationsMap any more after calling build_index()"); return m_map.size(); } /** - * Build an index with the contents of this stash and return it. - * - * After you get the index you can not use the stash any more! - */ + * Build an index for member to parent lookups from the contents + * of this stash and return it. + * + * After you get the index you can not use the stash any more! + * + * @deprecated Use build_member_to_parent_index() instead. + */ RelationsMapIndex build_index() { assert(m_valid && "You can't use the RelationsMap any more after calling build_index()"); m_map.sort_unique(); @@ -283,6 +377,54 @@ namespace osmium { return RelationsMapIndex{std::move(m_map)}; } + /** + * Build an index for member to parent lookups from the contents + * of this stash and return it. + * + * After you get the index you can not use the stash any more! + */ + RelationsMapIndex build_member_to_parent_index() { + assert(m_valid && "You can't use the RelationsMap any more after calling build_member_to_parent_index()"); + m_map.sort_unique(); +#ifndef NDEBUG + m_valid = false; +#endif + return RelationsMapIndex{std::move(m_map)}; + } + + /** + * Build an index for parent to member lookups from the contents + * of this stash and return it. + * + * After you get the index you can not use the stash any more! + */ + RelationsMapIndex build_parent_to_member_index() { + assert(m_valid && "You can't use the RelationsMap any more after calling build_parent_to_member_index()"); + m_map.flip_in_place(); + m_map.sort_unique(); +#ifndef NDEBUG + m_valid = false; +#endif + return RelationsMapIndex{std::move(m_map)}; + } + + /** + * Build indexes for member-to-parent and parent-to-member lookups + * from the contents of this stash and return them. + * + * After you get the index you can not use the stash any more! + */ + RelationsMapIndexes build_indexes() { + assert(m_valid && "You can't use the RelationsMap any more after calling build_indexes()"); + auto reverse_map = m_map.flip_copy(); + reverse_map.sort_unique(); + m_map.sort_unique(); +#ifndef NDEBUG + m_valid = false; +#endif + return RelationsMapIndexes{std::move(m_map), std::move(reverse_map)}; + } + }; // class RelationsMapStash } // namespace index diff --git a/contrib/libosmium/osmium/io/detail/pbf_input_format.hpp b/contrib/libosmium/osmium/io/detail/pbf_input_format.hpp index be5f86097..74fc4f89d 100644 --- a/contrib/libosmium/osmium/io/detail/pbf_input_format.hpp +++ b/contrib/libosmium/osmium/io/detail/pbf_input_format.hpp @@ -33,7 +33,6 @@ DEALINGS IN THE SOFTWARE. */ -#include #include #include #include @@ -42,6 +41,7 @@ DEALINGS IN THE SOFTWARE. #include #include #include +#include #include #include diff --git a/contrib/libosmium/osmium/io/detail/pbf_output_format.hpp b/contrib/libosmium/osmium/io/detail/pbf_output_format.hpp index d431a6796..00b364f6d 100644 --- a/contrib/libosmium/osmium/io/detail/pbf_output_format.hpp +++ b/contrib/libosmium/osmium/io/detail/pbf_output_format.hpp @@ -33,7 +33,6 @@ DEALINGS IN THE SOFTWARE. */ -#include #include #include #include diff --git a/contrib/libosmium/osmium/io/detail/protobuf_tags.hpp b/contrib/libosmium/osmium/io/detail/protobuf_tags.hpp index 8bb2e56c6..58fff5694 100644 --- a/contrib/libosmium/osmium/io/detail/protobuf_tags.hpp +++ b/contrib/libosmium/osmium/io/detail/protobuf_tags.hpp @@ -169,4 +169,4 @@ namespace osmium { } // namespace osmium -#endif // OSMIUM_IO_DETAIL_PROTOBUF_TAGS_HPP +#endif // OSMIUM_IO_DETAIL_PROTOBUF_TAGS_HPP diff --git a/contrib/libosmium/osmium/io/detail/write_thread.hpp b/contrib/libosmium/osmium/io/detail/write_thread.hpp index 1e22155f4..1b0745184 100644 --- a/contrib/libosmium/osmium/io/detail/write_thread.hpp +++ b/contrib/libosmium/osmium/io/detail/write_thread.hpp @@ -33,11 +33,11 @@ DEALINGS IN THE SOFTWARE. */ -#include #include #include #include #include +#include #include #include diff --git a/contrib/libosmium/osmium/io/detail/xml_output_format.hpp b/contrib/libosmium/osmium/io/detail/xml_output_format.hpp index 44d68313c..130a3c734 100644 --- a/contrib/libosmium/osmium/io/detail/xml_output_format.hpp +++ b/contrib/libosmium/osmium/io/detail/xml_output_format.hpp @@ -33,7 +33,6 @@ DEALINGS IN THE SOFTWARE. */ -#include #include #include #include diff --git a/contrib/libosmium/osmium/osm/crc.hpp b/contrib/libosmium/osmium/osm/crc.hpp index 5de52ed18..c191db93a 100644 --- a/contrib/libosmium/osmium/osm/crc.hpp +++ b/contrib/libosmium/osmium/osm/crc.hpp @@ -246,4 +246,4 @@ namespace osmium { } // namespace osmium -#endif // OSMIUM_OSM_CRC +#endif // OSMIUM_OSM_CRC_HPP diff --git a/contrib/libosmium/osmium/relations/collector.hpp b/contrib/libosmium/osmium/relations/collector.hpp index bede76405..ee17d47b8 100644 --- a/contrib/libosmium/osmium/relations/collector.hpp +++ b/contrib/libosmium/osmium/relations/collector.hpp @@ -293,17 +293,6 @@ namespace osmium { void flush() { } - /** - * This removes all relations that have already been assembled - * from the m_relations vector. - */ - void clean_assembled_relations() { - m_relations.erase( - std::remove_if(m_relations.begin(), m_relations.end(), has_all_members()), - m_relations.end() - ); - } - const osmium::Relation& get_relation(size_t offset) const { assert(m_relations_buffer.committed() > offset); return m_relations_buffer.get(offset); diff --git a/contrib/libosmium/osmium/tags/filter.hpp b/contrib/libosmium/osmium/tags/filter.hpp index 22abb1430..e8f3da806 100644 --- a/contrib/libosmium/osmium/tags/filter.hpp +++ b/contrib/libosmium/osmium/tags/filter.hpp @@ -34,6 +34,7 @@ DEALINGS IN THE SOFTWARE. */ #include +#include #include #include #include @@ -49,27 +50,41 @@ namespace osmium { template struct match_key { - bool operator()(const TKey& rule_key, const char* tag_key) { + bool operator()(const TKey& rule_key, const char* tag_key) const { return rule_key == tag_key; } }; // struct match_key + template <> + struct match_key { + bool operator()(const std::string& rule_key, const char* tag_key) const { + return !std::strcmp(rule_key.c_str(), tag_key); + } + }; // struct match_key + struct match_key_prefix { - bool operator()(const std::string& rule_key, const char* tag_key) { + bool operator()(const std::string& rule_key, const char* tag_key) const { return rule_key.compare(0, std::string::npos, tag_key, 0, rule_key.size()) == 0; } }; // struct match_key_prefix template struct match_value { - bool operator()(const TValue& rule_value, const char* tag_value) { + bool operator()(const TValue& rule_value, const char* tag_value) const { return rule_value == tag_value; } }; // struct match_value + template <> + struct match_value { + bool operator()(const std::string& rule_value, const char* tag_value) const { + return !std::strcmp(rule_value.c_str(), tag_value); + } + }; // struct match_value + template <> struct match_value { - bool operator()(const bool, const char*) { + bool operator()(const bool, const char*) const noexcept { return true; } }; // struct match_value @@ -138,15 +153,19 @@ namespace osmium { /** * Return the number of rules in this filter. + * + * Complexity: Constant. */ - size_t count() const { - return m_rules.count(); + size_t count() const noexcept { + return m_rules.size(); } /** * Is this filter empty, ie are there no rules defined? + * + * Complexity: Constant. */ - bool empty() const { + bool empty() const noexcept { return m_rules.empty(); } diff --git a/contrib/libosmium/osmium/tags/regex_filter.hpp b/contrib/libosmium/osmium/tags/regex_filter.hpp index 72c3217e5..16aefe0cc 100644 --- a/contrib/libosmium/osmium/tags/regex_filter.hpp +++ b/contrib/libosmium/osmium/tags/regex_filter.hpp @@ -44,14 +44,14 @@ namespace osmium { template <> struct match_key { - bool operator()(const std::regex& rule_key, const char* tag_key) { + bool operator()(const std::regex& rule_key, const char* tag_key) const { return std::regex_match(tag_key, rule_key); } }; // struct match_key template <> struct match_value { - bool operator()(const std::regex& rule_value, const char* tag_value) { + bool operator()(const std::regex& rule_value, const char* tag_value) const { return std::regex_match(tag_value, rule_value); } }; // struct match_value diff --git a/contrib/libosmium/osmium/tags/taglist.hpp b/contrib/libosmium/osmium/tags/taglist.hpp index 30db9e99c..3f640d6b2 100644 --- a/contrib/libosmium/osmium/tags/taglist.hpp +++ b/contrib/libosmium/osmium/tags/taglist.hpp @@ -34,6 +34,7 @@ DEALINGS IN THE SOFTWARE. */ #include +#include #include // IWYU pragma: keep #include @@ -46,18 +47,18 @@ namespace osmium { namespace tags { template - inline bool match_any_of(const osmium::TagList& tag_list, TFilter&& filter) { - return std::any_of(tag_list.cbegin(), tag_list.cend(), std::forward(filter)); + inline bool match_any_of(const osmium::TagList& tag_list, const TFilter& filter) { + return std::any_of(tag_list.cbegin(), tag_list.cend(), std::cref(filter)); } template - inline bool match_all_of(const osmium::TagList& tag_list, TFilter&& filter) { - return std::all_of(tag_list.cbegin(), tag_list.cend(), std::forward(filter)); + inline bool match_all_of(const osmium::TagList& tag_list, const TFilter& filter) { + return std::all_of(tag_list.cbegin(), tag_list.cend(), std::cref(filter)); } template - inline bool match_none_of(const osmium::TagList& tag_list, TFilter&& filter) { - return std::none_of(tag_list.cbegin(), tag_list.cend(), std::forward(filter)); + inline bool match_none_of(const osmium::TagList& tag_list, const TFilter& filter) { + return std::none_of(tag_list.cbegin(), tag_list.cend(), std::cref(filter)); } } // namespace tags diff --git a/contrib/libosmium/osmium/thread/function_wrapper.hpp b/contrib/libosmium/osmium/thread/function_wrapper.hpp index 2ada084d3..eeb942542 100644 --- a/contrib/libosmium/osmium/thread/function_wrapper.hpp +++ b/contrib/libosmium/osmium/thread/function_wrapper.hpp @@ -33,8 +33,8 @@ DEALINGS IN THE SOFTWARE. */ -#include #include +#include namespace osmium { diff --git a/contrib/libosmium/osmium/thread/pool.hpp b/contrib/libosmium/osmium/thread/pool.hpp index 06d96f52a..a89b3e940 100644 --- a/contrib/libosmium/osmium/thread/pool.hpp +++ b/contrib/libosmium/osmium/thread/pool.hpp @@ -33,11 +33,11 @@ DEALINGS IN THE SOFTWARE. */ -#include #include #include #include #include +#include #include #include diff --git a/contrib/libosmium/osmium/thread/util.hpp b/contrib/libosmium/osmium/thread/util.hpp index 4eb59c9d1..53d16a19e 100644 --- a/contrib/libosmium/osmium/thread/util.hpp +++ b/contrib/libosmium/osmium/thread/util.hpp @@ -115,4 +115,4 @@ namespace osmium { } // namespace osmium -#endif // OSMIUM_THREAD_UTIL_HPP +#endif // OSMIUM_THREAD_UTIL_HPP diff --git a/contrib/libosmium/osmium/util/file.hpp b/contrib/libosmium/osmium/util/file.hpp index a26d88fc1..e4083c0d9 100644 --- a/contrib/libosmium/osmium/util/file.hpp +++ b/contrib/libosmium/osmium/util/file.hpp @@ -70,14 +70,14 @@ namespace osmium { // https://msdn.microsoft.com/en-us/library/dfbc2kec.aspx const auto size = ::_filelengthi64(fd); if (size == -1L) { - throw std::system_error(errno, std::system_category(), "_filelengthi64 failed"); + throw std::system_error{errno, std::system_category(), "Could not get file size"}; } return size_t(size); #else // Unix implementation struct stat s; if (::fstat(fd, &s) != 0) { - throw std::system_error(errno, std::system_category(), "fstat failed"); + throw std::system_error{errno, std::system_category(), "Could not get file size"}; } return size_t(s.st_size); #endif @@ -97,13 +97,13 @@ namespace osmium { // https://msdn.microsoft.com/en-us/library/14h5k7ff.aspx struct _stat64 s; if (::_stati64(name, &s) != 0) { - throw std::system_error(errno, std::system_category(), "_stati64 failed"); + throw std::system_error{errno, std::system_category(), std::string{"Could not get file size of file '"} + name + "'"}; } #else // Unix implementation struct stat s; if (::stat(name, &s) != 0) { - throw std::system_error(errno, std::system_category(), "stat failed"); + throw std::system_error{errno, std::system_category(), std::string{"Could not get file size of file '"} + name + "'"}; } #endif return size_t(s.st_size); @@ -136,7 +136,7 @@ namespace osmium { #else if (::ftruncate(fd, static_cast_with_assert(new_size)) != 0) { #endif - throw std::system_error(errno, std::system_category(), "resizing file failed"); + throw std::system_error{errno, std::system_category(), "Could not resize file"}; } } diff --git a/osmium-builder.cpp b/osmium-builder.cpp index 914ac8ff3..ec5f25b8b 100644 --- a/osmium-builder.cpp +++ b/osmium-builder.cpp @@ -3,7 +3,7 @@ #include #include -#include +#include #include "osmium-builder.hpp" @@ -132,10 +132,10 @@ osmium_builder_t::get_wkb_line(osmium::WayNodeList const &nodes, bool do_split) osmium_builder_t::wkb_t osmium_builder_t::get_wkb_polygon(osmium::Way const &way) { - osmium::area::Assembler assembler{area_config}; + osmium::area::GeomAssembler assembler{area_config}; m_buffer.clear(); - if (!assembler.make_area(way, m_buffer)) { + if (!assembler(way, m_buffer)) { return wkb_t(); } @@ -149,10 +149,10 @@ osmium_builder_t::get_wkb_multipolygon(osmium::Relation const &rel, osmium::memory::Buffer const &ways) { wkbs_t ret; - osmium::area::Assembler assembler{area_config}; + osmium::area::GeomAssembler assembler{area_config}; m_buffer.clear(); - if (assembler.make_area(rel, ways, m_buffer)) { + if (assembler(rel, ways, m_buffer)) { if (m_build_multigeoms) { ret.push_back(create_multipolygon(m_buffer.get(0))); } else { From cd9a0a73ed9f460485c27f3de28a3fecb7711a7c Mon Sep 17 00:00:00 2001 From: Sarah Hoffmann Date: Sun, 12 Feb 2017 11:12:35 +0100 Subject: [PATCH 23/30] fix number of polygons when creating MP --- osmium-builder.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/osmium-builder.cpp b/osmium-builder.cpp index ec5f25b8b..50be96870 100644 --- a/osmium-builder.cpp +++ b/osmium-builder.cpp @@ -346,9 +346,10 @@ osmium_builder_t::create_multipolygon(osmium::Area const &area) for (auto it = area.cbegin(); it != area.cend(); ++it) { if (it->type() == osmium::item_type::outer_ring) { auto &ring = static_cast(*it); - if (num_polygons > 0) { + if (num_rings > 0) { m_writer.multipolygon_polygon_finish(num_rings); num_rings = 0; + ++num_polygons; } m_writer.polygon_start(); m_writer.polygon_ring_start(); @@ -365,8 +366,9 @@ osmium_builder_t::create_multipolygon(osmium::Area const &area) } // if there are no polygons, this area is invalid - if (num_polygons > 0) { + if (num_rings > 0) { m_writer.multipolygon_polygon_finish(num_rings); + ++num_polygons; ret = m_writer.multipolygon_finish(num_polygons); } else { ret.clear(); From 14130fc59b7d56c3f3b011b03f7cd0bf031d4063 Mon Sep 17 00:00:00 2001 From: Sarah Hoffmann Date: Sun, 12 Feb 2017 14:57:56 +0100 Subject: [PATCH 24/30] fix location lookup in nodes table when node appears twice --- middle-pgsql.cpp | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/middle-pgsql.cpp b/middle-pgsql.cpp index b9a12eb4a..147ad53be 100644 --- a/middle-pgsql.cpp +++ b/middle-pgsql.cpp @@ -290,7 +290,6 @@ size_t middle_pgsql_t::local_nodes_get_list(osmium::WayNodeList *nodes) const { size_t count = 0; std::string buffer("{"); - std::unordered_map node_pos; // get nodes where possible from cache, // at the same time build a list for querying missing nodes from DB @@ -301,14 +300,13 @@ size_t middle_pgsql_t::local_nodes_get_list(osmium::WayNodeList *nodes) const n.set_location(loc); ++count; } else { - node_pos.emplace(n.ref(), pos); buffer += std::to_string(n.ref()); buffer += ','; } ++pos; } - if (node_pos.empty()) { + if (count == pos) { return count; // all ids found in cache, nothing more to do } @@ -325,15 +323,23 @@ size_t middle_pgsql_t::local_nodes_get_list(osmium::WayNodeList *nodes) const paramValues, PGRES_TUPLES_OK); auto countPG = PQntuples(res); + std::unordered_map locs; for (int i = 0; i < countPG; ++i) { - auto &npos = node_pos.at(strtoosmid(PQgetvalue(res, i, 0), nullptr, 10)); + locs.emplace(strtoosmid(PQgetvalue(res, i, 0), nullptr, 10), + osmium::Location((int)strtol(PQgetvalue(res, i, 2), nullptr, 10), + (int)strtol(PQgetvalue(res, i, 1), nullptr, 10))); + } + + for (auto &n : *nodes) { + auto el = locs.find(n.ref()); + if (el != locs.end()) { + n.set_location(el->second); + ++count; + } - osmium::Location loc((int)strtol(PQgetvalue(res, i, 2), nullptr, 10), - (int)strtol(PQgetvalue(res, i, 1), nullptr, 10)); - (*nodes)[npos].set_location(loc); } - return count + (size_t) countPG; + return count; } void middle_pgsql_t::nodes_set(osmium::Node const &node) From 364cb406bbfc89caa22d4bb906eca039ce96526d Mon Sep 17 00:00:00 2001 From: Sarah Hoffmann Date: Sun, 12 Feb 2017 14:59:04 +0100 Subject: [PATCH 25/30] ignore invalid nodes when building areas --- osmium-builder.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/osmium-builder.cpp b/osmium-builder.cpp index 50be96870..012a08eb1 100644 --- a/osmium-builder.cpp +++ b/osmium-builder.cpp @@ -9,8 +9,6 @@ namespace { -osmium::area::AssemblerConfig area_config; - inline double distance(osmium::geom::Coordinates p1, osmium::geom::Coordinates p2) { @@ -132,6 +130,8 @@ osmium_builder_t::get_wkb_line(osmium::WayNodeList const &nodes, bool do_split) osmium_builder_t::wkb_t osmium_builder_t::get_wkb_polygon(osmium::Way const &way) { + osmium::area::AssemblerConfig area_config; + area_config.ignore_invalid_locations = true; osmium::area::GeomAssembler assembler{area_config}; m_buffer.clear(); @@ -149,6 +149,8 @@ osmium_builder_t::get_wkb_multipolygon(osmium::Relation const &rel, osmium::memory::Buffer const &ways) { wkbs_t ret; + osmium::area::AssemblerConfig area_config; + area_config.ignore_invalid_locations = true; osmium::area::GeomAssembler assembler{area_config}; m_buffer.clear(); From d4911465f5cf184583e83ce22b74f24c882f4064 Mon Sep 17 00:00:00 2001 From: Sarah Hoffmann Date: Sun, 12 Feb 2017 16:59:43 +0100 Subject: [PATCH 26/30] restore previous polygon behaviour - way_area should not appear in hstore column in roads and line table - when using -G simple polygons should still be of type POLYGON --- osmium-builder.cpp | 49 ++++++++++++---------------------------------- output-pgsql.cpp | 24 ++++++++++++----------- 2 files changed, 26 insertions(+), 47 deletions(-) diff --git a/osmium-builder.cpp b/osmium-builder.cpp index 012a08eb1..5d1ec506a 100644 --- a/osmium-builder.cpp +++ b/osmium-builder.cpp @@ -340,44 +340,21 @@ osmium_builder_t::create_multipolygon(osmium::Area const &area) { wkb_t ret; - try { - size_t num_polygons = 0; - size_t num_rings = 0; + auto polys = create_polygons(area); + + switch (polys.size()) { + case 0: + break; //nothing + case 1: + ret = polys[0]; + break; + default: m_writer.multipolygon_start(); - - for (auto it = area.cbegin(); it != area.cend(); ++it) { - if (it->type() == osmium::item_type::outer_ring) { - auto &ring = static_cast(*it); - if (num_rings > 0) { - m_writer.multipolygon_polygon_finish(num_rings); - num_rings = 0; - ++num_polygons; - } - m_writer.polygon_start(); - m_writer.polygon_ring_start(); - auto num_points = add_mp_points(ring); - m_writer.polygon_ring_finish(num_points); - ++num_rings; - } else if (it->type() == osmium::item_type::inner_ring) { - auto &ring = static_cast(*it); - m_writer.polygon_ring_start(); - auto num_points = add_mp_points(ring); - m_writer.polygon_ring_finish(num_points); - ++num_rings; - } - } - - // if there are no polygons, this area is invalid - if (num_rings > 0) { - m_writer.multipolygon_polygon_finish(num_rings); - ++num_polygons; - ret = m_writer.multipolygon_finish(num_polygons); - } else { - ret.clear(); + for (auto const &p : polys) { + m_writer.add_sub_geometry(p); } - - } catch (osmium::geometry_error &e) { - /* ignored */ + ret = m_writer.multipolygon_finish(polys.size()); + break; } return ret; diff --git a/output-pgsql.cpp b/output-pgsql.cpp index ccd38dc90..7e2326fd6 100644 --- a/output-pgsql.cpp +++ b/output-pgsql.cpp @@ -370,6 +370,19 @@ int output_pgsql_t::pgsql_process_relation(osmium::Relation const &rel, m_mid->nodes_get_list(&(w.nodes())); } + // linear features and boundaries + // Needs to be done before the polygon treatment below because + // for boundaries the way_area tag may be added. + if (!make_polygon) { + auto wkbs = m_builder.get_wkb_multiline(buffer, true); + for (auto const &wkb : wkbs) { + expire.from_wkb(wkb.c_str(), -rel.id()); + m_tables[t_line]->write_row(-rel.id(), outtags, wkb); + if (roads) + m_tables[t_roads]->write_row(-rel.id(), outtags, wkb); + } + } + // multipolygons and boundaries if (make_boundary || make_polygon) { auto wkbs = m_builder.get_wkb_multipolygon(rel, buffer); @@ -409,17 +422,6 @@ int output_pgsql_t::pgsql_process_relation(osmium::Relation const &rel, } } - // linear features and boundaries - if (!make_polygon) { - auto wkbs = m_builder.get_wkb_multiline(buffer, true); - for (auto const &wkb : wkbs) { - expire.from_wkb(wkb.c_str(), -rel.id()); - m_tables[t_line]->write_row(-rel.id(), outtags, wkb); - if (roads) - m_tables[t_roads]->write_row(-rel.id(), outtags, wkb); - } - } - return 0; } From 0333e11adff2a7e09fb3ae00a0de69ae5dc87ad6 Mon Sep 17 00:00:00 2001 From: Sarah Hoffmann Date: Sun, 12 Feb 2017 17:01:25 +0100 Subject: [PATCH 27/30] adapt regression tests to new libosmium geometry creation Changes are mainly due to broken polygons not being added anywhere and a more eager line merging algorithm for routes. --- tests/regression-test.py | 48 ++++++++++++++++++++-------------------- 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/tests/regression-test.py b/tests/regression-test.py index b93a8e3d9..12fbe6407 100755 --- a/tests/regression-test.py +++ b/tests/regression-test.py @@ -19,27 +19,27 @@ #**************************************************************** sql_test_statements=[ ( 0, 'Basic point count', 'SELECT count(*) FROM planet_osm_point;', 1342 ), - ( 1, 'Basic line count', 'SELECT count(*) FROM planet_osm_line;', 3300 ), + ( 1, 'Basic line count', 'SELECT count(*) FROM planet_osm_line;', 3231 ), ( 2, 'Basic road count', 'SELECT count(*) FROM planet_osm_roads;', 375 ), - ( 3, 'Basic polygon count', 'SELECT count(*) FROM planet_osm_polygon;', 4128 ), - ( 4, 'Basic latlon line count', 'SELECT count(*) FROM planet_osm_line;', 3298 ), + ( 3, 'Basic polygon count', 'SELECT count(*) FROM planet_osm_polygon;', 4127 ), + ( 4, 'Basic latlon line count', 'SELECT count(*) FROM planet_osm_line;', 3229 ), ( 5, 'Basic latlon road count', 'SELECT count(*) FROM planet_osm_roads;', 374 ), ( 6, 'Basic post-diff point count', 'SELECT count(*) FROM planet_osm_point;', 1457 ), - ( 7, 'Basic post-diff line count', 'SELECT count(*) FROM planet_osm_line;', 3344 ), - ( 8, 'Basic post-diff road count', 'SELECT count(*) FROM planet_osm_roads;', 381 ), - ( 9, 'Basic post-diff polygon count', 'SELECT count(*) FROM planet_osm_polygon;', 4275 ), + ( 7, 'Basic post-diff line count', 'SELECT count(*) FROM planet_osm_line;', 3274 ), + ( 8, 'Basic post-diff road count', 'SELECT count(*) FROM planet_osm_roads;', 380 ), + ( 9, 'Basic post-diff polygon count', 'SELECT count(*) FROM planet_osm_polygon;', 4274 ), ( 10, 'Absence of nodes table', 'SELECT count(*) FROM pg_tables WHERE tablename = \'planet_osm_nodes\'', 0), ( 11, 'Absence of way table', 'SELECT count(*) FROM pg_tables WHERE tablename = \'planet_osm_ways\'', 0), ( 12, 'Absence of rel line', 'SELECT count(*) FROM pg_tables WHERE tablename = \'planet_osm_rels\'', 0), - ( 13, 'Basic polygon area', 'SELECT round(sum(cast(ST_Area(way) as numeric)),0) FROM planet_osm_polygon;', 1223800703), - ( 14, 'Gazetteer place count', 'SELECT count(*) FROM place', 2837), + ( 13, 'Basic polygon area', 'SELECT round(sum(cast(ST_Area(way) as numeric)),0) FROM planet_osm_polygon;', 1210958566), + ( 14, 'Gazetteer place count', 'SELECT count(*) FROM place', 2836), ( 15, 'Gazetteer place node count', 'SELECT count(*) FROM place WHERE osm_type = \'N\'', 759), ( 16, 'Gazetteer place way count', 'SELECT count(*) FROM place WHERE osm_type = \'W\'', 2059), - ( 17, 'Gazetteer place rel count', 'SELECT count(*) FROM place WHERE osm_type = \'R\'', 19), - ( 18, 'Gazetteer post-diff place count', 'SELECT count(*) FROM place', 2878), + ( 17, 'Gazetteer place rel count', 'SELECT count(*) FROM place WHERE osm_type = \'R\'', 18), + ( 18, 'Gazetteer post-diff place count', 'SELECT count(*) FROM place', 2877), ( 19, 'Gazetteer post-diff place node count', 'SELECT count(*) FROM place WHERE osm_type = \'N\'', 764), ( 20, 'Gazetteer post-diff place way count', 'SELECT count(*) FROM place WHERE osm_type = \'W\'', 2095), - ( 21, 'Gazetteer post-diff place rel count', 'SELECT count(*) FROM place WHERE osm_type = \'R\'', 19), + ( 21, 'Gazetteer post-diff place rel count', 'SELECT count(*) FROM place WHERE osm_type = \'R\'', 18), ( 22, 'Gazetteer housenumber count', 'SELECT count(*) FROM place WHERE housenumber is not null', 199), ( 23, 'Gazetteer post-diff housenumber count count', 'SELECT count(*) FROM place WHERE housenumber is not null', 199), ( 24, 'Gazetteer isin count', 'SELECT count(*) FROM place WHERE isin is not null', 239), @@ -95,19 +95,19 @@ ( 50, 'Multipolygon nested outer ways. Both outer and inner ways are from multiple ways (multigeometry)', 'SELECT ST_NumGeometries(way) FROM planet_osm_polygon WHERE osm_id = -7 and landuse = \'farmland\' and name = \'Name_rel15\'', 2), ( 51, 'Basic hstore point count', 'SELECT count(*) FROM planet_osm_point;', 1360 ), - ( 52, 'Basic hstore line count', 'SELECT count(*) FROM planet_osm_line;', 3323 ), + ( 52, 'Basic hstore line count', 'SELECT count(*) FROM planet_osm_line;', 3254 ), ( 53, 'Basic hstore road count', 'SELECT count(*) FROM planet_osm_roads;', 375 ), - ( 54, 'Basic hstore polygon count', 'SELECT count(*) FROM planet_osm_polygon;', 4128 ), + ( 54, 'Basic hstore polygon count', 'SELECT count(*) FROM planet_osm_polygon;', 4127 ), ( 55, 'Basic post-diff point count', 'SELECT count(*) FROM planet_osm_point;', 1475 ), - ( 56, 'Basic post-diff line count', 'SELECT count(*) FROM planet_osm_line;', 3367 ), - ( 57, 'Basic post-diff road count', 'SELECT count(*) FROM planet_osm_roads;', 381 ), - ( 58, 'Basic post-diff polygon count', 'SELECT count(*) FROM planet_osm_polygon;', 4275 ), + ( 56, 'Basic post-diff line count', 'SELECT count(*) FROM planet_osm_line;', 3297 ), + ( 57, 'Basic post-diff road count', 'SELECT count(*) FROM planet_osm_roads;', 380 ), + ( 58, 'Basic post-diff polygon count', 'SELECT count(*) FROM planet_osm_polygon;', 4274 ), ( 59, 'Extra hstore full tags point count', 'SELECT count(*) FROM planet_osm_point WHERE tags ? \'osm_user\' and tags ? \'osm_version\' and tags ? \'osm_uid\' and tags ? \'osm_changeset\'', 1360), ( 60, 'Extra hstore full tags line count', - 'SELECT count(*) FROM planet_osm_line WHERE tags ? \'osm_user\' and tags ? \'osm_version\' and tags ? \'osm_uid\' and tags ? \'osm_changeset\'', 3323), + 'SELECT count(*) FROM planet_osm_line WHERE tags ? \'osm_user\' and tags ? \'osm_version\' and tags ? \'osm_uid\' and tags ? \'osm_changeset\'', 3254), ( 61, 'Extra hstore full tags polygon count', - 'SELECT count(*) FROM planet_osm_polygon WHERE tags ? \'osm_user\' and tags ? \'osm_version\' and tags ? \'osm_uid\' and tags ? \'osm_changeset\'', 4128), + 'SELECT count(*) FROM planet_osm_polygon WHERE tags ? \'osm_user\' and tags ? \'osm_version\' and tags ? \'osm_uid\' and tags ? \'osm_changeset\'', 4127), ( 62, 'Multipolygon copying of tags from outer with extra tags on relation', 'SELECT round(ST_Area(way)) FROM planet_osm_polygon WHERE osm_id = -22', 20878), ( 63, 'Multipolygon copying of tags from outer with extra tags on relation (abscence of way)', @@ -166,16 +166,16 @@ 'SELECT round(ST_Area(way)) FROM planet_osm_polygon WHERE osm_id = -35 and "natural" = \'water\'', 15737), ( 90, 'Multipolygon tags on relation two outer diff remove way from relation (presence of single way)', 'SELECT round(ST_Area(way)) FROM planet_osm_polygon WHERE osm_id = 102 and "natural" = \'water\'', 12994), - ( 91, 'Basic line length', 'SELECT round(sum(ST_Length(way))) FROM planet_osm_line;', 4269394), + ( 91, 'Basic line length', 'SELECT round(sum(ST_Length(way))) FROM planet_osm_line;', 4211350), ( 92, 'Basic line length', 'SELECT round(sum(ST_Length(way))) FROM planet_osm_roads;', 2032023), ( 93, 'Basic number of hstore points tags', 'SELECT sum(array_length(akeys(tags),1)) FROM planet_osm_point;', 4228), ( 94, 'Basic number of hstore roads tags', 'SELECT sum(array_length(akeys(tags),1)) FROM planet_osm_roads;', 2317), - ( 95, 'Basic number of hstore lines tags', 'SELECT sum(array_length(akeys(tags),1)) FROM planet_osm_line;', 11134), - ( 96, 'Basic number of hstore polygons tags', 'SELECT sum(array_length(akeys(tags),1)) FROM planet_osm_polygon;', 9541), + ( 95, 'Basic number of hstore lines tags', 'SELECT sum(array_length(akeys(tags),1)) FROM planet_osm_line;', 10387), + ( 96, 'Basic number of hstore polygons tags', 'SELECT sum(array_length(akeys(tags),1)) FROM planet_osm_polygon;', 9531), ( 97, 'Diff import number of hstore points tags', 'SELECT sum(array_length(akeys(tags),1)) FROM planet_osm_point;', 4352), - ( 98, 'Diff import number of hstore roads tags', 'SELECT sum(array_length(akeys(tags),1)) FROM planet_osm_roads;', 2341), - ( 99, 'Diff import number of hstore lines tags', 'SELECT sum(array_length(akeys(tags),1)) FROM planet_osm_line;', 11257), - ( 100, 'Diff import number of hstore polygons tags', 'SELECT sum(array_length(akeys(tags),1)) FROM planet_osm_polygon;', 9835), + ( 98, 'Diff import number of hstore roads tags', 'SELECT sum(array_length(akeys(tags),1)) FROM planet_osm_roads;', 2336), + ( 99, 'Diff import number of hstore lines tags', 'SELECT sum(array_length(akeys(tags),1)) FROM planet_osm_line;', 10505), + ( 100, 'Diff import number of hstore polygons tags', 'SELECT sum(array_length(akeys(tags),1)) FROM planet_osm_polygon;', 9825), #**** Tests to check if inner polygon appears when outer tags change after initially identicall inner and outer way tags in a multi-polygon **** #**** These tests are currently broken and noted in trac ticket #2853 **** ( 101, 'Multipolygon identical tags on inner and outer (presence of relation)', From e7e355bd22390530abb9fa35d25da719a3f37c69 Mon Sep 17 00:00:00 2001 From: Sarah Hoffmann Date: Tue, 14 Feb 2017 23:00:07 +0100 Subject: [PATCH 28/30] use size_t type for position --- wkb.hpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/wkb.hpp b/wkb.hpp index c775c4a39..f7082d96f 100644 --- a/wkb.hpp +++ b/wkb.hpp @@ -236,8 +236,8 @@ class parser_t explicit parser_t(char const *wkb) : m_wkb(wkb), m_pos(0) {} explicit parser_t(std::string const &wkb) : m_wkb(wkb.c_str()), m_pos(0) {} - unsigned save_pos() const { return m_pos; } - void rewind(unsigned pos) { m_pos = pos; } + size_t save_pos() const { return m_pos; } + void rewind(size_t pos) { m_pos = pos; } int read_header() { @@ -353,7 +353,7 @@ class parser_t } char const *m_wkb; - unsigned m_pos; + size_t m_pos; }; } // namespace From 5131a637f1cb0a3600465a2699cab28d94655ccb Mon Sep 17 00:00:00 2001 From: Sarah Hoffmann Date: Tue, 28 Feb 2017 22:12:57 +0100 Subject: [PATCH 29/30] check for bad endian setting when converting hex WKBs --- wkb.hpp | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/wkb.hpp b/wkb.hpp index f7082d96f..5c26f5018 100644 --- a/wkb.hpp +++ b/wkb.hpp @@ -24,6 +24,15 @@ enum geometry_type : uint32_t wkb_srid = 0x20000000 // SRID-presence flag (EWKB) }; +enum wkb_byte_order_type_t : uint8_t +{ +#if __BYTE_ORDER == __LITTLE_ENDIAN + Endian = 1 // Little Endian +#else + Endian = 0, // Big Endian +#endif +}; + /** * Writer for EWKB data suitable for postgres. * @@ -31,11 +40,6 @@ enum geometry_type : uint32_t */ class writer_t { - enum wkb_byte_order_type_t : uint8_t - { - XDR = 0, // Big Endian - NDR = 1 // Little Endian - }; std::string m_data; int m_srid; @@ -46,11 +50,7 @@ class writer_t size_t header(std::string &str, geometry_type type, bool add_length) const { -#if __BYTE_ORDER == __LITTLE_ENDIAN - str_push(str, NDR); -#else - str_push(str, XDR); -#endif + str_push(str, Endian); str_push(str, type | wkb_srid); str_push(str, m_srid); @@ -230,6 +230,10 @@ class parser_t } } + if (out[0] != Endian) + throw std::runtime_error( + "Endian setting of database WKB not supported."); + return out; } From b2a2d584b344fbbcf03a0ca2cec526add22fcfc5 Mon Sep 17 00:00:00 2001 From: Sarah Hoffmann Date: Wed, 1 Mar 2017 20:18:05 +0100 Subject: [PATCH 30/30] improve bad endian exception message --- wkb.hpp | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/wkb.hpp b/wkb.hpp index 5c26f5018..64896c9af 100644 --- a/wkb.hpp +++ b/wkb.hpp @@ -232,7 +232,13 @@ class parser_t if (out[0] != Endian) throw std::runtime_error( - "Endian setting of database WKB not supported."); +#if __BYTE_ORDER == __LITTLE_ENDIAN + "Geometries in the database are returned in big-endian byte order. " +#else + "Geometries in the database are returned in little-endian byte order. " +#endif + "osm2pgsql can only process geometries in native byte order." + ); return out; }