diff --git a/.travis.yml b/.travis.yml index c5070e2d3..863ddae8f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,8 +8,6 @@ addons: packages: - g++-4.8 - libexpat1-dev - - libgeos-dev - - libgeos++-dev - libpq-dev - libbz2-dev - libproj-dev @@ -44,7 +42,6 @@ install: before_script: - $CXX --version - xml2-config --version - - geos-config --version - proj | head -n1 - lua -v script: diff --git a/CMakeLists.txt b/CMakeLists.txt index ae96f4ddd..e984054fd 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -92,7 +92,7 @@ endif() include_directories(${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_BINARY_DIR}) -find_package(Osmium REQUIRED COMPONENTS io geos proj) +find_package(Osmium REQUIRED COMPONENTS io proj) include_directories(SYSTEM ${OSMIUM_INCLUDE_DIRS}) if (WITH_LUA) @@ -167,7 +167,6 @@ endif() set(osm2pgsql_lib_SOURCES expire-tiles.cpp - geometry-builder.cpp geometry-processor.cpp id-tracker.cpp middle-pgsql.cpp @@ -177,6 +176,7 @@ set(osm2pgsql_lib_SOURCES node-ram-cache.cpp options.cpp osmdata.cpp + osmium-builder.cpp output-gazetteer.cpp output-multi.cpp output-null.cpp @@ -195,7 +195,6 @@ set(osm2pgsql_lib_SOURCES util.cpp wildcmp.cpp expire-tiles.hpp - geometry-builder.hpp geometry-processor.hpp id-tracker.hpp middle-pgsql.hpp @@ -205,6 +204,7 @@ set(osm2pgsql_lib_SOURCES node-ram-cache.hpp options.hpp osmdata.hpp + osmium-builder.hpp osmtypes.hpp output-gazetteer.hpp output-multi.hpp @@ -224,6 +224,7 @@ set(osm2pgsql_lib_SOURCES tagtransform.hpp util.hpp wildcmp.hpp + wkb.hpp ) add_library(osm2pgsql_lib STATIC ${osm2pgsql_lib_SOURCES}) diff --git a/README.md b/README.md index f2b04f8db..8c74b3282 100644 --- a/README.md +++ b/README.md @@ -35,7 +35,6 @@ to configure and build itself and requires Required libraries are * [expat](http://www.libexpat.org/) -* [geos](http://trac.osgeo.org/geos) * [proj](http://proj.osgeo.org/) * [bzip2](http://www.bzip.org/) * [zlib](http://www.zlib.net/) @@ -57,15 +56,14 @@ On a Debian or Ubuntu system, this can be done with: ```sh sudo apt-get install make cmake g++ libboost-dev libboost-system-dev \ libboost-filesystem-dev libexpat1-dev zlib1g-dev \ - libbz2-dev libpq-dev libgeos-dev libgeos++-dev libproj-dev lua5.2 \ - liblua5.2-dev + libbz2-dev libpq-dev libproj-dev lua5.2 liblua5.2-dev ``` On a Fedora system, use ```sh sudo yum install cmake gcc-c++ boost-devel expat-devel zlib-devel bzip2-devel \ - postgresql-devel geos-devel proj-devel proj-epsg lua-devel + postgresql-devel proj-devel proj-epsg lua-devel ``` On RedHat / CentOS first run `sudo yum install epel-release` then install @@ -75,7 +73,7 @@ On a FreeBSD system, use ```sh pkg install devel/cmake devel/boost-libs textproc/expat2 \ - databases/postgresql94-client graphics/geos graphics/proj lang/lua52 + databases/postgresql94-client graphics/proj lang/lua52 ``` Once dependencies are installed, use CMake to build the Makefiles in a separate folder @@ -110,7 +108,7 @@ cmake .. -G "Unix Makefiles" -DCMAKE_BUILD_TYPE=Debug -DBUILD_TESTS=ON ## Usage ## -Osm2pgsql has one program, the executable itself, which has **44** command line +Osm2pgsql has one program, the executable itself, which has **43** command line options. Before loading into a database, the database must be created and the PostGIS diff --git a/appveyor.yml b/appveyor.yml index 60f64e7a0..55be325f8 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -47,7 +47,6 @@ build_script: - copy /y ..\*.style osm2pgsql-bin - copy /y ..\*.lua osm2pgsql-bin - copy /y %PREFIX%\bin\lua.dll osm2pgsql-bin - - copy /y %PREFIX%\bin\geos.dll osm2pgsql-bin - copy /y "%PSQL_ROOT%\bin\libpq.dll" osm2pgsql-bin - copy /y "%PSQL_ROOT%\bin\libintl-8.dll" osm2pgsql-bin - copy /y "%PSQL_ROOT%\bin\libeay32.dll" osm2pgsql-bin diff --git a/contrib/libosmium/README.contrib b/contrib/libosmium/README.contrib index a97e3229d..98e88d6cd 100644 --- a/contrib/libosmium/README.contrib +++ b/contrib/libosmium/README.contrib @@ -1,2 +1,2 @@ Source: https://github.com/osmcode/libosmium -Revision: v2.11.0 +Revision: ab8236405330607932c8f6395956ec6d9bcdee5c diff --git a/contrib/libosmium/osmium/area/assembler.hpp b/contrib/libosmium/osmium/area/assembler.hpp index 3509b57bc..8ec793579 100644 --- a/contrib/libosmium/osmium/area/assembler.hpp +++ b/contrib/libosmium/osmium/area/assembler.hpp @@ -38,253 +38,50 @@ DEALINGS IN THE SOFTWARE. #include #include #include +#include #include #include -#include #include #include #include -#include -#include #include #include +#include #include #include #include #include #include -#include #include #include #include #include #include #include -#include -#include - -#include -#include -#include -#include -#include namespace osmium { namespace area { - /** - * Configuration for osmium::area::Assembler objects. Create this - * once, set the options you want and then re-use it every time you - * create an Assembler object. - */ - struct AssemblerConfig { - - /** - * Optional pointer to problem reporter. - */ - osmium::area::ProblemReporter* problem_reporter = nullptr; - - /** - * Debug level. If this is greater than zero, debug messages will - * be printed to stderr. Available levels are 1 to 3. Note that - * level 2 and above will generate a lot of messages! - */ - int debug_level = 0; - - /** - * The roles of multipolygon members are ignored when assembling - * multipolygons, because they are often missing or wrong. If this - * is set, the roles are checked after the multipolygons are built - * against what the assembly process decided where the inner and - * outer rings are. This slows down the processing, so it only - * makes sense if you want to get the problem reports. - */ - bool check_roles = false; - - /** - * When the assembler can't create an area, usually because its - * geometry would be invalid, it will create an "empty" area object - * without rings. This allows you to detect where an area was - * invalid. - * - * If this is set to false, invalid areas will simply be discarded. - */ - bool create_empty_areas = true; - - /** - * Create areas for (multi)polygons where the tags are on the - * relation. - * - * If this is set to false, those areas will simply be discarded. - */ - bool create_new_style_polygons = true; - - /** - * Create areas for (multi)polygons where the tags are on the - * outer way(s). - * - * If this is set to false, those areas will simply be discarded. - */ - bool create_old_style_polygons = true; - - /** - * Create areas for polygons created from ways. - * - * If this is set to false, those areas will simply be discarded. - */ - bool create_way_polygons = true; - - /** - * Keep the type tag from multipolygon relations on the area - * object. By default this is false, and the type tag will be - * removed. - */ - bool keep_type_tag = false; - - AssemblerConfig() noexcept = default; - - /** - * Constructor - * @deprecated Use default constructor and set values afterwards. - */ - explicit AssemblerConfig(osmium::area::ProblemReporter* pr, bool d = false) : - problem_reporter(pr), - debug_level(d) { - } - - /** - * Enable or disable debug output to stderr. This is for Osmium - * developers only. - * - * @deprecated Set debug_level directly. - */ - OSMIUM_DEPRECATED void enable_debug_output(bool d = true) { - debug_level = d; - } - - }; // struct AssemblerConfig - - namespace detail { - - using open_ring_its_type = std::list::iterator>; - - struct location_to_ring_map { - osmium::Location location; - open_ring_its_type::iterator ring_it; - bool start; - - location_to_ring_map(const osmium::Location& l, const open_ring_its_type::iterator& r, bool s) noexcept : - location(l), - ring_it(r), - start(s) { - } - - explicit location_to_ring_map(const osmium::Location& l) noexcept : - location(l), - ring_it(), - start(false) { - } - - const detail::ProtoRing& ring() const noexcept { - return **ring_it; - } - - }; // struct location_to_ring_map - - inline bool operator==(const location_to_ring_map& lhs, const location_to_ring_map& rhs) noexcept { - return lhs.location == rhs.location; - } - - inline bool operator<(const location_to_ring_map& lhs, const location_to_ring_map& rhs) noexcept { - return lhs.location < rhs.location; - } - - } // namespace detail - /** * Assembles area objects from closed ways or multipolygon relations * and their members. */ - class Assembler { - - using open_ring_its_type = detail::open_ring_its_type; - using location_to_ring_map = detail::location_to_ring_map; - - struct slocation { - - static constexpr const uint32_t invalid_item = 1 << 30; - - uint32_t item : 31; - uint32_t reverse : 1; - - slocation() noexcept : - item(invalid_item), - reverse(false) { - } - - explicit slocation(uint32_t n, bool r = false) noexcept : - item(n), - reverse(r) { - } - - osmium::Location location(const detail::SegmentList& segment_list) const noexcept { - const auto& segment = segment_list[item]; - return reverse ? segment.second().location() : segment.first().location(); - } - - const osmium::NodeRef& node_ref(const detail::SegmentList& segment_list) const noexcept { - const auto& segment = segment_list[item]; - return reverse ? segment.second() : segment.first(); - } - - osmium::Location location(const detail::SegmentList& segment_list, const osmium::Location& default_location) const noexcept { - if (item == invalid_item) { - return default_location; - } - return location(segment_list); - } - - }; // struct slocation - - // Configuration settings for this Assembler - const AssemblerConfig& m_config; - - // List of segments (connection between two nodes) - osmium::area::detail::SegmentList m_segment_list; - - // The rings we are building from the segments - std::list m_rings; - - // All node locations - std::vector m_locations; - - // All locations where more than two segments start/end - std::vector m_split_locations; - - // Statistics - area_stats m_stats; - - // The number of members the multipolygon relation has - size_t m_num_members = 0; - - bool debug() const noexcept { - return m_config.debug_level > 1; - } + class Assembler : public detail::BasicAssembler { bool report_ways() const noexcept { - if (!m_config.problem_reporter) { + if (!config().problem_reporter) { return false; } - return m_stats.duplicate_nodes || - m_stats.duplicate_segments || - m_stats.intersections || - m_stats.open_rings || - m_stats.short_ways || - m_stats.touching_rings || - m_stats.ways_in_multiple_rings || - m_stats.wrong_role; + return stats().duplicate_nodes || + stats().duplicate_segments || + stats().intersections || + stats().open_rings || + stats().short_ways || + stats().touching_rings || + stats().ways_in_multiple_rings || + stats().wrong_role; } void add_tags_to_area(osmium::builder::AreaBuilder& builder, const osmium::Way& way) const { @@ -295,7 +92,7 @@ namespace osmium { std::map counter; for (const osmium::Way* way : ways) { for (const auto& tag : way->tags()) { - std::string kv {tag.key()}; + std::string kv{tag.key()}; kv.append(1, '\0'); kv.append(tag.value()); ++counter[kv]; @@ -342,7 +139,7 @@ namespace osmium { } void add_tags_to_area(osmium::builder::AreaBuilder& builder, const osmium::Relation& relation) { - const auto count = std::count_if(relation.tags().cbegin(), relation.tags().cend(), filter()); + const auto count = std::count_if(relation.tags().cbegin(), relation.tags().cend(), std::cref(filter())); if (debug()) { std::cerr << " found " << count << " tags on relation (without ignored ones)\n"; @@ -353,18 +150,18 @@ namespace osmium { std::cerr << " use tags from relation\n"; } - if (m_config.keep_type_tag) { + if (config().keep_type_tag) { builder.add_item(relation.tags()); } else { copy_tags_without_type(builder, relation.tags()); } } else { - ++m_stats.no_tags_on_relation; + ++stats().no_tags_on_relation; if (debug()) { std::cerr << " use tags from outer ways\n"; } std::set ways; - for (const auto& ring : m_rings) { + for (const auto& ring : rings()) { if (ring.is_outer()) { ring.get_ways(ways); } @@ -384,989 +181,12 @@ namespace osmium { } } - template - static void build_ring_from_proto_ring(osmium::builder::AreaBuilder& builder, const detail::ProtoRing& ring) { - TBuilder ring_builder{builder}; - ring_builder.add_node_ref(ring.get_node_ref_start()); - for (const auto& segment : ring.segments()) { - ring_builder.add_node_ref(segment->stop()); - } - } - - /** - * Append each outer ring together with its inner rings to the - * area in the buffer. - */ - void add_rings_to_area(osmium::builder::AreaBuilder& builder) const { - for (const detail::ProtoRing& ring : m_rings) { - if (ring.is_outer()) { - build_ring_from_proto_ring(builder, ring); - for (const detail::ProtoRing* inner : ring.inner_rings()) { - build_ring_from_proto_ring(builder, *inner); - } - } - } - } - - void check_inner_outer_roles() { - if (debug()) { - std::cerr << " Checking inner/outer roles\n"; - } - - std::unordered_map way_rings; - std::unordered_set ways_in_multiple_rings; - - for (const detail::ProtoRing& ring : m_rings) { - for (const auto& segment : ring.segments()) { - assert(segment->way()); - - if (!segment->role_empty() && (ring.is_outer() ? !segment->role_outer() : !segment->role_inner())) { - ++m_stats.wrong_role; - if (debug()) { - std::cerr << " Segment " << *segment << " from way " << segment->way()->id() << " has role '" << segment->role_name() - << "', but should have role '" << (ring.is_outer() ? "outer" : "inner") << "'\n"; - } - if (m_config.problem_reporter) { - if (ring.is_outer()) { - m_config.problem_reporter->report_role_should_be_outer(segment->way()->id(), segment->first().location(), segment->second().location()); - } else { - m_config.problem_reporter->report_role_should_be_inner(segment->way()->id(), segment->first().location(), segment->second().location()); - } - } - } - - auto& r = way_rings[segment->way()]; - if (!r) { - r = ˚ - } else if (r != &ring) { - ways_in_multiple_rings.insert(segment->way()); - } - - } - } - - for (const osmium::Way* way : ways_in_multiple_rings) { - ++m_stats.ways_in_multiple_rings; - if (debug()) { - std::cerr << " Way " << way->id() << " is in multiple rings\n"; - } - if (m_config.problem_reporter) { - m_config.problem_reporter->report_way_in_multiple_rings(*way); - } - } - - } - - detail::NodeRefSegment* get_next_segment(const osmium::Location& location) { - auto it = std::lower_bound(m_locations.begin(), m_locations.end(), slocation{}, [this, &location](const slocation& lhs, const slocation& rhs) { - return lhs.location(m_segment_list, location) < rhs.location(m_segment_list, location); - }); - - assert(it != m_locations.end()); - if (m_segment_list[it->item].is_done()) { - ++it; - } - assert(it != m_locations.end()); - - assert(!m_segment_list[it->item].is_done()); - return &m_segment_list[it->item]; - } - - class rings_stack_element { - - int32_t m_y; - detail::ProtoRing* m_ring_ptr; - - public: - - rings_stack_element(int32_t y, detail::ProtoRing* ring_ptr) : - m_y(y), - m_ring_ptr(ring_ptr) { - } - - int32_t y() const noexcept { - return m_y; - } - - const detail::ProtoRing& ring() const noexcept { - return *m_ring_ptr; - } - - detail::ProtoRing* ring_ptr() noexcept { - return m_ring_ptr; - } - - bool operator==(const rings_stack_element& rhs) const noexcept { - return m_ring_ptr == rhs.m_ring_ptr; - } - - bool operator<(const rings_stack_element& rhs) const noexcept { - return m_y < rhs.m_y; - } - - }; // class ring_stack_element - - using rings_stack = std::vector; - - void remove_duplicates(rings_stack& outer_rings) { - while (true) { - const auto it = std::adjacent_find(outer_rings.begin(), outer_rings.end()); - if (it == outer_rings.end()) { - return; - } - outer_rings.erase(it, std::next(it, 2)); - } - } - - detail::ProtoRing* find_enclosing_ring(detail::NodeRefSegment* segment) { - if (debug()) { - std::cerr << " Looking for ring enclosing " << *segment << "\n"; - } - - const auto location = segment->first().location(); - const auto end_location = segment->second().location(); - - while (segment->first().location() == location) { - if (segment == &m_segment_list.back()) { - break; - } - ++segment; - } - - int nesting = 0; - - rings_stack outer_rings; - while (segment >= &m_segment_list.front()) { - if (!segment->is_direction_done()) { - --segment; - continue; - } - if (debug()) { - std::cerr << " Checking against " << *segment << "\n"; - } - const osmium::Location& a = segment->first().location(); - const osmium::Location& b = segment->second().location(); - - if (segment->first().location() == location) { - const int64_t ax = a.x(); - const int64_t bx = b.x(); - const int64_t lx = end_location.x(); - const int64_t ay = a.y(); - const int64_t by = b.y(); - const int64_t ly = end_location.y(); - const auto z = (bx - ax)*(ly - ay) - (by - ay)*(lx - ax); - if (debug()) { - std::cerr << " Segment XXXX z=" << z << "\n"; - } - if (z > 0) { - nesting += segment->is_reverse() ? -1 : 1; - if (debug()) { - std::cerr << " Segment is below (nesting=" << nesting << ")\n"; - } - if (segment->ring()->is_outer()) { - if (debug()) { - std::cerr << " Segment belongs to outer ring\n"; - } - outer_rings.emplace_back(a.y(), segment->ring()); - } - } - } else if (a.x() <= location.x() && location.x() < b.x()) { - if (debug()) { - std::cerr << " Is in x range\n"; - } - - const int64_t ax = a.x(); - const int64_t bx = b.x(); - const int64_t lx = location.x(); - const int64_t ay = a.y(); - const int64_t by = b.y(); - const int64_t ly = location.y(); - const auto z = (bx - ax)*(ly - ay) - (by - ay)*(lx - ax); - - if (z >= 0) { - nesting += segment->is_reverse() ? -1 : 1; - if (debug()) { - std::cerr << " Segment is below (nesting=" << nesting << ")\n"; - } - if (segment->ring()->is_outer()) { - if (debug()) { - std::cerr << " Segment belongs to outer ring\n"; - } - const int32_t y = int32_t(ay + (by - ay) * (lx - ax) / (bx - ax)); - outer_rings.emplace_back(y, segment->ring()); - } - } - } - --segment; - } - - if (nesting % 2 == 0) { - if (debug()) { - std::cerr << " Decided that this is an outer ring\n"; - } - return nullptr; - } else { - if (debug()) { - std::cerr << " Decided that this is an inner ring\n"; - } - assert(!outer_rings.empty()); - - std::sort(outer_rings.rbegin(), outer_rings.rend()); - if (debug()) { - for (const auto& o : outer_rings) { - std::cerr << " y=" << o.y() << " " << o.ring() << "\n"; - } - } - - remove_duplicates(outer_rings); - if (debug()) { - std::cerr << " after remove duplicates:\n"; - for (const auto& o : outer_rings) { - std::cerr << " y=" << o.y() << " " << o.ring() << "\n"; - } - } - - assert(!outer_rings.empty()); - return outer_rings.front().ring_ptr(); - } - } - - bool is_split_location(const osmium::Location& location) const noexcept { - return std::find(m_split_locations.cbegin(), m_split_locations.cend(), location) != m_split_locations.cend(); - } - - uint32_t add_new_ring(slocation& node) { - detail::NodeRefSegment* segment = &m_segment_list[node.item]; - assert(!segment->is_done()); - - if (debug()) { - std::cerr << " Starting new ring at location " << node.location(m_segment_list) << " with segment " << *segment << "\n"; - } - - if (node.reverse) { - segment->reverse(); - } - - detail::ProtoRing* outer_ring = nullptr; - - if (segment != &m_segment_list.front()) { - outer_ring = find_enclosing_ring(segment); - } - segment->mark_direction_done(); - - m_rings.emplace_back(segment); - detail::ProtoRing* ring = &m_rings.back(); - if (outer_ring) { - if (debug()) { - std::cerr << " This is an inner ring. Outer ring is " << *outer_ring << "\n"; - } - outer_ring->add_inner_ring(ring); - ring->set_outer_ring(outer_ring); - } else if (debug()) { - std::cerr << " This is an outer ring\n"; - } - - const osmium::Location& first_location = node.location(m_segment_list); - osmium::Location last_location = segment->stop().location(); - - uint32_t nodes = 1; - while (first_location != last_location) { - ++nodes; - detail::NodeRefSegment* next_segment = get_next_segment(last_location); - next_segment->mark_direction_done(); - if (next_segment->start().location() != last_location) { - next_segment->reverse(); - } - ring->add_segment_back(next_segment); - if (debug()) { - std::cerr << " Next segment is " << *next_segment << "\n"; - } - last_location = next_segment->stop().location(); - } - - ring->fix_direction(); - - if (debug()) { - std::cerr << " Completed ring: " << *ring << "\n"; - } - - return nodes; - } - - uint32_t add_new_ring_complex(slocation& node) { - detail::NodeRefSegment* segment = &m_segment_list[node.item]; - assert(!segment->is_done()); - - if (debug()) { - std::cerr << " Starting new ring at location " << node.location(m_segment_list) << " with segment " << *segment << "\n"; - } - - if (node.reverse) { - segment->reverse(); - } - - m_rings.emplace_back(segment); - detail::ProtoRing* ring = &m_rings.back(); - - const osmium::Location& first_location = node.location(m_segment_list); - osmium::Location last_location = segment->stop().location(); - - uint32_t nodes = 1; - while (first_location != last_location && !is_split_location(last_location)) { - ++nodes; - detail::NodeRefSegment* next_segment = get_next_segment(last_location); - if (next_segment->start().location() != last_location) { - next_segment->reverse(); - } - ring->add_segment_back(next_segment); - if (debug()) { - std::cerr << " Next segment is " << *next_segment << "\n"; - } - last_location = next_segment->stop().location(); - } - - if (debug()) { - if (first_location == last_location) { - std::cerr << " Completed ring: " << *ring << "\n"; - } else { - std::cerr << " Completed partial ring: " << *ring << "\n"; - } - } - - return nodes; - } - - void create_locations_list() { - m_locations.reserve(m_segment_list.size() * 2); - - for (uint32_t n = 0; n < m_segment_list.size(); ++n) { - m_locations.emplace_back(n, false); - m_locations.emplace_back(n, true); - } - - std::stable_sort(m_locations.begin(), m_locations.end(), [this](const slocation& lhs, const slocation& rhs) { - return lhs.location(m_segment_list) < rhs.location(m_segment_list); - }); - } - - void find_inner_outer_complex(detail::ProtoRing* ring) { - detail::ProtoRing* outer_ring = find_enclosing_ring(ring->min_segment()); - if (outer_ring) { - outer_ring->add_inner_ring(ring); - ring->set_outer_ring(outer_ring); - } - ring->fix_direction(); - ring->mark_direction_done(); - } - - void find_inner_outer_complex() { - if (debug()) { - std::cerr << " Finding inner/outer rings\n"; - } - std::vector rings; - rings.reserve(m_rings.size()); - for (auto& ring : m_rings) { - if (ring.closed()) { - rings.push_back(&ring); - } - } - - if (rings.empty()) { - return; - } - - std::sort(rings.begin(), rings.end(), [](detail::ProtoRing* a, detail::ProtoRing* b) { - return a->min_segment() < b->min_segment(); - }); - - rings.front()->fix_direction(); - rings.front()->mark_direction_done(); - if (debug()) { - std::cerr << " First ring is outer: " << *rings.front() << "\n"; - } - for (auto it = std::next(rings.begin()); it != rings.end(); ++it) { - if (debug()) { - std::cerr << " Checking (at min segment " << *((*it)->min_segment()) << ") ring " << **it << "\n"; - } - find_inner_outer_complex(*it); - if (debug()) { - std::cerr << " Ring is " << ((*it)->is_outer() ? "OUTER: " : "INNER: ") << **it << "\n"; - } - } - } - - /** - * Finds all locations where more than two segments meet. If there - * are any open rings found along the way, they are reported - * and the function returns false. - */ - bool find_split_locations() { - osmium::Location previous_location; - for (auto it = m_locations.cbegin(); it != m_locations.cend(); ++it) { - const osmium::NodeRef& nr = it->node_ref(m_segment_list); - const osmium::Location& loc = nr.location(); - if (std::next(it) == m_locations.cend() || loc != std::next(it)->location(m_segment_list)) { - if (debug()) { - std::cerr << " Found open ring at " << nr << "\n"; - } - if (m_config.problem_reporter) { - const auto& segment = m_segment_list[it->item]; - m_config.problem_reporter->report_ring_not_closed(nr, segment.way()); - } - ++m_stats.open_rings; - } else { - if (loc == previous_location && (m_split_locations.empty() || m_split_locations.back() != previous_location )) { - m_split_locations.push_back(previous_location); - } - ++it; - if (it == m_locations.end()) { - break; - } - } - previous_location = loc; - } - return m_stats.open_rings == 0; - } - - void create_rings_simple_case() { - auto count_remaining = m_segment_list.size(); - for (slocation& sl : m_locations) { - const detail::NodeRefSegment& segment = m_segment_list[sl.item]; - if (!segment.is_done()) { - count_remaining -= add_new_ring(sl); - if (count_remaining == 0) { - return; - } - } - } - } - - std::vector create_location_to_ring_map(open_ring_its_type& open_ring_its) { - std::vector xrings; - xrings.reserve(open_ring_its.size() * 2); - - for (auto it = open_ring_its.begin(); it != open_ring_its.end(); ++it) { - if (debug()) { - std::cerr << " Ring: " << **it << "\n"; - } - xrings.emplace_back((*it)->get_node_ref_start().location(), it, true); - xrings.emplace_back((*it)->get_node_ref_stop().location(), it, false); - } - - std::sort(xrings.begin(), xrings.end()); - - return xrings; - } - - void merge_two_rings(open_ring_its_type& open_ring_its, const location_to_ring_map& m1, const location_to_ring_map& m2) { - auto& r1 = *m1.ring_it; - auto& r2 = *m2.ring_it; - - if (r1->get_node_ref_stop().location() == r2->get_node_ref_start().location()) { - r1->join_forward(*r2); - } else if (r1->get_node_ref_stop().location() == r2->get_node_ref_stop().location()) { - r1->join_backward(*r2); - } else if (r1->get_node_ref_start().location() == r2->get_node_ref_start().location()) { - r1->reverse(); - r1->join_forward(*r2); - } else if (r1->get_node_ref_start().location() == r2->get_node_ref_stop().location()) { - r1->reverse(); - r1->join_backward(*r2); - } else { - assert(false); - } - - m_rings.erase(r2); - open_ring_its.remove(r2); - - if (r1->closed()) { - open_ring_its.remove(r1); - } - } - - bool try_to_merge(open_ring_its_type& open_ring_its) { - if (open_ring_its.empty()) { - return false; - } - - if (debug()) { - std::cerr << " Trying to merge " << open_ring_its.size() << " open rings\n"; - } - - std::vector xrings = create_location_to_ring_map(open_ring_its); - - auto it = xrings.cbegin(); - while (it != xrings.cend()) { - it = std::adjacent_find(it, xrings.cend()); - if (it == xrings.cend()) { - return false; - } - auto after = std::next(it, 2); - if (after == xrings.cend() || after->location != it->location) { - if (debug()) { - std::cerr << " Merging two rings\n"; - } - merge_two_rings(open_ring_its, *it, *std::next(it)); - return true; - } - while (it != xrings.cend() && it->location == after->location) { - ++it; - } - } - - return false; - } - - bool there_are_open_rings() const noexcept { - return std::any_of(m_rings.cbegin(), m_rings.cend(), [](const detail::ProtoRing& ring){ - return !ring.closed(); - }); - } - - struct candidate { - int64_t sum; - std::vector> rings; - osmium::Location start_location; - osmium::Location stop_location; - - explicit candidate(location_to_ring_map& ring, bool reverse) : - sum(ring.ring().sum()), - rings(), - start_location(ring.ring().get_node_ref_start().location()), - stop_location(ring.ring().get_node_ref_stop().location()) { - rings.emplace_back(ring, reverse); - } - - bool closed() const noexcept { - return start_location == stop_location; - } - - }; - - void find_candidates(std::vector& candidates, std::unordered_set& loc_done, const std::vector& xrings, candidate& cand) { - if (debug()) { - std::cerr << " find_candidates sum=" << cand.sum << " start=" << cand.start_location << " stop=" << cand.stop_location << "\n"; - for (const auto& ring : cand.rings) { - std::cerr << " " << ring.first.ring() << (ring.second ? " reverse" : "") << "\n"; - } - } - - const auto connections = make_range(std::equal_range(xrings.cbegin(), - xrings.cend(), - location_to_ring_map{cand.stop_location})); - - assert(connections.begin() != connections.end()); - - assert(!cand.rings.empty()); - const detail::ProtoRing* ring_leading_here = &cand.rings.back().first.ring(); - for (const location_to_ring_map& m : connections) { - const detail::ProtoRing& ring = m.ring(); - - if (&ring != ring_leading_here) { - if (debug()) { - std::cerr << " next possible connection: " << ring << (m.start ? "" : " reverse") << "\n"; - } - - candidate c = cand; - if (m.start) { - c.rings.emplace_back(m, false); - c.stop_location = ring.get_node_ref_stop().location(); - c.sum += ring.sum(); - } else { - c.rings.emplace_back(m, true); - c.stop_location = ring.get_node_ref_start().location(); - c.sum -= ring.sum(); - } - if (c.closed()) { - if (debug()) { - std::cerr << " found candidate\n"; - } - candidates.push_back(c); - } else if (loc_done.count(c.stop_location) == 0) { - if (debug()) { - std::cerr << " recurse...\n"; - } - loc_done.insert(c.stop_location); - find_candidates(candidates, loc_done, xrings, c); - if (debug()) { - std::cerr << " ...back\n"; - } - } else if (debug()) { - std::cerr << " loop found\n"; - } - } - } - } - - /** - * If there are multiple open rings and mltiple ways to join them, - * this function is called. It will take the first open ring and - * try recursively all ways of closing it. Of all the candidates - * the one with the smallest area is chosen and closed. If it - * can't close this ring, an error is reported and the function - * returns false. - */ - bool join_connected_rings(open_ring_its_type& open_ring_its) { - assert(!open_ring_its.empty()); - - if (debug()) { - std::cerr << " Trying to merge " << open_ring_its.size() << " open rings\n"; - } - - std::vector xrings = create_location_to_ring_map(open_ring_its); - - const auto ring_min = std::min_element(xrings.begin(), xrings.end(), [](const location_to_ring_map& lhs, const location_to_ring_map& rhs) { - return lhs.ring().min_segment() < rhs.ring().min_segment(); - }); - - find_inner_outer_complex(); - detail::ProtoRing* outer_ring = find_enclosing_ring(ring_min->ring().min_segment()); - bool ring_min_is_outer = !outer_ring; - if (debug()) { - std::cerr << " Open ring is " << (ring_min_is_outer ? "outer" : "inner") << " ring\n"; - } - for (auto& ring : m_rings) { - ring.reset(); - } - - candidate cand{*ring_min, false}; - - // Locations we have visited while finding candidates, used - // to detect loops. - std::unordered_set loc_done; - - loc_done.insert(cand.stop_location); - - std::vector candidates; - find_candidates(candidates, loc_done, xrings, cand); - - if (candidates.empty()) { - if (debug()) { - std::cerr << " Found no candidates\n"; - } - if (!open_ring_its.empty()) { - ++m_stats.open_rings; - if (m_config.problem_reporter) { - for (auto& it : open_ring_its) { - m_config.problem_reporter->report_ring_not_closed(it->get_node_ref_start()); - m_config.problem_reporter->report_ring_not_closed(it->get_node_ref_stop()); - } - } - } - return false; - } - - if (debug()) { - std::cerr << " Found candidates:\n"; - for (const auto& cand : candidates) { - std::cerr << " sum=" << cand.sum << "\n"; - for (const auto& ring : cand.rings) { - std::cerr << " " << ring.first.ring() << (ring.second ? " reverse" : "") << "\n"; - } - } - } - - // Find the candidate with the smallest/largest area - const auto chosen_cand = ring_min_is_outer ? - std::min_element(candidates.cbegin(), candidates.cend(), [](const candidate& lhs, const candidate& rhs) { - return std::abs(lhs.sum) < std::abs(rhs.sum); - }) : - std::max_element(candidates.cbegin(), candidates.cend(), [](const candidate& lhs, const candidate& rhs) { - return std::abs(lhs.sum) < std::abs(rhs.sum); - }); - - if (debug()) { - std::cerr << " Decided on: sum=" << chosen_cand->sum << "\n"; - for (const auto& ring : chosen_cand->rings) { - std::cerr << " " << ring.first.ring() << (ring.second ? " reverse" : "") << "\n"; - } - } - - // Join all (open) rings in the candidate to get one closed ring. - assert(chosen_cand->rings.size() > 1); - const auto& first_ring = chosen_cand->rings.front().first; - for (auto it = chosen_cand->rings.begin() + 1; it != chosen_cand->rings.end(); ++it) { - merge_two_rings(open_ring_its, first_ring, it->first); - } - - if (debug()) { - std::cerr << " Merged to " << first_ring.ring() << "\n"; - } - - return true; - } - - bool create_rings_complex_case() { - // First create all the (partial) rings starting at the split locations - auto count_remaining = m_segment_list.size(); - for (const osmium::Location& location : m_split_locations) { - const auto locs = make_range(std::equal_range(m_locations.begin(), - m_locations.end(), - slocation{}, - [this, &location](const slocation& lhs, const slocation& rhs) { - return lhs.location(m_segment_list, location) < rhs.location(m_segment_list, location); - })); - for (auto& loc : locs) { - if (!m_segment_list[loc.item].is_done()) { - count_remaining -= add_new_ring_complex(loc); - if (count_remaining == 0) { - break; - } - } - } - } - - // Now find all the rest of the rings (ie not starting at split locations) - if (count_remaining > 0) { - for (slocation& sl : m_locations) { - const detail::NodeRefSegment& segment = m_segment_list[sl.item]; - if (!segment.is_done()) { - count_remaining -= add_new_ring_complex(sl); - if (count_remaining == 0) { - break; - } - } - } - } - - // Now all segments are in exactly one (partial) ring. - - // If there are open rings, try to join them to create closed - // rings. - if (there_are_open_rings()) { - ++m_stats.area_really_complex_case; - - open_ring_its_type open_ring_its; - for (auto it = m_rings.begin(); it != m_rings.end(); ++it) { - if (!it->closed()) { - open_ring_its.push_back(it); - } - } - - while (!open_ring_its.empty()) { - if (debug()) { - std::cerr << " There are " << open_ring_its.size() << " open rings\n"; - } - while (try_to_merge(open_ring_its)); - - if (!open_ring_its.empty()) { - if (debug()) { - std::cerr << " After joining obvious cases there are still " << open_ring_its.size() << " open rings\n"; - } - if (!join_connected_rings(open_ring_its)) { - return false; - } - } - } - - if (debug()) { - std::cerr << " Joined all open rings\n"; - } - } - - // Now all rings are complete. - - find_inner_outer_complex(); - - return true; - } - - /** - * Checks if any ways were completely removed in the - * erase_duplicate_segments step. - */ - bool ways_were_lost() { - std::unordered_set ways_in_segments; - - for (const auto& segment : m_segment_list) { - ways_in_segments.insert(segment.way()); - } - - return ways_in_segments.size() < m_num_members; - } - - /** - * Create rings from segments. - */ - bool create_rings() { - m_stats.nodes += m_segment_list.size(); - - // Sort the list of segments (from left to right and bottom - // to top). - osmium::Timer timer_sort; - m_segment_list.sort(); - timer_sort.stop(); - - // Remove duplicate segments. Removal is in pairs, so if there - // are two identical segments, they will both be removed. If - // there are three, two will be removed and one remains. - osmium::Timer timer_dupl; - m_stats.duplicate_segments = m_segment_list.erase_duplicate_segments(m_config.problem_reporter); - timer_dupl.stop(); - - // If there are no segments left at this point, this isn't - // a valid area. - if (m_segment_list.empty()) { - if (debug()) { - std::cerr << " No segments left\n"; - } - return false; - } - - // If one or more complete ways was removed because of - // duplicate segments, this isn't a valid area. - if (ways_were_lost()) { - if (debug()) { - std::cerr << " Complete ways removed because of duplicate segments\n"; - } - return false; - } - - if (m_config.debug_level >= 3) { - std::cerr << "Sorted de-duplicated segment list:\n"; - for (const auto& s : m_segment_list) { - std::cerr << " " << s << "\n"; - } - } - - // Now we look for segments crossing each other. If there are - // any, the multipolygon is invalid. - // In the future this could be improved by trying to fix those - // cases. - osmium::Timer timer_intersection; - m_stats.intersections = m_segment_list.find_intersections(m_config.problem_reporter); - timer_intersection.stop(); - - if (m_stats.intersections) { - return false; - } - - // This creates an ordered list of locations of both endpoints - // of all segments with pointers back to the segments. We will - // use this list later to quickly find which segment(s) fits - // onto a known segment. - osmium::Timer timer_locations_list; - create_locations_list(); - timer_locations_list.stop(); - - // Find all locations where more than two segments start or - // end. We call those "split" locations. If there are any - // "spike" segments found while doing this, we know the area - // geometry isn't valid and return. - osmium::Timer timer_split; - if (!find_split_locations()) { - return false; - } - timer_split.stop(); - - // Now report all split locations to the problem reporter. - m_stats.touching_rings += m_split_locations.size(); - if (!m_split_locations.empty()) { - if (debug()) { - std::cerr << " Found split locations:\n"; - } - for (const auto& location : m_split_locations) { - if (m_config.problem_reporter) { - auto it = std::lower_bound(m_locations.cbegin(), m_locations.cend(), slocation{}, [this, &location](const slocation& lhs, const slocation& rhs) { - return lhs.location(m_segment_list, location) < rhs.location(m_segment_list, location); - }); - assert(it != m_locations.cend()); - const osmium::object_id_type id = it->node_ref(m_segment_list).ref(); - m_config.problem_reporter->report_touching_ring(id, location); - } - if (debug()) { - std::cerr << " " << location << "\n"; - } - } - } - - // From here on we use two different algorithms depending on - // whether there were any split locations or not. If there - // are no splits, we use the faster "simple algorithm", if - // there are, we use the slower "complex algorithm". - osmium::Timer timer_simple_case; - osmium::Timer timer_complex_case; - if (m_split_locations.empty()) { - if (debug()) { - std::cerr << " No split locations -> using simple algorithm\n"; - } - ++m_stats.area_simple_case; - - timer_simple_case.start(); - create_rings_simple_case(); - timer_simple_case.stop(); - } else { - if (debug()) { - std::cerr << " Found split locations -> using complex algorithm\n"; - } - ++m_stats.area_touching_rings_case; - - timer_complex_case.start(); - if (!create_rings_complex_case()) { - return false; - } - timer_complex_case.stop(); - } - - // If the assembler was so configured, now check whether the - // member roles are correctly tagged. - if (m_config.check_roles && m_stats.from_relations) { - osmium::Timer timer_roles; - check_inner_outer_roles(); - timer_roles.stop(); - } - - m_stats.outer_rings = std::count_if(m_rings.cbegin(), m_rings.cend(), [](const detail::ProtoRing& ring){ - return ring.is_outer(); - }); - m_stats.inner_rings = m_rings.size() - m_stats.outer_rings; - -#ifdef OSMIUM_WITH_TIMER - std::cout << m_stats.nodes << ' ' << m_stats.outer_rings << ' ' << m_stats.inner_rings << - ' ' << timer_sort.elapsed_microseconds() << - ' ' << timer_dupl.elapsed_microseconds() << - ' ' << timer_intersection.elapsed_microseconds() << - ' ' << timer_locations_list.elapsed_microseconds() << - ' ' << timer_split.elapsed_microseconds(); - - if (m_split_locations.empty()) { - std::cout << ' ' << timer_simple_case.elapsed_microseconds() << - " 0"; - } else { - std::cout << " 0" << - ' ' << timer_complex_case.elapsed_microseconds(); - } - - std::cout << -# ifdef OSMIUM_AREA_CHECK_INNER_OUTER_ROLES - ' ' << timer_roles.elapsed_microseconds() << -# else - " 0" << -# endif - '\n'; -#endif - - return true; - } - -#ifdef OSMIUM_WITH_TIMER - static bool print_header() { - std::cout << "nodes outer_rings inner_rings sort dupl intersection locations split simple_case complex_case roles_check\n"; - return true; - } - - static bool init_header() { - static bool printed_print_header = print_header(); - return printed_print_header; - } -#endif - bool create_area(osmium::memory::Buffer& out_buffer, const osmium::Way& way) { osmium::builder::AreaBuilder builder{out_buffer}; builder.initialize_from_object(way); const bool area_okay = create_rings(); - if (area_okay || m_config.create_empty_areas) { + if (area_okay || config().create_empty_areas) { add_tags_to_area(builder, way); } if (area_okay) { @@ -1374,19 +194,19 @@ namespace osmium { } if (report_ways()) { - m_config.problem_reporter->report_way(way); + config().problem_reporter->report_way(way); } - return area_okay || m_config.create_empty_areas; + return area_okay || config().create_empty_areas; } bool create_area(osmium::memory::Buffer& out_buffer, const osmium::Relation& relation, const std::vector& members) { - m_num_members = members.size(); + set_num_members(members.size()); osmium::builder::AreaBuilder builder{out_buffer}; builder.initialize_from_object(relation); const bool area_okay = create_rings(); - if (area_okay || m_config.create_empty_areas) { + if (area_okay || config().create_empty_areas) { add_tags_to_area(builder, relation); } if (area_okay) { @@ -1395,11 +215,11 @@ namespace osmium { if (report_ways()) { for (const osmium::Way* way : members) { - m_config.problem_reporter->report_way(*way); + config().problem_reporter->report_way(*way); } } - return area_okay || m_config.create_empty_areas; + return area_okay || config().create_empty_areas; } public: @@ -1407,11 +227,7 @@ namespace osmium { using config_type = osmium::area::AssemblerConfig; explicit Assembler(const config_type& config) : - m_config(config), - m_segment_list(config.debug_level > 1) { -#ifdef OSMIUM_WITH_TIMER - init_header(); -#endif + detail::BasicAssembler(config) { } ~Assembler() noexcept = default; @@ -1419,52 +235,63 @@ namespace osmium { /** * Assemble an area from the given way. * The resulting area is put into the out_buffer. + * + * @returns false if there was some kind of error building the + * area, true otherwise. */ - void operator()(const osmium::Way& way, osmium::memory::Buffer& out_buffer) { - if (!m_config.create_way_polygons) { - return; + bool operator()(const osmium::Way& way, osmium::memory::Buffer& out_buffer) { + if (!config().create_way_polygons) { + return true; } if (way.tags().has_tag("area", "no")) { - return; + return true; } - if (m_config.problem_reporter) { - m_config.problem_reporter->set_object(osmium::item_type::way, way.id()); - m_config.problem_reporter->set_nodes(way.nodes().size()); + if (config().problem_reporter) { + config().problem_reporter->set_object(osmium::item_type::way, way.id()); + config().problem_reporter->set_nodes(way.nodes().size()); } // Ignore (but count) ways without segments. if (way.nodes().size() < 2) { - ++m_stats.short_ways; - return; + ++stats().short_ways; + return false; } if (!way.ends_have_same_id()) { - ++m_stats.duplicate_nodes; - if (m_config.problem_reporter) { - m_config.problem_reporter->report_duplicate_node(way.nodes().front().ref(), way.nodes().back().ref(), way.nodes().front().location()); + ++stats().duplicate_nodes; + if (config().problem_reporter) { + config().problem_reporter->report_duplicate_node(way.nodes().front().ref(), way.nodes().back().ref(), way.nodes().front().location()); } } - ++m_stats.from_ways; - m_stats.duplicate_nodes += m_segment_list.extract_segments_from_way(m_config.problem_reporter, way); + ++stats().from_ways; + stats().invalid_locations = segment_list().extract_segments_from_way(config().problem_reporter, + stats().duplicate_nodes, + way); + if (!config().ignore_invalid_locations && stats().invalid_locations > 0) { + return false; + } - if (m_config.debug_level > 0) { - std::cerr << "\nAssembling way " << way.id() << " containing " << m_segment_list.size() << " nodes\n"; + if (config().debug_level > 0) { + std::cerr << "\nAssembling way " << way.id() << " containing " << segment_list().size() << " nodes\n"; } // Now create the Area object and add the attributes and tags // from the way. - if (create_area(out_buffer, way)) { + const bool okay = create_area(out_buffer, way); + if (okay) { out_buffer.commit(); } else { out_buffer.rollback(); } if (debug()) { - std::cerr << "Done: " << m_stats << "\n"; + std::cerr << "Done: " << stats() << "\n"; } + + return okay; } /** @@ -1489,38 +316,48 @@ namespace osmium { /** * Assemble an area from the given relation and its members. * The resulting area is put into the out_buffer. + * + * @returns false if there was some kind of error building the + * area(s), true otherwise. */ - void operator()(const osmium::Relation& relation, const std::vector& members, osmium::memory::Buffer& out_buffer) { + bool operator()(const osmium::Relation& relation, const std::vector& members, osmium::memory::Buffer& out_buffer) { assert(relation.members().size() >= members.size()); - if (m_config.problem_reporter) { - m_config.problem_reporter->set_object(osmium::item_type::relation, relation.id()); + if (config().problem_reporter) { + config().problem_reporter->set_object(osmium::item_type::relation, relation.id()); } if (relation.members().empty()) { - ++m_stats.no_way_in_mp_relation; - return; + ++stats().no_way_in_mp_relation; + return false; } - ++m_stats.from_relations; - m_stats.duplicate_nodes += m_segment_list.extract_segments_from_ways(m_config.problem_reporter, relation, members); - m_stats.member_ways = members.size(); + ++stats().from_relations; + stats().invalid_locations = segment_list().extract_segments_from_ways(config().problem_reporter, + stats().duplicate_nodes, + relation, + members); + if (!config().ignore_invalid_locations && stats().invalid_locations > 0) { + return false; + } + stats().member_ways = members.size(); - if (m_stats.member_ways == 1) { - ++m_stats.single_way_in_mp_relation; + if (stats().member_ways == 1) { + ++stats().single_way_in_mp_relation; } - if (m_config.debug_level > 0) { - std::cerr << "\nAssembling relation " << relation.id() << " containing " << members.size() << " way members with " << m_segment_list.size() << " nodes\n"; + if (config().debug_level > 0) { + std::cerr << "\nAssembling relation " << relation.id() << " containing " << members.size() << " way members with " << segment_list().size() << " nodes\n"; } const size_t area_offset = out_buffer.committed(); // Now create the Area object and add the attributes and tags // from the relation. - if (create_area(out_buffer, relation, members)) { - if ((m_config.create_new_style_polygons && m_stats.no_tags_on_relation == 0) || - (m_config.create_old_style_polygons && m_stats.no_tags_on_relation != 0)) { + bool okay = create_area(out_buffer, relation, members); + if (okay) { + if ((config().create_new_style_polygons && stats().no_tags_on_relation == 0) || + (config().create_old_style_polygons && stats().no_tags_on_relation != 0)) { out_buffer.commit(); } else { out_buffer.rollback(); @@ -1536,23 +373,23 @@ namespace osmium { // just built, add them to a list and later build areas for // them, too. std::vector ways_that_should_be_areas; - if (m_stats.wrong_role == 0) { + if (stats().wrong_role == 0) { detail::for_each_member(relation, members, [this, &ways_that_should_be_areas, &area_tags](const osmium::RelationMember& member, const osmium::Way& way) { if (!std::strcmp(member.role(), "inner")) { if (!way.nodes().empty() && way.is_closed() && way.tags().size() > 0) { - const auto d = std::count_if(way.tags().cbegin(), way.tags().cend(), filter()); + const auto d = std::count_if(way.tags().cbegin(), way.tags().cend(), std::cref(filter())); if (d > 0) { - osmium::tags::KeyFilter::iterator way_fi_begin(filter(), way.tags().cbegin(), way.tags().cend()); - osmium::tags::KeyFilter::iterator way_fi_end(filter(), way.tags().cend(), way.tags().cend()); - osmium::tags::KeyFilter::iterator area_fi_begin(filter(), area_tags.cbegin(), area_tags.cend()); - osmium::tags::KeyFilter::iterator area_fi_end(filter(), area_tags.cend(), area_tags.cend()); + osmium::tags::KeyFilter::iterator way_fi_begin(std::cref(filter()), way.tags().cbegin(), way.tags().cend()); + osmium::tags::KeyFilter::iterator way_fi_end(std::cref(filter()), way.tags().cend(), way.tags().cend()); + osmium::tags::KeyFilter::iterator area_fi_begin(std::cref(filter()), area_tags.cbegin(), area_tags.cend()); + osmium::tags::KeyFilter::iterator area_fi_end(std::cref(filter()), area_tags.cend(), area_tags.cend()); if (!std::equal(way_fi_begin, way_fi_end, area_fi_begin) || d != std::distance(area_fi_begin, area_fi_end)) { ways_that_should_be_areas.push_back(&way); } else { - ++m_stats.inner_with_same_tags; - if (m_config.problem_reporter) { - m_config.problem_reporter->report_inner_with_same_tags(way); + ++stats().inner_with_same_tags; + if (config().problem_reporter) { + config().problem_reporter->report_inner_with_same_tags(way); } } } @@ -1562,22 +399,18 @@ namespace osmium { } if (debug()) { - std::cerr << "Done: " << m_stats << "\n"; + std::cerr << "Done: " << stats() << "\n"; } // Now build areas for all ways found in the last step. for (const osmium::Way* way : ways_that_should_be_areas) { - Assembler assembler(m_config); - assembler(*way, out_buffer); + Assembler assembler{config()}; + if (!assembler(*way, out_buffer)) { + okay = false; + } } - } - /** - * Get statistics from assembler. Call this after running the - * assembler to get statistics and data about errors. - */ - const osmium::area::area_stats& stats() const noexcept { - return m_stats; + return okay; } }; // class Assembler diff --git a/contrib/libosmium/osmium/area/assembler_config.hpp b/contrib/libosmium/osmium/area/assembler_config.hpp new file mode 100644 index 000000000..2e49080a4 --- /dev/null +++ b/contrib/libosmium/osmium/area/assembler_config.hpp @@ -0,0 +1,152 @@ +#ifndef OSMIUM_AREA_ASSEMBLER_CONFIG_HPP +#define OSMIUM_AREA_ASSEMBLER_CONFIG_HPP + +/* + +This file is part of Osmium (http://osmcode.org/libosmium). + +Copyright 2013-2017 Jochen Topf and others (see README). + +Boost Software License - Version 1.0 - August 17th, 2003 + +Permission is hereby granted, free of charge, to any person or organization +obtaining a copy of the software and accompanying documentation covered by +this license (the "Software") to use, reproduce, display, distribute, +execute, and transmit the Software, and to prepare derivative works of the +Software, and to permit third-parties to whom the Software is furnished to +do so, all subject to the following: + +The copyright notices in the Software and this entire statement, including +the above license grant, this restriction and the following disclaimer, +must be included in all copies of the Software, in whole or in part, and +all derivative works of the Software, unless such copies or derivative +works are solely in the form of machine-executable object code generated by +a source language processor. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT +SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE +FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, +ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. + +*/ + +#include + +namespace osmium { + + namespace area { + + class ProblemReporter; + + /** + * Configuration for osmium::area::Assembler objects. Create this + * once, set the options you want and then re-use it every time you + * create an Assembler object. + */ + struct AssemblerConfig { + + /** + * Optional pointer to problem reporter. + */ + osmium::area::ProblemReporter* problem_reporter = nullptr; + + /** + * Debug level. If this is greater than zero, debug messages will + * be printed to stderr. Available levels are 1 to 3. Note that + * level 2 and above will generate a lot of messages! + */ + int debug_level = 0; + + /** + * The roles of multipolygon members are ignored when assembling + * multipolygons, because they are often missing or wrong. If this + * is set, the roles are checked after the multipolygons are built + * against what the assembly process decided where the inner and + * outer rings are. This slows down the processing, so it only + * makes sense if you want to get the problem reports. + */ + bool check_roles = false; + + /** + * When the assembler can't create an area, usually because its + * geometry would be invalid, it will create an "empty" area object + * without rings. This allows you to detect where an area was + * invalid. + * + * If this is set to false, invalid areas will simply be discarded. + */ + bool create_empty_areas = true; + + /** + * Create areas for (multi)polygons where the tags are on the + * relation. + * + * If this is set to false, those areas will simply be discarded. + */ + bool create_new_style_polygons = true; + + /** + * Create areas for (multi)polygons where the tags are on the + * outer way(s). + * + * If this is set to false, those areas will simply be discarded. + */ + bool create_old_style_polygons = true; + + /** + * Create areas for polygons created from ways. + * + * If this is set to false, those areas will simply be discarded. + */ + bool create_way_polygons = true; + + /** + * Keep the type tag from multipolygon relations on the area + * object. By default this is false, and the type tag will be + * removed. + */ + bool keep_type_tag = false; + + /** + * If there is an invalid location in any of the ways needed for + * assembling the multipolygon, the assembler will normally fail. + * If this is set, the assembler will silently ignore the invalid + * locations pretending them to be not referenced from the ways. + * This will allow some areas to be built, others will now be + * incorrect. This can sometimes be useful to assemble areas + * crossing the boundary of an extract, but you will also get + * geometrically valid but wrong (multi)polygons. + */ + bool ignore_invalid_locations = false; + + AssemblerConfig() noexcept = default; + + /** + * Constructor + * @deprecated Use default constructor and set values afterwards. + */ + explicit AssemblerConfig(osmium::area::ProblemReporter* pr, bool d = false) : + problem_reporter(pr), + debug_level(d) { + } + + /** + * Enable or disable debug output to stderr. This is for Osmium + * developers only. + * + * @deprecated Set debug_level directly. + */ + OSMIUM_DEPRECATED void enable_debug_output(bool d = true) { + debug_level = d; + } + + }; // struct AssemblerConfig + + } // namespace area + +} // namespace osmium + +#endif // OSMIUM_AREA_ASSEMBLER_CONFIG_HPP diff --git a/contrib/libosmium/osmium/area/detail/basic_assembler.hpp b/contrib/libosmium/osmium/area/detail/basic_assembler.hpp new file mode 100644 index 000000000..13bd976c2 --- /dev/null +++ b/contrib/libosmium/osmium/area/detail/basic_assembler.hpp @@ -0,0 +1,1204 @@ +#ifndef OSMIUM_AREA_DETAIL_BASIC_ASSEMBLER_HPP +#define OSMIUM_AREA_DETAIL_BASIC_ASSEMBLER_HPP + +/* + +This file is part of Osmium (http://osmcode.org/libosmium). + +Copyright 2013-2017 Jochen Topf and others (see README). + +Boost Software License - Version 1.0 - August 17th, 2003 + +Permission is hereby granted, free of charge, to any person or organization +obtaining a copy of the software and accompanying documentation covered by +this license (the "Software") to use, reproduce, display, distribute, +execute, and transmit the Software, and to prepare derivative works of the +Software, and to permit third-parties to whom the Software is furnished to +do so, all subject to the following: + +The copyright notices in the Software and this entire statement, including +the above license grant, this restriction and the following disclaimer, +must be included in all copies of the Software, in whole or in part, and +all derivative works of the Software, unless such copies or derivative +works are solely in the form of machine-executable object code generated by +a source language processor. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT +SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE +FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, +ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. + +*/ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +namespace osmium { + + namespace area { + + namespace detail { + + using open_ring_its_type = std::list::iterator>; + + struct location_to_ring_map { + osmium::Location location; + open_ring_its_type::iterator ring_it; + bool start; + + location_to_ring_map(const osmium::Location& l, const open_ring_its_type::iterator& r, bool s) noexcept : + location(l), + ring_it(r), + start(s) { + } + + explicit location_to_ring_map(const osmium::Location& l) noexcept : + location(l), + ring_it(), + start(false) { + } + + const detail::ProtoRing& ring() const noexcept { + return **ring_it; + } + + }; // struct location_to_ring_map + + inline bool operator==(const location_to_ring_map& lhs, const location_to_ring_map& rhs) noexcept { + return lhs.location == rhs.location; + } + + inline bool operator<(const location_to_ring_map& lhs, const location_to_ring_map& rhs) noexcept { + return lhs.location < rhs.location; + } + + /** + * Class for assembling ways and relations into multipolygons + * (areas). Contains the basic functionality needed but is not + * used directly. Use the osmium::area::Assembler class instead. + */ + class BasicAssembler { + + struct slocation { + + static constexpr const uint32_t invalid_item = 1 << 30; + + uint32_t item : 31; + uint32_t reverse : 1; + + slocation() noexcept : + item(invalid_item), + reverse(false) { + } + + explicit slocation(uint32_t n, bool r = false) noexcept : + item(n), + reverse(r) { + } + + osmium::Location location(const detail::SegmentList& segment_list) const noexcept { + const auto& segment = segment_list[item]; + return reverse ? segment.second().location() : segment.first().location(); + } + + const osmium::NodeRef& node_ref(const detail::SegmentList& segment_list) const noexcept { + const auto& segment = segment_list[item]; + return reverse ? segment.second() : segment.first(); + } + + osmium::Location location(const detail::SegmentList& segment_list, const osmium::Location& default_location) const noexcept { + if (item == invalid_item) { + return default_location; + } + return location(segment_list); + } + + }; // struct slocation + + // Configuration settings for this Assembler + const AssemblerConfig& m_config; + + // List of segments (connection between two nodes) + osmium::area::detail::SegmentList m_segment_list; + + // The rings we are building from the segments + std::list m_rings; + + // All node locations + std::vector m_locations; + + // All locations where more than two segments start/end + std::vector m_split_locations; + + // Statistics + area_stats m_stats; + + // The number of members the multipolygon relation has + size_t m_num_members = 0; + + template + static void build_ring_from_proto_ring(osmium::builder::AreaBuilder& builder, const detail::ProtoRing& ring) { + TBuilder ring_builder{builder}; + ring_builder.add_node_ref(ring.get_node_ref_start()); + for (const auto& segment : ring.segments()) { + ring_builder.add_node_ref(segment->stop()); + } + } + + void check_inner_outer_roles() { + if (debug()) { + std::cerr << " Checking inner/outer roles\n"; + } + + std::unordered_map way_rings; + std::unordered_set ways_in_multiple_rings; + + for (const detail::ProtoRing& ring : m_rings) { + for (const auto& segment : ring.segments()) { + assert(segment->way()); + + if (!segment->role_empty() && (ring.is_outer() ? !segment->role_outer() : !segment->role_inner())) { + ++m_stats.wrong_role; + if (debug()) { + std::cerr << " Segment " << *segment << " from way " << segment->way()->id() << " has role '" << segment->role_name() + << "', but should have role '" << (ring.is_outer() ? "outer" : "inner") << "'\n"; + } + if (m_config.problem_reporter) { + if (ring.is_outer()) { + m_config.problem_reporter->report_role_should_be_outer(segment->way()->id(), segment->first().location(), segment->second().location()); + } else { + m_config.problem_reporter->report_role_should_be_inner(segment->way()->id(), segment->first().location(), segment->second().location()); + } + } + } + + auto& r = way_rings[segment->way()]; + if (!r) { + r = ˚ + } else if (r != &ring) { + ways_in_multiple_rings.insert(segment->way()); + } + + } + } + + for (const osmium::Way* way : ways_in_multiple_rings) { + ++m_stats.ways_in_multiple_rings; + if (debug()) { + std::cerr << " Way " << way->id() << " is in multiple rings\n"; + } + if (m_config.problem_reporter) { + m_config.problem_reporter->report_way_in_multiple_rings(*way); + } + } + + } + + detail::NodeRefSegment* get_next_segment(const osmium::Location& location) { + auto it = std::lower_bound(m_locations.begin(), m_locations.end(), slocation{}, [this, &location](const slocation& lhs, const slocation& rhs) { + return lhs.location(m_segment_list, location) < rhs.location(m_segment_list, location); + }); + + assert(it != m_locations.end()); + if (m_segment_list[it->item].is_done()) { + ++it; + } + assert(it != m_locations.end()); + + assert(!m_segment_list[it->item].is_done()); + return &m_segment_list[it->item]; + } + + class rings_stack_element { + + int32_t m_y; + detail::ProtoRing* m_ring_ptr; + + public: + + rings_stack_element(int32_t y, detail::ProtoRing* ring_ptr) : + m_y(y), + m_ring_ptr(ring_ptr) { + } + + int32_t y() const noexcept { + return m_y; + } + + const detail::ProtoRing& ring() const noexcept { + return *m_ring_ptr; + } + + detail::ProtoRing* ring_ptr() noexcept { + return m_ring_ptr; + } + + bool operator==(const rings_stack_element& rhs) const noexcept { + return m_ring_ptr == rhs.m_ring_ptr; + } + + bool operator<(const rings_stack_element& rhs) const noexcept { + return m_y < rhs.m_y; + } + + }; // class ring_stack_element + + using rings_stack = std::vector; + + void remove_duplicates(rings_stack& outer_rings) { + while (true) { + const auto it = std::adjacent_find(outer_rings.begin(), outer_rings.end()); + if (it == outer_rings.end()) { + return; + } + outer_rings.erase(it, std::next(it, 2)); + } + } + + detail::ProtoRing* find_enclosing_ring(detail::NodeRefSegment* segment) { + if (debug()) { + std::cerr << " Looking for ring enclosing " << *segment << "\n"; + } + + const auto location = segment->first().location(); + const auto end_location = segment->second().location(); + + while (segment->first().location() == location) { + if (segment == &m_segment_list.back()) { + break; + } + ++segment; + } + + int nesting = 0; + + rings_stack outer_rings; + while (segment >= &m_segment_list.front()) { + if (!segment->is_direction_done()) { + --segment; + continue; + } + if (debug()) { + std::cerr << " Checking against " << *segment << "\n"; + } + const osmium::Location& a = segment->first().location(); + const osmium::Location& b = segment->second().location(); + + if (segment->first().location() == location) { + const int64_t ax = a.x(); + const int64_t bx = b.x(); + const int64_t lx = end_location.x(); + const int64_t ay = a.y(); + const int64_t by = b.y(); + const int64_t ly = end_location.y(); + const auto z = (bx - ax)*(ly - ay) - (by - ay)*(lx - ax); + if (debug()) { + std::cerr << " Segment XXXX z=" << z << "\n"; + } + if (z > 0) { + nesting += segment->is_reverse() ? -1 : 1; + if (debug()) { + std::cerr << " Segment is below (nesting=" << nesting << ")\n"; + } + if (segment->ring()->is_outer()) { + if (debug()) { + std::cerr << " Segment belongs to outer ring\n"; + } + outer_rings.emplace_back(a.y(), segment->ring()); + } + } + } else if (a.x() <= location.x() && location.x() < b.x()) { + if (debug()) { + std::cerr << " Is in x range\n"; + } + + const int64_t ax = a.x(); + const int64_t bx = b.x(); + const int64_t lx = location.x(); + const int64_t ay = a.y(); + const int64_t by = b.y(); + const int64_t ly = location.y(); + const auto z = (bx - ax)*(ly - ay) - (by - ay)*(lx - ax); + + if (z >= 0) { + nesting += segment->is_reverse() ? -1 : 1; + if (debug()) { + std::cerr << " Segment is below (nesting=" << nesting << ")\n"; + } + if (segment->ring()->is_outer()) { + if (debug()) { + std::cerr << " Segment belongs to outer ring\n"; + } + const int32_t y = int32_t(ay + (by - ay) * (lx - ax) / (bx - ax)); + outer_rings.emplace_back(y, segment->ring()); + } + } + } + --segment; + } + + if (nesting % 2 == 0) { + if (debug()) { + std::cerr << " Decided that this is an outer ring\n"; + } + return nullptr; + } else { + if (debug()) { + std::cerr << " Decided that this is an inner ring\n"; + } + assert(!outer_rings.empty()); + + std::sort(outer_rings.rbegin(), outer_rings.rend()); + if (debug()) { + for (const auto& o : outer_rings) { + std::cerr << " y=" << o.y() << " " << o.ring() << "\n"; + } + } + + remove_duplicates(outer_rings); + if (debug()) { + std::cerr << " after remove duplicates:\n"; + for (const auto& o : outer_rings) { + std::cerr << " y=" << o.y() << " " << o.ring() << "\n"; + } + } + + assert(!outer_rings.empty()); + return outer_rings.front().ring_ptr(); + } + } + + bool is_split_location(const osmium::Location& location) const noexcept { + return std::find(m_split_locations.cbegin(), m_split_locations.cend(), location) != m_split_locations.cend(); + } + + uint32_t add_new_ring(slocation& node) { + detail::NodeRefSegment* segment = &m_segment_list[node.item]; + assert(!segment->is_done()); + + if (debug()) { + std::cerr << " Starting new ring at location " << node.location(m_segment_list) << " with segment " << *segment << "\n"; + } + + if (node.reverse) { + segment->reverse(); + } + + detail::ProtoRing* outer_ring = nullptr; + + if (segment != &m_segment_list.front()) { + outer_ring = find_enclosing_ring(segment); + } + segment->mark_direction_done(); + + m_rings.emplace_back(segment); + detail::ProtoRing* ring = &m_rings.back(); + if (outer_ring) { + if (debug()) { + std::cerr << " This is an inner ring. Outer ring is " << *outer_ring << "\n"; + } + outer_ring->add_inner_ring(ring); + ring->set_outer_ring(outer_ring); + } else if (debug()) { + std::cerr << " This is an outer ring\n"; + } + + const osmium::Location& first_location = node.location(m_segment_list); + osmium::Location last_location = segment->stop().location(); + + uint32_t nodes = 1; + while (first_location != last_location) { + ++nodes; + detail::NodeRefSegment* next_segment = get_next_segment(last_location); + next_segment->mark_direction_done(); + if (next_segment->start().location() != last_location) { + next_segment->reverse(); + } + ring->add_segment_back(next_segment); + if (debug()) { + std::cerr << " Next segment is " << *next_segment << "\n"; + } + last_location = next_segment->stop().location(); + } + + ring->fix_direction(); + + if (debug()) { + std::cerr << " Completed ring: " << *ring << "\n"; + } + + return nodes; + } + + uint32_t add_new_ring_complex(slocation& node) { + detail::NodeRefSegment* segment = &m_segment_list[node.item]; + assert(!segment->is_done()); + + if (debug()) { + std::cerr << " Starting new ring at location " << node.location(m_segment_list) << " with segment " << *segment << "\n"; + } + + if (node.reverse) { + segment->reverse(); + } + + m_rings.emplace_back(segment); + detail::ProtoRing* ring = &m_rings.back(); + + const osmium::Location& first_location = node.location(m_segment_list); + osmium::Location last_location = segment->stop().location(); + + uint32_t nodes = 1; + while (first_location != last_location && !is_split_location(last_location)) { + ++nodes; + detail::NodeRefSegment* next_segment = get_next_segment(last_location); + if (next_segment->start().location() != last_location) { + next_segment->reverse(); + } + ring->add_segment_back(next_segment); + if (debug()) { + std::cerr << " Next segment is " << *next_segment << "\n"; + } + last_location = next_segment->stop().location(); + } + + if (debug()) { + if (first_location == last_location) { + std::cerr << " Completed ring: " << *ring << "\n"; + } else { + std::cerr << " Completed partial ring: " << *ring << "\n"; + } + } + + return nodes; + } + + void create_locations_list() { + m_locations.reserve(m_segment_list.size() * 2); + + for (uint32_t n = 0; n < m_segment_list.size(); ++n) { + m_locations.emplace_back(n, false); + m_locations.emplace_back(n, true); + } + + std::stable_sort(m_locations.begin(), m_locations.end(), [this](const slocation& lhs, const slocation& rhs) { + return lhs.location(m_segment_list) < rhs.location(m_segment_list); + }); + } + + void find_inner_outer_complex(detail::ProtoRing* ring) { + detail::ProtoRing* outer_ring = find_enclosing_ring(ring->min_segment()); + if (outer_ring) { + outer_ring->add_inner_ring(ring); + ring->set_outer_ring(outer_ring); + } + ring->fix_direction(); + ring->mark_direction_done(); + } + + void find_inner_outer_complex() { + if (debug()) { + std::cerr << " Finding inner/outer rings\n"; + } + std::vector rings; + rings.reserve(m_rings.size()); + for (auto& ring : m_rings) { + if (ring.closed()) { + rings.push_back(&ring); + } + } + + if (rings.empty()) { + return; + } + + std::sort(rings.begin(), rings.end(), [](detail::ProtoRing* a, detail::ProtoRing* b) { + return a->min_segment() < b->min_segment(); + }); + + rings.front()->fix_direction(); + rings.front()->mark_direction_done(); + if (debug()) { + std::cerr << " First ring is outer: " << *rings.front() << "\n"; + } + for (auto it = std::next(rings.begin()); it != rings.end(); ++it) { + if (debug()) { + std::cerr << " Checking (at min segment " << *((*it)->min_segment()) << ") ring " << **it << "\n"; + } + find_inner_outer_complex(*it); + if (debug()) { + std::cerr << " Ring is " << ((*it)->is_outer() ? "OUTER: " : "INNER: ") << **it << "\n"; + } + } + } + + /** + * Finds all locations where more than two segments meet. If there + * are any open rings found along the way, they are reported + * and the function returns false. + */ + bool find_split_locations() { + osmium::Location previous_location; + for (auto it = m_locations.cbegin(); it != m_locations.cend(); ++it) { + const osmium::NodeRef& nr = it->node_ref(m_segment_list); + const osmium::Location& loc = nr.location(); + if (std::next(it) == m_locations.cend() || loc != std::next(it)->location(m_segment_list)) { + if (debug()) { + std::cerr << " Found open ring at " << nr << "\n"; + } + if (m_config.problem_reporter) { + const auto& segment = m_segment_list[it->item]; + m_config.problem_reporter->report_ring_not_closed(nr, segment.way()); + } + ++m_stats.open_rings; + } else { + if (loc == previous_location && (m_split_locations.empty() || m_split_locations.back() != previous_location )) { + m_split_locations.push_back(previous_location); + } + ++it; + if (it == m_locations.end()) { + break; + } + } + previous_location = loc; + } + return m_stats.open_rings == 0; + } + + void create_rings_simple_case() { + auto count_remaining = m_segment_list.size(); + for (slocation& sl : m_locations) { + const detail::NodeRefSegment& segment = m_segment_list[sl.item]; + if (!segment.is_done()) { + count_remaining -= add_new_ring(sl); + if (count_remaining == 0) { + return; + } + } + } + } + + std::vector create_location_to_ring_map(open_ring_its_type& open_ring_its) { + std::vector xrings; + xrings.reserve(open_ring_its.size() * 2); + + for (auto it = open_ring_its.begin(); it != open_ring_its.end(); ++it) { + if (debug()) { + std::cerr << " Ring: " << **it << "\n"; + } + xrings.emplace_back((*it)->get_node_ref_start().location(), it, true); + xrings.emplace_back((*it)->get_node_ref_stop().location(), it, false); + } + + std::sort(xrings.begin(), xrings.end()); + + return xrings; + } + + void merge_two_rings(open_ring_its_type& open_ring_its, const location_to_ring_map& m1, const location_to_ring_map& m2) { + auto& r1 = *m1.ring_it; + auto& r2 = *m2.ring_it; + + if (r1->get_node_ref_stop().location() == r2->get_node_ref_start().location()) { + r1->join_forward(*r2); + } else if (r1->get_node_ref_stop().location() == r2->get_node_ref_stop().location()) { + r1->join_backward(*r2); + } else if (r1->get_node_ref_start().location() == r2->get_node_ref_start().location()) { + r1->reverse(); + r1->join_forward(*r2); + } else if (r1->get_node_ref_start().location() == r2->get_node_ref_stop().location()) { + r1->reverse(); + r1->join_backward(*r2); + } else { + assert(false); + } + + m_rings.erase(r2); + open_ring_its.remove(r2); + + if (r1->closed()) { + open_ring_its.remove(r1); + } + } + + bool try_to_merge(open_ring_its_type& open_ring_its) { + if (open_ring_its.empty()) { + return false; + } + + if (debug()) { + std::cerr << " Trying to merge " << open_ring_its.size() << " open rings\n"; + } + + std::vector xrings = create_location_to_ring_map(open_ring_its); + + auto it = xrings.cbegin(); + while (it != xrings.cend()) { + it = std::adjacent_find(it, xrings.cend()); + if (it == xrings.cend()) { + return false; + } + auto after = std::next(it, 2); + if (after == xrings.cend() || after->location != it->location) { + if (debug()) { + std::cerr << " Merging two rings\n"; + } + merge_two_rings(open_ring_its, *it, *std::next(it)); + return true; + } + while (it != xrings.cend() && it->location == after->location) { + ++it; + } + } + + return false; + } + + bool there_are_open_rings() const noexcept { + return std::any_of(m_rings.cbegin(), m_rings.cend(), [](const detail::ProtoRing& ring){ + return !ring.closed(); + }); + } + + struct candidate { + int64_t sum; + std::vector> rings; + osmium::Location start_location; + osmium::Location stop_location; + + explicit candidate(location_to_ring_map& ring, bool reverse) : + sum(ring.ring().sum()), + rings(), + start_location(ring.ring().get_node_ref_start().location()), + stop_location(ring.ring().get_node_ref_stop().location()) { + rings.emplace_back(ring, reverse); + } + + bool closed() const noexcept { + return start_location == stop_location; + } + + }; + + void find_candidates(std::vector& candidates, std::unordered_set& loc_done, const std::vector& xrings, candidate& cand) { + if (debug()) { + std::cerr << " find_candidates sum=" << cand.sum << " start=" << cand.start_location << " stop=" << cand.stop_location << "\n"; + for (const auto& ring : cand.rings) { + std::cerr << " " << ring.first.ring() << (ring.second ? " reverse" : "") << "\n"; + } + } + + const auto connections = make_range(std::equal_range(xrings.cbegin(), + xrings.cend(), + location_to_ring_map{cand.stop_location})); + + assert(connections.begin() != connections.end()); + + assert(!cand.rings.empty()); + const detail::ProtoRing* ring_leading_here = &cand.rings.back().first.ring(); + for (const location_to_ring_map& m : connections) { + const detail::ProtoRing& ring = m.ring(); + + if (&ring != ring_leading_here) { + if (debug()) { + std::cerr << " next possible connection: " << ring << (m.start ? "" : " reverse") << "\n"; + } + + candidate c = cand; + if (m.start) { + c.rings.emplace_back(m, false); + c.stop_location = ring.get_node_ref_stop().location(); + c.sum += ring.sum(); + } else { + c.rings.emplace_back(m, true); + c.stop_location = ring.get_node_ref_start().location(); + c.sum -= ring.sum(); + } + if (c.closed()) { + if (debug()) { + std::cerr << " found candidate\n"; + } + candidates.push_back(c); + } else if (loc_done.count(c.stop_location) == 0) { + if (debug()) { + std::cerr << " recurse...\n"; + } + loc_done.insert(c.stop_location); + find_candidates(candidates, loc_done, xrings, c); + if (debug()) { + std::cerr << " ...back\n"; + } + } else if (debug()) { + std::cerr << " loop found\n"; + } + } + } + } + + /** + * If there are multiple open rings and multiple ways to join them, + * this function is called. It will take the first open ring and + * try recursively all ways of closing it. Of all the candidates + * the one with the smallest area is chosen and closed. If it + * can't close this ring, an error is reported and the function + * returns false. + */ + bool join_connected_rings(open_ring_its_type& open_ring_its) { + assert(!open_ring_its.empty()); + + if (debug()) { + std::cerr << " Trying to merge " << open_ring_its.size() << " open rings\n"; + } + + std::vector xrings = create_location_to_ring_map(open_ring_its); + + const auto ring_min = std::min_element(xrings.begin(), xrings.end(), [](const location_to_ring_map& lhs, const location_to_ring_map& rhs) { + return lhs.ring().min_segment() < rhs.ring().min_segment(); + }); + + find_inner_outer_complex(); + detail::ProtoRing* outer_ring = find_enclosing_ring(ring_min->ring().min_segment()); + bool ring_min_is_outer = !outer_ring; + if (debug()) { + std::cerr << " Open ring is " << (ring_min_is_outer ? "outer" : "inner") << " ring\n"; + } + for (auto& ring : m_rings) { + ring.reset(); + } + + candidate cand{*ring_min, false}; + + // Locations we have visited while finding candidates, used + // to detect loops. + std::unordered_set loc_done; + + loc_done.insert(cand.stop_location); + + std::vector candidates; + find_candidates(candidates, loc_done, xrings, cand); + + if (candidates.empty()) { + if (debug()) { + std::cerr << " Found no candidates\n"; + } + if (!open_ring_its.empty()) { + ++m_stats.open_rings; + if (m_config.problem_reporter) { + for (auto& it : open_ring_its) { + m_config.problem_reporter->report_ring_not_closed(it->get_node_ref_start()); + m_config.problem_reporter->report_ring_not_closed(it->get_node_ref_stop()); + } + } + } + return false; + } + + if (debug()) { + std::cerr << " Found candidates:\n"; + for (const auto& cand : candidates) { + std::cerr << " sum=" << cand.sum << "\n"; + for (const auto& ring : cand.rings) { + std::cerr << " " << ring.first.ring() << (ring.second ? " reverse" : "") << "\n"; + } + } + } + + // Find the candidate with the smallest/largest area + const auto chosen_cand = ring_min_is_outer ? + std::min_element(candidates.cbegin(), candidates.cend(), [](const candidate& lhs, const candidate& rhs) { + return std::abs(lhs.sum) < std::abs(rhs.sum); + }) : + std::max_element(candidates.cbegin(), candidates.cend(), [](const candidate& lhs, const candidate& rhs) { + return std::abs(lhs.sum) < std::abs(rhs.sum); + }); + + if (debug()) { + std::cerr << " Decided on: sum=" << chosen_cand->sum << "\n"; + for (const auto& ring : chosen_cand->rings) { + std::cerr << " " << ring.first.ring() << (ring.second ? " reverse" : "") << "\n"; + } + } + + // Join all (open) rings in the candidate to get one closed ring. + assert(chosen_cand->rings.size() > 1); + const auto& first_ring = chosen_cand->rings.front().first; + for (auto it = chosen_cand->rings.begin() + 1; it != chosen_cand->rings.end(); ++it) { + merge_two_rings(open_ring_its, first_ring, it->first); + } + + if (debug()) { + std::cerr << " Merged to " << first_ring.ring() << "\n"; + } + + return true; + } + + bool create_rings_complex_case() { + // First create all the (partial) rings starting at the split locations + auto count_remaining = m_segment_list.size(); + for (const osmium::Location& location : m_split_locations) { + const auto locs = make_range(std::equal_range(m_locations.begin(), + m_locations.end(), + slocation{}, + [this, &location](const slocation& lhs, const slocation& rhs) { + return lhs.location(m_segment_list, location) < rhs.location(m_segment_list, location); + })); + for (auto& loc : locs) { + if (!m_segment_list[loc.item].is_done()) { + count_remaining -= add_new_ring_complex(loc); + if (count_remaining == 0) { + break; + } + } + } + } + + // Now find all the rest of the rings (ie not starting at split locations) + if (count_remaining > 0) { + for (slocation& sl : m_locations) { + const detail::NodeRefSegment& segment = m_segment_list[sl.item]; + if (!segment.is_done()) { + count_remaining -= add_new_ring_complex(sl); + if (count_remaining == 0) { + break; + } + } + } + } + + // Now all segments are in exactly one (partial) ring. + + // If there are open rings, try to join them to create closed + // rings. + if (there_are_open_rings()) { + ++m_stats.area_really_complex_case; + + open_ring_its_type open_ring_its; + for (auto it = m_rings.begin(); it != m_rings.end(); ++it) { + if (!it->closed()) { + open_ring_its.push_back(it); + } + } + + while (!open_ring_its.empty()) { + if (debug()) { + std::cerr << " There are " << open_ring_its.size() << " open rings\n"; + } + while (try_to_merge(open_ring_its)); + + if (!open_ring_its.empty()) { + if (debug()) { + std::cerr << " After joining obvious cases there are still " << open_ring_its.size() << " open rings\n"; + } + if (!join_connected_rings(open_ring_its)) { + return false; + } + } + } + + if (debug()) { + std::cerr << " Joined all open rings\n"; + } + } + + // Now all rings are complete. + + find_inner_outer_complex(); + + return true; + } + + /** + * Checks if any ways were completely removed in the + * erase_duplicate_segments step. + */ + bool ways_were_lost() { + std::unordered_set ways_in_segments; + + for (const auto& segment : m_segment_list) { + ways_in_segments.insert(segment.way()); + } + + return ways_in_segments.size() < m_num_members; + } + +#ifdef OSMIUM_WITH_TIMER + static bool print_header() { + std::cout << "nodes outer_rings inner_rings sort dupl intersection locations split simple_case complex_case roles_check\n"; + return true; + } + + static bool init_header() { + static bool printed_print_header = print_header(); + return printed_print_header; + } +#endif + + protected: + + const std::list& rings() const noexcept { + return m_rings; + } + + void set_num_members(size_t size) noexcept { + m_num_members = size; + } + + osmium::area::detail::SegmentList& segment_list() noexcept { + return m_segment_list; + } + + /** + * Append each outer ring together with its inner rings to the + * area in the buffer. + */ + void add_rings_to_area(osmium::builder::AreaBuilder& builder) const { + for (const detail::ProtoRing& ring : m_rings) { + if (ring.is_outer()) { + build_ring_from_proto_ring(builder, ring); + for (const detail::ProtoRing* inner : ring.inner_rings()) { + build_ring_from_proto_ring(builder, *inner); + } + } + } + } + + /** + * Create rings from segments. + */ + bool create_rings() { + m_stats.nodes += m_segment_list.size(); + + // Sort the list of segments (from left to right and bottom + // to top). + osmium::Timer timer_sort; + m_segment_list.sort(); + timer_sort.stop(); + + // Remove duplicate segments. Removal is in pairs, so if there + // are two identical segments, they will both be removed. If + // there are three, two will be removed and one remains. + osmium::Timer timer_dupl; + m_stats.duplicate_segments = m_segment_list.erase_duplicate_segments(m_config.problem_reporter); + timer_dupl.stop(); + + // If there are no segments left at this point, this isn't + // a valid area. + if (m_segment_list.empty()) { + if (debug()) { + std::cerr << " No segments left\n"; + } + return false; + } + + // If one or more complete ways was removed because of + // duplicate segments, this isn't a valid area. + if (ways_were_lost()) { + if (debug()) { + std::cerr << " Complete ways removed because of duplicate segments\n"; + } + return false; + } + + if (m_config.debug_level >= 3) { + std::cerr << "Sorted de-duplicated segment list:\n"; + for (const auto& s : m_segment_list) { + std::cerr << " " << s << "\n"; + } + } + + // Now we look for segments crossing each other. If there are + // any, the multipolygon is invalid. + // In the future this could be improved by trying to fix those + // cases. + osmium::Timer timer_intersection; + m_stats.intersections = m_segment_list.find_intersections(m_config.problem_reporter); + timer_intersection.stop(); + + if (m_stats.intersections) { + return false; + } + + // This creates an ordered list of locations of both endpoints + // of all segments with pointers back to the segments. We will + // use this list later to quickly find which segment(s) fits + // onto a known segment. + osmium::Timer timer_locations_list; + create_locations_list(); + timer_locations_list.stop(); + + // Find all locations where more than two segments start or + // end. We call those "split" locations. If there are any + // "spike" segments found while doing this, we know the area + // geometry isn't valid and return. + osmium::Timer timer_split; + if (!find_split_locations()) { + return false; + } + timer_split.stop(); + + // Now report all split locations to the problem reporter. + m_stats.touching_rings += m_split_locations.size(); + if (!m_split_locations.empty()) { + if (debug()) { + std::cerr << " Found split locations:\n"; + } + for (const auto& location : m_split_locations) { + if (m_config.problem_reporter) { + auto it = std::lower_bound(m_locations.cbegin(), m_locations.cend(), slocation{}, [this, &location](const slocation& lhs, const slocation& rhs) { + return lhs.location(m_segment_list, location) < rhs.location(m_segment_list, location); + }); + assert(it != m_locations.cend()); + const osmium::object_id_type id = it->node_ref(m_segment_list).ref(); + m_config.problem_reporter->report_touching_ring(id, location); + } + if (debug()) { + std::cerr << " " << location << "\n"; + } + } + } + + // From here on we use two different algorithms depending on + // whether there were any split locations or not. If there + // are no splits, we use the faster "simple algorithm", if + // there are, we use the slower "complex algorithm". + osmium::Timer timer_simple_case; + osmium::Timer timer_complex_case; + if (m_split_locations.empty()) { + if (debug()) { + std::cerr << " No split locations -> using simple algorithm\n"; + } + ++m_stats.area_simple_case; + + timer_simple_case.start(); + create_rings_simple_case(); + timer_simple_case.stop(); + } else { + if (debug()) { + std::cerr << " Found split locations -> using complex algorithm\n"; + } + ++m_stats.area_touching_rings_case; + + timer_complex_case.start(); + if (!create_rings_complex_case()) { + return false; + } + timer_complex_case.stop(); + } + + // If the assembler was so configured, now check whether the + // member roles are correctly tagged. + if (m_config.check_roles && m_stats.from_relations) { + osmium::Timer timer_roles; + check_inner_outer_roles(); + timer_roles.stop(); + } + + m_stats.outer_rings = std::count_if(m_rings.cbegin(), m_rings.cend(), [](const detail::ProtoRing& ring){ + return ring.is_outer(); + }); + m_stats.inner_rings = m_rings.size() - m_stats.outer_rings; + +#ifdef OSMIUM_WITH_TIMER + std::cout << m_stats.nodes << ' ' << m_stats.outer_rings << ' ' << m_stats.inner_rings << + ' ' << timer_sort.elapsed_microseconds() << + ' ' << timer_dupl.elapsed_microseconds() << + ' ' << timer_intersection.elapsed_microseconds() << + ' ' << timer_locations_list.elapsed_microseconds() << + ' ' << timer_split.elapsed_microseconds(); + + if (m_split_locations.empty()) { + std::cout << ' ' << timer_simple_case.elapsed_microseconds() << + " 0"; + } else { + std::cout << " 0" << + ' ' << timer_complex_case.elapsed_microseconds(); + } + + std::cout << +# ifdef OSMIUM_AREA_CHECK_INNER_OUTER_ROLES + ' ' << timer_roles.elapsed_microseconds() << +# else + " 0" << +# endif + '\n'; +#endif + + return true; + } + + public: + + using config_type = osmium::area::AssemblerConfig; + + explicit BasicAssembler(const config_type& config) : + m_config(config), + m_segment_list(config.debug_level > 1) { +#ifdef OSMIUM_WITH_TIMER + init_header(); +#endif + } + + ~BasicAssembler() noexcept = default; + + const AssemblerConfig& config() const noexcept { + return m_config; + } + + bool debug() const noexcept { + return m_config.debug_level > 1; + } + + /** + * Get statistics from assembler. Call this after running the + * assembler to get statistics and data about errors. + */ + const osmium::area::area_stats& stats() const noexcept { + return m_stats; + } + + osmium::area::area_stats& stats() noexcept { + return m_stats; + } + + }; // class BasicAssembler + + } // namespace detail + + } // namespace area + +} // namespace osmium + +#endif // OSMIUM_AREA_DETAIL_BASIC_ASSEMBLER_HPP diff --git a/contrib/libosmium/osmium/area/detail/segment_list.hpp b/contrib/libosmium/osmium/area/detail/segment_list.hpp index 64b2b4e5a..2dc58a355 100644 --- a/contrib/libosmium/osmium/area/detail/segment_list.hpp +++ b/contrib/libosmium/osmium/area/detail/segment_list.hpp @@ -110,11 +110,18 @@ namespace osmium { }); } - uint32_t extract_segments_from_way_impl(osmium::area::ProblemReporter* problem_reporter, const osmium::Way& way, role_type role) { - uint32_t duplicate_nodes = 0; + uint32_t extract_segments_from_way_impl(osmium::area::ProblemReporter* problem_reporter, uint64_t& duplicate_nodes, const osmium::Way& way, role_type role) { + uint32_t invalid_locations = 0; osmium::NodeRef previous_nr; for (const osmium::NodeRef& nr : way.nodes()) { + if (!nr.location().valid()) { + ++invalid_locations; + if (problem_reporter) { + problem_reporter->report_invalid_location(way.id(), nr.ref()); + } + continue; + } if (previous_nr.location()) { if (previous_nr.location() != nr.location()) { m_segments.emplace_back(previous_nr, nr, role, &way); @@ -128,7 +135,7 @@ namespace osmium { previous_nr = nr; } - return duplicate_nodes; + return invalid_locations; } public: @@ -213,19 +220,19 @@ namespace osmium { * same node or different nodes with same location) are * removed after reporting the duplicate node. */ - uint32_t extract_segments_from_way(osmium::area::ProblemReporter* problem_reporter, const osmium::Way& way) { + uint32_t extract_segments_from_way(osmium::area::ProblemReporter* problem_reporter, uint64_t& duplicate_nodes, const osmium::Way& way) { if (way.nodes().empty()) { return 0; } m_segments.reserve(way.nodes().size() - 1); - return extract_segments_from_way_impl(problem_reporter, way, role_type::outer); + return extract_segments_from_way_impl(problem_reporter, duplicate_nodes, way, role_type::outer); } /** * Extract all segments from all ways that make up this * multipolygon relation and add them to the list. */ - uint32_t extract_segments_from_ways(osmium::area::ProblemReporter* problem_reporter, const osmium::Relation& relation, const std::vector& members) { + uint32_t extract_segments_from_ways(osmium::area::ProblemReporter* problem_reporter, uint64_t& duplicate_nodes, const osmium::Relation& relation, const std::vector& members) { assert(relation.members().size() >= members.size()); const size_t num_segments = get_num_segments(members); @@ -234,12 +241,13 @@ namespace osmium { } m_segments.reserve(num_segments); - uint32_t duplicate_nodes = 0; - for_each_member(relation, members, [this, &problem_reporter, &duplicate_nodes](const osmium::RelationMember& member, const osmium::Way& way) { - duplicate_nodes += extract_segments_from_way_impl(problem_reporter, way, parse_role(member.role())); + uint32_t invalid_locations = 0; + for_each_member(relation, members, [&](const osmium::RelationMember& member, const osmium::Way& way) { + const auto role = parse_role(member.role()); + invalid_locations += extract_segments_from_way_impl(problem_reporter, duplicate_nodes, way, role); }); - return duplicate_nodes; + return invalid_locations; } /** diff --git a/contrib/libosmium/osmium/area/detail/vector.hpp b/contrib/libosmium/osmium/area/detail/vector.hpp index 14bf0cb7b..e8ff0a501 100644 --- a/contrib/libosmium/osmium/area/detail/vector.hpp +++ b/contrib/libosmium/osmium/area/detail/vector.hpp @@ -118,4 +118,4 @@ namespace osmium { } // namespace osmium -#endif // OSMIUM_AREA_DETAIL_VECTOR_HPP +#endif // OSMIUM_AREA_DETAIL_VECTOR_HPP diff --git a/contrib/libosmium/osmium/area/geom_assembler.hpp b/contrib/libosmium/osmium/area/geom_assembler.hpp new file mode 100644 index 000000000..84870ae9f --- /dev/null +++ b/contrib/libosmium/osmium/area/geom_assembler.hpp @@ -0,0 +1,131 @@ +#ifndef OSMIUM_AREA_GEOM_ASSEMBLER_HPP +#define OSMIUM_AREA_GEOM_ASSEMBLER_HPP + +/* + +This file is part of Osmium (http://osmcode.org/libosmium). + +Copyright 2013-2017 Jochen Topf and others (see README). + +Boost Software License - Version 1.0 - August 17th, 2003 + +Permission is hereby granted, free of charge, to any person or organization +obtaining a copy of the software and accompanying documentation covered by +this license (the "Software") to use, reproduce, display, distribute, +execute, and transmit the Software, and to prepare derivative works of the +Software, and to permit third-parties to whom the Software is furnished to +do so, all subject to the following: + +The copyright notices in the Software and this entire statement, including +the above license grant, this restriction and the following disclaimer, +must be included in all copies of the Software, in whole or in part, and +all derivative works of the Software, unless such copies or derivative +works are solely in the form of machine-executable object code generated by +a source language processor. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT +SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE +FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, +ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. + +*/ + +#include +#include +#include +#include +#include +#include +#include + +namespace osmium { + + namespace area { + + /** + * Assembles area objects from closed ways or multipolygon relations + * and their members. Unlike the Assembler, this one doesn't take + * tags into account at all. And it doesn't interpret all the config + * settings and doesn't do all the checks and error reporting the + * Assembler does. + * + * This class was developed specifically for the need of osm2pgsql. + * Unless you know what you are doing, use the Assembler class instead + * of this class. Contact the Libosmium developers if you want to use + * this class. + */ + class GeomAssembler : public detail::BasicAssembler { + + public: + + using config_type = osmium::area::AssemblerConfig; + + explicit GeomAssembler(const config_type& config) : + detail::BasicAssembler(config) { + } + + ~GeomAssembler() noexcept = default; + + /** + * Assemble an area from the given way. + * + * The resulting area is put into the out_buffer. + * + * @returns false if there was some kind of error building the + * area, true otherwise. + */ + bool operator()(const osmium::Way& way, osmium::memory::Buffer& out_buffer) { + segment_list().extract_segments_from_way(config().problem_reporter, stats().duplicate_nodes, way); + + if (!create_rings()) { + return false; + } + + { + osmium::builder::AreaBuilder builder{out_buffer}; + builder.initialize_from_object(way); + add_rings_to_area(builder); + } + out_buffer.commit(); + + return true; + } + + /** + * Assemble an area from the given relation and its member ways + * which are in the ways_buffer. + * + * The resulting area is put into the out_buffer. + * + * @returns false if there was some kind of error building the + * area, true otherwise. + */ + bool operator()(const osmium::Relation& relation, const osmium::memory::Buffer& ways_buffer, osmium::memory::Buffer& out_buffer) { + for (const auto& way : ways_buffer.select()) { + segment_list().extract_segments_from_way(config().problem_reporter, stats().duplicate_nodes, way); + } + + if (!create_rings()) { + return false; + } + + { + osmium::builder::AreaBuilder builder{out_buffer}; + builder.initialize_from_object(relation); + add_rings_to_area(builder); + } + out_buffer.commit(); + + return true; + } + + }; // class GeomAssembler + + } // namespace area + +} // namespace osmium + +#endif // OSMIUM_AREA_GEOM_ASSEMBLER_HPP diff --git a/contrib/libosmium/osmium/area/problem_reporter.hpp b/contrib/libosmium/osmium/area/problem_reporter.hpp index 8a72c1ad9..373403b93 100644 --- a/contrib/libosmium/osmium/area/problem_reporter.hpp +++ b/contrib/libosmium/osmium/area/problem_reporter.hpp @@ -189,6 +189,15 @@ namespace osmium { virtual void report_inner_with_same_tags(const osmium::Way& way) { } + /** + * Report an invalid location in a way. + * + * @param way_id ID of the way the node is in. + * @param node_id ID of the node with the invalid location. + */ + virtual void report_invalid_location(osmium::object_id_type way_id, osmium::object_id_type node_id) { + } + /** * In addition to reporting specific problems, this is used to * report all ways belonging to areas having problems. diff --git a/contrib/libosmium/osmium/area/problem_reporter_exception.hpp b/contrib/libosmium/osmium/area/problem_reporter_exception.hpp index 152be40a9..cc5eb98bd 100644 --- a/contrib/libosmium/osmium/area/problem_reporter_exception.hpp +++ b/contrib/libosmium/osmium/area/problem_reporter_exception.hpp @@ -60,58 +60,64 @@ namespace osmium { ~ProblemReporterException() override = default; void report_duplicate_node(osmium::object_id_type node_id1, osmium::object_id_type node_id2, osmium::Location location) override { - m_sstream.str(); + m_sstream.str(""); ProblemReporterStream::report_duplicate_node(node_id1, node_id2, location); - throw std::runtime_error(m_sstream.str()); + throw std::runtime_error{m_sstream.str()}; } void report_touching_ring(osmium::object_id_type node_id, osmium::Location location) override { - m_sstream.str(); + m_sstream.str(""); ProblemReporterStream::report_touching_ring(node_id, location); - throw std::runtime_error(m_sstream.str()); + throw std::runtime_error{m_sstream.str()}; } void report_intersection(osmium::object_id_type way1_id, osmium::Location way1_seg_start, osmium::Location way1_seg_end, osmium::object_id_type way2_id, osmium::Location way2_seg_start, osmium::Location way2_seg_end, osmium::Location intersection) override { - m_sstream.str(); + m_sstream.str(""); ProblemReporterStream::report_intersection(way1_id, way1_seg_start, way1_seg_end, way2_id, way2_seg_start, way2_seg_end, intersection); - throw std::runtime_error(m_sstream.str()); + throw std::runtime_error{m_sstream.str()}; } void report_duplicate_segment(const osmium::NodeRef& nr1, const osmium::NodeRef& nr2) override { - m_sstream.str(); + m_sstream.str(""); ProblemReporterStream::report_duplicate_segment(nr1, nr2); - throw std::runtime_error(m_sstream.str()); + throw std::runtime_error{m_sstream.str()}; } void report_ring_not_closed(const osmium::NodeRef& nr, const osmium::Way* way = nullptr) override { - m_sstream.str(); + m_sstream.str(""); ProblemReporterStream::report_ring_not_closed(nr, way); - throw std::runtime_error(m_sstream.str()); + throw std::runtime_error{m_sstream.str()}; } void report_role_should_be_outer(osmium::object_id_type way_id, osmium::Location seg_start, osmium::Location seg_end) override { - m_sstream.str(); + m_sstream.str(""); ProblemReporterStream::report_role_should_be_outer(way_id, seg_start, seg_end); - throw std::runtime_error(m_sstream.str()); + throw std::runtime_error{m_sstream.str()}; } void report_role_should_be_inner(osmium::object_id_type way_id, osmium::Location seg_start, osmium::Location seg_end) override { - m_sstream.str(); + m_sstream.str(""); ProblemReporterStream::report_role_should_be_inner(way_id, seg_start, seg_end); - throw std::runtime_error(m_sstream.str()); + throw std::runtime_error{m_sstream.str()}; } void report_way_in_multiple_rings(const osmium::Way& way) override { - m_sstream.str(); + m_sstream.str(""); ProblemReporterStream::report_way_in_multiple_rings(way); - throw std::runtime_error(m_sstream.str()); + throw std::runtime_error{m_sstream.str()}; } void report_inner_with_same_tags(const osmium::Way& way) override { - m_sstream.str(); + m_sstream.str(""); ProblemReporterStream::report_inner_with_same_tags(way); - throw std::runtime_error(m_sstream.str()); + throw std::runtime_error{m_sstream.str()}; + } + + void report_invalid_location(osmium::object_id_type way_id, osmium::object_id_type node_id) override { + m_sstream.str(""); + ProblemReporterStream::report_invalid_location(way_id, node_id); + throw std::runtime_error{m_sstream.str()}; } }; // class ProblemReporterException diff --git a/contrib/libosmium/osmium/area/problem_reporter_ogr.hpp b/contrib/libosmium/osmium/area/problem_reporter_ogr.hpp index 914c8bad8..2d8026a9f 100644 --- a/contrib/libosmium/osmium/area/problem_reporter_ogr.hpp +++ b/contrib/libosmium/osmium/area/problem_reporter_ogr.hpp @@ -73,14 +73,14 @@ namespace osmium { gdalcpp::Layer m_layer_ways; void set_object(gdalcpp::Feature& feature) { - const char t[2] = { osmium::item_type_to_char(m_object_type), '\0' }; + const char t[2] = {osmium::item_type_to_char(m_object_type), '\0'}; feature.set_field("obj_type", t); feature.set_field("obj_id", int32_t(m_object_id)); feature.set_field("nodes", int32_t(m_nodes)); } void write_point(const char* problem_type, osmium::object_id_type id1, osmium::object_id_type id2, osmium::Location location) { - gdalcpp::Feature feature(m_layer_perror, m_ogr_factory.create_point(location)); + gdalcpp::Feature feature{m_layer_perror, m_ogr_factory.create_point(location)}; set_object(feature); feature.set_field("id1", double(id1)); feature.set_field("id2", double(id2)); @@ -93,7 +93,7 @@ namespace osmium { ogr_linestring->addPoint(loc1.lon(), loc1.lat()); ogr_linestring->addPoint(loc2.lon(), loc2.lat()); - gdalcpp::Feature feature(m_layer_lerror, std::move(ogr_linestring)); + gdalcpp::Feature feature{m_layer_lerror, std::move(ogr_linestring)}; set_object(feature); feature.set_field("id1", static_cast(id1)); feature.set_field("id2", static_cast(id2)); @@ -175,7 +175,7 @@ namespace osmium { return; } try { - gdalcpp::Feature feature(m_layer_lerror, m_ogr_factory.create_linestring(way)); + gdalcpp::Feature feature{m_layer_lerror, m_ogr_factory.create_linestring(way)}; set_object(feature); feature.set_field("id1", int32_t(way.id())); feature.set_field("id2", 0); @@ -191,7 +191,7 @@ namespace osmium { return; } try { - gdalcpp::Feature feature(m_layer_lerror, m_ogr_factory.create_linestring(way)); + gdalcpp::Feature feature{m_layer_lerror, m_ogr_factory.create_linestring(way)}; set_object(feature); feature.set_field("id1", int32_t(way.id())); feature.set_field("id2", 0); @@ -212,7 +212,7 @@ namespace osmium { return; } try { - gdalcpp::Feature feature(m_layer_ways, m_ogr_factory.create_linestring(way)); + gdalcpp::Feature feature{m_layer_ways, m_ogr_factory.create_linestring(way)}; set_object(feature); feature.set_field("way_id", int32_t(way.id())); feature.add_to_layer(); diff --git a/contrib/libosmium/osmium/area/problem_reporter_stream.hpp b/contrib/libosmium/osmium/area/problem_reporter_stream.hpp index 0c008f131..910eb4ec0 100644 --- a/contrib/libosmium/osmium/area/problem_reporter_stream.hpp +++ b/contrib/libosmium/osmium/area/problem_reporter_stream.hpp @@ -114,6 +114,11 @@ namespace osmium { *m_out << "way_id=" << way.id() << '\n'; } + void report_invalid_location(osmium::object_id_type way_id, osmium::object_id_type node_id) override { + header("invalid location"); + *m_out << "way_id=" << way_id << " node_id=" << node_id << '\n'; + } + }; // class ProblemReporterStream } // namespace area diff --git a/contrib/libosmium/osmium/area/stats.hpp b/contrib/libosmium/osmium/area/stats.hpp index 07be78376..48b40882b 100644 --- a/contrib/libosmium/osmium/area/stats.hpp +++ b/contrib/libosmium/osmium/area/stats.hpp @@ -68,6 +68,7 @@ namespace osmium { uint64_t touching_rings = 0; ///< Rings touching in a node uint64_t ways_in_multiple_rings = 0; ///< Different segments of a way ended up in different rings uint64_t wrong_role = 0; ///< Member has wrong role (not "outer", "inner", or empty) + uint64_t invalid_locations = 0; ///< Invalid location found area_stats& operator+=(const area_stats& other) noexcept { area_really_complex_case += other.area_really_complex_case; @@ -91,6 +92,7 @@ namespace osmium { touching_rings += other.touching_rings; ways_in_multiple_rings += other.ways_in_multiple_rings; wrong_role += other.wrong_role; + invalid_locations += invalid_locations; return *this; } @@ -118,7 +120,8 @@ namespace osmium { << " single_way_in_mp_relation=" << s.single_way_in_mp_relation << " touching_rings=" << s.touching_rings << " ways_in_multiple_rings=" << s.ways_in_multiple_rings - << " wrong_role=" << s.wrong_role; + << " wrong_role=" << s.wrong_role + << " invalid_locations=" << s.invalid_locations; } } // namespace area diff --git a/contrib/libosmium/osmium/builder/builder.hpp b/contrib/libosmium/osmium/builder/builder.hpp index 107f4a9d0..d4358843b 100644 --- a/contrib/libosmium/osmium/builder/builder.hpp +++ b/contrib/libosmium/osmium/builder/builder.hpp @@ -167,6 +167,20 @@ namespace osmium { return length; } + /** + * Append data to buffer and append an additional \0. + * + * @param data Pointer to data. + * @param length Length of data in bytes. + * @returns The number of bytes appended (length + 1). + */ + osmium::memory::item_size_type append_with_zero(const char* data, const osmium::memory::item_size_type length) { + unsigned char* target = reserve_space(length + 1); + std::copy_n(reinterpret_cast(data), length, target); + target[length] = '\0'; + return length + 1; + } + /** * Append \0-terminated string to buffer. * @@ -180,9 +194,11 @@ namespace osmium { /** * Append '\0' to the buffer. * + * @deprecated Use append_with_zero() instead. + * * @returns The number of bytes appended (always 1). */ - osmium::memory::item_size_type append_zero() { + OSMIUM_DEPRECATED osmium::memory::item_size_type append_zero() { *reserve_space(1) = '\0'; return 1; } diff --git a/contrib/libosmium/osmium/builder/builder_helper.hpp b/contrib/libosmium/osmium/builder/builder_helper.hpp index 8bfa21e24..a273e5041 100644 --- a/contrib/libosmium/osmium/builder/builder_helper.hpp +++ b/contrib/libosmium/osmium/builder/builder_helper.hpp @@ -71,7 +71,7 @@ namespace osmium { * @deprecated * Use osmium::builder::add_tag_list() instead. */ - inline const osmium::TagList& build_tag_list(osmium::memory::Buffer& buffer, const std::initializer_list>& tags) { + OSMIUM_DEPRECATED inline const osmium::TagList& build_tag_list(osmium::memory::Buffer& buffer, const std::initializer_list>& tags) { const size_t pos = buffer.committed(); { osmium::builder::TagListBuilder tl_builder(buffer); @@ -87,7 +87,7 @@ namespace osmium { * @deprecated * Use osmium::builder::add_tag_list() instead. */ - inline const osmium::TagList& build_tag_list_from_map(osmium::memory::Buffer& buffer, const std::map& tags) { + OSMIUM_DEPRECATED inline const osmium::TagList& build_tag_list_from_map(osmium::memory::Buffer& buffer, const std::map& tags) { const size_t pos = buffer.committed(); { osmium::builder::TagListBuilder tl_builder(buffer); @@ -103,7 +103,7 @@ namespace osmium { * @deprecated * Use osmium::builder::add_tag_list() instead. */ - inline const osmium::TagList& build_tag_list_from_func(osmium::memory::Buffer& buffer, std::function func) { + OSMIUM_DEPRECATED inline const osmium::TagList& build_tag_list_from_func(osmium::memory::Buffer& buffer, std::function func) { const size_t pos = buffer.committed(); { osmium::builder::TagListBuilder tl_builder(buffer); diff --git a/contrib/libosmium/osmium/builder/osm_object_builder.hpp b/contrib/libosmium/osmium/builder/osm_object_builder.hpp index 1dd5f972f..d4483b192 100644 --- a/contrib/libosmium/osmium/builder/osm_object_builder.hpp +++ b/contrib/libosmium/osmium/builder/osm_object_builder.hpp @@ -99,7 +99,8 @@ namespace osmium { if (std::strlen(value) > osmium::max_osm_string_length) { throw std::length_error("OSM tag value is too long"); } - add_size(append(key) + append(value)); + add_size(append(key)); + add_size(append(value)); } /** @@ -117,8 +118,8 @@ namespace osmium { if (value_length > osmium::max_osm_string_length) { throw std::length_error("OSM tag value is too long"); } - add_size(append(key, osmium::memory::item_size_type(key_length)) + append_zero() + - append(value, osmium::memory::item_size_type(value_length)) + append_zero()); + add_size(append_with_zero(key, osmium::memory::item_size_type(key_length))); + add_size(append_with_zero(value, osmium::memory::item_size_type(value_length))); } /** @@ -134,8 +135,8 @@ namespace osmium { if (value.size() > osmium::max_osm_string_length) { throw std::length_error("OSM tag value is too long"); } - add_size(append(key.data(), osmium::memory::item_size_type(key.size()) + 1) + - append(value.data(), osmium::memory::item_size_type(value.size()) + 1)); + add_size(append(key.data(), osmium::memory::item_size_type(key.size()) + 1)); + add_size(append(value.data(), osmium::memory::item_size_type(value.size()) + 1)); } /** @@ -144,7 +145,8 @@ namespace osmium { * @param tag Tag. */ void add_tag(const osmium::Tag& tag) { - add_size(append(tag.key()) + append(tag.value())); + add_size(append(tag.key())); + add_size(append(tag.value())); } /** @@ -226,7 +228,7 @@ namespace osmium { throw std::length_error("OSM relation member role is too long"); } member.set_role_size(osmium::string_size_type(length) + 1); - add_size(append(role, osmium::memory::item_size_type(length)) + append_zero()); + add_size(append_with_zero(role, osmium::memory::item_size_type(length))); add_padding(true); } @@ -310,7 +312,7 @@ namespace osmium { throw std::length_error("OSM user name is too long"); } comment.set_user_size(osmium::string_size_type(length) + 1); - add_size(append(user, osmium::memory::item_size_type(length)) + append_zero()); + add_size(append_with_zero(user, osmium::memory::item_size_type(length))); } void add_text(osmium::ChangesetComment& comment, const char* text, const size_t length) { @@ -322,7 +324,7 @@ namespace osmium { throw std::length_error("OSM changeset comment is too long"); } comment.set_text_size(osmium::string_size_type(length) + 1); - add_size(append(text, osmium::memory::item_size_type(length)) + append_zero()); + add_size(append_with_zero(text, osmium::memory::item_size_type(length))); add_padding(true); } diff --git a/contrib/libosmium/osmium/geom/factory.hpp b/contrib/libosmium/osmium/geom/factory.hpp index 6f8d954b0..39d1eacea 100644 --- a/contrib/libosmium/osmium/geom/factory.hpp +++ b/contrib/libosmium/osmium/geom/factory.hpp @@ -192,7 +192,7 @@ namespace osmium { using multipolygon_type = typename TGeomImpl::multipolygon_type; using ring_type = typename TGeomImpl::ring_type; - int epsg() const { + int epsg() const noexcept { return m_projection.epsg(); } diff --git a/contrib/libosmium/osmium/geom/mercator_projection.hpp b/contrib/libosmium/osmium/geom/mercator_projection.hpp index 814bce632..a6b798aa2 100644 --- a/contrib/libosmium/osmium/geom/mercator_projection.hpp +++ b/contrib/libosmium/osmium/geom/mercator_projection.hpp @@ -118,7 +118,7 @@ namespace osmium { * @pre @code c.valid() @endcode */ inline Coordinates lonlat_to_mercator(const Coordinates& c) { - return Coordinates(detail::lon_to_x(c.x), detail::lat_to_y(c.y)); + return Coordinates{detail::lon_to_x(c.x), detail::lat_to_y(c.y)}; } /** @@ -127,7 +127,7 @@ namespace osmium { * @pre @code c.valid() @endcode */ inline Coordinates mercator_to_lonlat(const Coordinates& c) { - return Coordinates(detail::x_to_lon(c.x), detail::y_to_lat(c.y)); + return Coordinates{detail::x_to_lon(c.x), detail::y_to_lat(c.y)}; } /** @@ -139,7 +139,7 @@ namespace osmium { public: Coordinates operator()(osmium::Location location) const { - return Coordinates {detail::lon_to_x(location.lon()), detail::lat_to_y(location.lat())}; + return Coordinates{detail::lon_to_x(location.lon()), detail::lat_to_y(location.lat())}; } int epsg() const noexcept { diff --git a/contrib/libosmium/osmium/geom/projection.hpp b/contrib/libosmium/osmium/geom/projection.hpp index 60e0ecf3d..132b0680e 100644 --- a/contrib/libosmium/osmium/geom/projection.hpp +++ b/contrib/libosmium/osmium/geom/projection.hpp @@ -48,6 +48,7 @@ DEALINGS IN THE SOFTWARE. #include #include +#include #include #include @@ -70,15 +71,15 @@ namespace osmium { public: - explicit CRS(const std::string& crs) : - m_crs(pj_init_plus(crs.c_str()), ProjCRSDeleter()) { + explicit CRS(const char* crs) : + m_crs(pj_init_plus(crs), ProjCRSDeleter()) { if (!m_crs) { - throw osmium::projection_error(std::string{"creation of CRS failed: "} + pj_strerrno(*pj_get_errno_ref())); + throw osmium::projection_error{std::string{"creation of CRS failed: "} + pj_strerrno(*pj_get_errno_ref())}; } } - explicit CRS(const char* crs) : - CRS(std::string{crs}) { + explicit CRS(const std::string& crs) : + CRS(crs.c_str()) { } explicit CRS(int epsg) : @@ -88,15 +89,15 @@ namespace osmium { /** * Get underlying projPJ handle from proj library. */ - projPJ get() const { + projPJ get() const noexcept { return m_crs.get(); } - bool is_latlong() const { + bool is_latlong() const noexcept { return pj_is_latlong(m_crs.get()) != 0; } - bool is_geocent() const { + bool is_geocent() const noexcept { return pj_is_geocent(m_crs.get()) != 0; } @@ -108,12 +109,12 @@ namespace osmium { * * Coordinates have to be in radians and are produced in radians. * - * @throws osmmium::projection_error if the projection fails + * @throws osmium::projection_error if the projection fails */ inline Coordinates transform(const CRS& src, const CRS& dest, Coordinates c) { - int result = pj_transform(src.get(), dest.get(), 1, 1, &c.x, &c.y, nullptr); + const int result = pj_transform(src.get(), dest.get(), 1, 1, &c.x, &c.y, nullptr); if (result != 0) { - throw osmium::projection_error(std::string("projection failed: ") + pj_strerrno(result)); + throw osmium::projection_error{std::string("projection failed: ") + pj_strerrno(result)}; } return c; } @@ -121,12 +122,19 @@ namespace osmium { /** * Functor that does projection from WGS84 (EPSG:4326) to the given * CRS. + * + * If this Projection is initialized with the constructor taking + * an integer with the epsg code 4326, no projection is done. If it + * is initialized with epsg code 3857 the Osmium-internal + * implementation of the Mercator projection is used, otherwise this + * falls back to using the proj.4 library. Note that this "magic" does + * not work if you use any of the constructors taking a string. */ class Projection { int m_epsg; std::string m_proj_string; - CRS m_crs_wgs84 {4326}; + CRS m_crs_wgs84{4326}; CRS m_crs_user; public: @@ -145,19 +153,23 @@ namespace osmium { explicit Projection(int epsg) : m_epsg(epsg), - m_proj_string(std::string("+init=epsg:") + std::to_string(epsg)), + m_proj_string(std::string{"+init=epsg:"} + std::to_string(epsg)), m_crs_user(epsg) { } Coordinates operator()(osmium::Location location) const { - Coordinates c {location.lon(), location.lat()}; - - if (m_epsg != 4326) { - c = transform(m_crs_wgs84, m_crs_user, Coordinates(deg_to_rad(location.lon()), deg_to_rad(location.lat()))); - if (m_crs_user.is_latlong()) { - c.x = rad_to_deg(c.x); - c.y = rad_to_deg(c.y); - } + if (m_epsg == 4326) { + return Coordinates{location.lon(), location.lat()}; + } else if (m_epsg == 3857) { + return Coordinates{detail::lon_to_x(location.lon()), + detail::lat_to_y(location.lat())}; + } + + Coordinates c{transform(m_crs_wgs84, m_crs_user, Coordinates{deg_to_rad(location.lon()), + deg_to_rad(location.lat())})}; + if (m_crs_user.is_latlong()) { + c.x = rad_to_deg(c.x); + c.y = rad_to_deg(c.y); } return c; diff --git a/contrib/libosmium/osmium/index/id_set.hpp b/contrib/libosmium/osmium/index/id_set.hpp index 50426b006..9f3631439 100644 --- a/contrib/libosmium/osmium/index/id_set.hpp +++ b/contrib/libosmium/osmium/index/id_set.hpp @@ -408,6 +408,7 @@ namespace osmium { }; // class IdSetSmall + /// @deprecated Use nrw_array helper class instead. template class IdSetType> class NWRIdSet { diff --git a/contrib/libosmium/osmium/index/map/sparse_mem_table.hpp b/contrib/libosmium/osmium/index/map/sparse_mem_table.hpp index f6a5fa34f..e2414624c 100644 --- a/contrib/libosmium/osmium/index/map/sparse_mem_table.hpp +++ b/contrib/libosmium/osmium/index/map/sparse_mem_table.hpp @@ -152,4 +152,4 @@ namespace osmium { #endif // OSMIUM_WITH_SPARSEHASH -#endif // OSMIUM_INDEX_BYID_SPARSE_MEM_TABLE_HPP +#endif // OSMIUM_INDEX_MAP_SPARSE_MEM_TABLE_HPP diff --git a/contrib/libosmium/osmium/index/nwr_array.hpp b/contrib/libosmium/osmium/index/nwr_array.hpp new file mode 100644 index 000000000..2c52c7e8f --- /dev/null +++ b/contrib/libosmium/osmium/index/nwr_array.hpp @@ -0,0 +1,59 @@ +#ifndef OSMIUM_INDEX_NWR_ARRAY_HPP +#define OSMIUM_INDEX_NWR_ARRAY_HPP + +/* + +This file is part of Osmium (http://osmcode.org/libosmium). + +Copyright 2013-2017 Jochen Topf and others (see README). + +Boost Software License - Version 1.0 - August 17th, 2003 + +Permission is hereby granted, free of charge, to any person or organization +obtaining a copy of the software and accompanying documentation covered by +this license (the "Software") to use, reproduce, display, distribute, +execute, and transmit the Software, and to prepare derivative works of the +Software, and to permit third-parties to whom the Software is furnished to +do so, all subject to the following: + +The copyright notices in the Software and this entire statement, including +the above license grant, this restriction and the following disclaimer, +must be included in all copies of the Software, in whole or in part, and +all derivative works of the Software, unless such copies or derivative +works are solely in the form of machine-executable object code generated by +a source language processor. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT +SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE +FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, +ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. + +*/ + +#include + +namespace osmium { + + template + class nwr_array { + + T m_sets[3]; + + public: + + T& operator()(osmium::item_type type) noexcept { + return m_sets[osmium::item_type_to_nwr_index(type)]; + } + + const T& operator()(osmium::item_type type) const noexcept { + return m_sets[osmium::item_type_to_nwr_index(type)]; + } + + }; // class nwr_array + +} // namespace osmium + +#endif // OSMIUM_INDEX_NWR_ARRAY_HPP diff --git a/contrib/libosmium/osmium/index/relations_map.hpp b/contrib/libosmium/osmium/index/relations_map.hpp index 5f46e6609..bb9f5a7ce 100644 --- a/contrib/libosmium/osmium/index/relations_map.hpp +++ b/contrib/libosmium/osmium/index/relations_map.hpp @@ -91,6 +91,24 @@ namespace osmium { m_map.emplace_back(key, value); } + typename std::enable_if::value>::type flip_in_place() { + for (auto& p : m_map) { + using std::swap; + swap(p.key, p.value); + } + } + + flat_map flip_copy() { + flat_map map; + map.reserve(m_map.size()); + + for (const auto& p : m_map) { + map.set(p.value, p.key); + } + + return map; + } + void sort_unique() { std::sort(m_map.begin(), m_map.end()); const auto last = std::unique(m_map.begin(), m_map.end()); @@ -111,39 +129,46 @@ namespace osmium { return m_map.size(); } + void reserve(size_t size) { + m_map.reserve(size); + } + }; // class flat_map } // namespace detail /** - * Index for looking up parent relation IDs given a member relation ID. - * You can not instantiate such an index yourself, instead you need to - * instantiate a RelationsMapStash, fill it and then create an index from - * it: - * - * @code - * RelationsMapStash stash; - * ... - * for_each_relation(const osmium::Relation& relation) { - * stash.add_members(relation); - * } - * ... - * const auto index = stash.build_index(); - * ... - * osmium::unsigned_object_id_type member_id = ...; - * index.for_each_parent(member_id, [](osmium::unsigned_object_id_type id) { - * ... - * }); - * ... - * @endcode - * - */ + * Index for looking up parent relation IDs given a member relation ID + * or the other way around. + * + * You can not instantiate such an index yourself, instead you need to + * instantiate a RelationsMapStash, fill it and then create an index + * from it: + * + * @code + * RelationsMapStash stash; + * ... + * for_each_relation(const osmium::Relation& relation) { + * stash.add_members(relation); + * } + * ... + * const auto index = stash.build_member_to_parent_index(); + * ... + * osmium::unsigned_object_id_type member_id = ...; + * index.for_each(member_id, [](osmium::unsigned_object_id_type parent_id) { + * ... + * }); + * ... + * @endcode + * + */ class RelationsMapIndex { friend class RelationsMapStash; + friend class RelationsMapIndexes; using map_type = detail::flat_map; + osmium::unsigned_object_id_type, uint32_t>; map_type m_map; @@ -162,56 +187,122 @@ namespace osmium { RelationsMapIndex& operator=(RelationsMapIndex&&) = default; /** - * Find the given relation id in the index and call the given function - * with all parent relation ids. - * - * @code - * osmium::unsigned_object_id_type member_id = 17; - * index.for_each_parent(member_id, [](osmium::unsigned_object_id_type id) { - * ... - * }); - * @endcode - * - * Complexity: Logarithmic in the number of elements in the index. - * (Lookup uses binary search.) - */ - template - void for_each_parent(osmium::unsigned_object_id_type member_id, Func&& func) const { + * Find the given relation id in the index and call the given + * function with all parent relation ids. + * + * @code + * osmium::unsigned_object_id_type member_id = 17; + * index.for_each_parent(member_id, [](osmium::unsigned_object_id_type id) { + * ... + * }); + * @endcode + * + * @deprecated Use for_each() instead. + * + * Complexity: Logarithmic in the number of elements in the index. + * (Lookup uses binary search.) + */ + template + void for_each_parent(osmium::unsigned_object_id_type member_id, TFunc&& func) const { const auto parents = m_map.get(member_id); for (auto it = parents.first; it != parents.second; ++it) { - std::forward(func)(it->value); + std::forward(func)(it->value); + } + } + + /** + * Find the given relation id in the index and call the given + * function with all related relation ids. + * + * @code + * osmium::unsigned_object_id_type id = 17; + * index.for_each(id, [](osmium::unsigned_object_id_type rid) { + * ... + * }); + * @endcode + * + * Complexity: Logarithmic in the number of elements in the index. + * (Lookup uses binary search.) + */ + template + void for_each(osmium::unsigned_object_id_type id, TFunc&& func) const { + const auto parents = m_map.get(id); + for (auto it = parents.first; it != parents.second; ++it) { + std::forward(func)(it->value); } } /** - * Is this index empty? - * - * Complexity: Constant. - */ + * Is this index empty? + * + * Complexity: Constant. + */ bool empty() const noexcept { return m_map.empty(); } /** - * How many entries are in this index? - * - * Complexity: Constant. - */ + * How many entries are in this index? + * + * Complexity: Constant. + */ size_t size() const noexcept { return m_map.size(); } - }; // RelationsMapIndex + }; // class RelationsMapIndex + + class RelationsMapIndexes { + + friend class RelationsMapStash; + + RelationsMapIndex m_member_to_parent; + RelationsMapIndex m_parent_to_member; + + RelationsMapIndexes(RelationsMapIndex::map_type&& map1, RelationsMapIndex::map_type&& map2) : + m_member_to_parent(std::move(map1)), + m_parent_to_member(std::move(map2)) { + } + + public: + + const RelationsMapIndex& member_to_parent() const noexcept { + return m_member_to_parent; + } + + const RelationsMapIndex& parent_to_member() const noexcept { + return m_parent_to_member; + } + + /** + * Is this index empty? + * + * Complexity: Constant. + */ + bool empty() const noexcept { + return m_member_to_parent.empty(); + } + + /** + * How many entries are in this index? + * + * Complexity: Constant. + */ + size_t size() const noexcept { + return m_member_to_parent.size(); + } + + }; // class RelationsMapIndexes /** - * The RelationsMapStash is used to build up the data needed to create - * an index of member relation ID to parent relation ID. See the - * RelationsMapIndex class for more. - */ + * The RelationsMapStash is used to build up the data needed to create + * an index of member relation ID to parent relation ID or the other + * way around. See the RelationsMapIndex class for more. + */ class RelationsMapStash { using map_type = detail::flat_map; + osmium::unsigned_object_id_type, uint32_t>; map_type m_map; @@ -230,16 +321,16 @@ namespace osmium { RelationsMapStash& operator=(RelationsMapStash&&) = default; /** - * Add mapping from member to parent relation in the stash. - */ + * Add mapping from member to parent relation in the stash. + */ void add(osmium::unsigned_object_id_type member_id, osmium::unsigned_object_id_type relation_id) { assert(m_valid && "You can't use the RelationsMap any more after calling build_index()"); m_map.set(member_id, relation_id); } /** - * Add mapping from all members to given parent relation in the stash. - */ + * Add mapping from all members to given parent relation in the stash. + */ void add_members(const osmium::Relation& relation) { assert(m_valid && "You can't use the RelationsMap any more after calling build_index()"); for (const auto& member : relation.members()) { @@ -250,30 +341,33 @@ namespace osmium { } /** - * Is this stash empty? - * - * Complexity: Constant. - */ + * Is this stash empty? + * + * Complexity: Constant. + */ bool empty() const noexcept { assert(m_valid && "You can't use the RelationsMap any more after calling build_index()"); return m_map.empty(); } /** - * How many entries are in this stash? - * - * Complexity: Constant. - */ + * How many entries are in this stash? + * + * Complexity: Constant. + */ size_t size() const noexcept { assert(m_valid && "You can't use the RelationsMap any more after calling build_index()"); return m_map.size(); } /** - * Build an index with the contents of this stash and return it. - * - * After you get the index you can not use the stash any more! - */ + * Build an index for member to parent lookups from the contents + * of this stash and return it. + * + * After you get the index you can not use the stash any more! + * + * @deprecated Use build_member_to_parent_index() instead. + */ RelationsMapIndex build_index() { assert(m_valid && "You can't use the RelationsMap any more after calling build_index()"); m_map.sort_unique(); @@ -283,6 +377,54 @@ namespace osmium { return RelationsMapIndex{std::move(m_map)}; } + /** + * Build an index for member to parent lookups from the contents + * of this stash and return it. + * + * After you get the index you can not use the stash any more! + */ + RelationsMapIndex build_member_to_parent_index() { + assert(m_valid && "You can't use the RelationsMap any more after calling build_member_to_parent_index()"); + m_map.sort_unique(); +#ifndef NDEBUG + m_valid = false; +#endif + return RelationsMapIndex{std::move(m_map)}; + } + + /** + * Build an index for parent to member lookups from the contents + * of this stash and return it. + * + * After you get the index you can not use the stash any more! + */ + RelationsMapIndex build_parent_to_member_index() { + assert(m_valid && "You can't use the RelationsMap any more after calling build_parent_to_member_index()"); + m_map.flip_in_place(); + m_map.sort_unique(); +#ifndef NDEBUG + m_valid = false; +#endif + return RelationsMapIndex{std::move(m_map)}; + } + + /** + * Build indexes for member-to-parent and parent-to-member lookups + * from the contents of this stash and return them. + * + * After you get the index you can not use the stash any more! + */ + RelationsMapIndexes build_indexes() { + assert(m_valid && "You can't use the RelationsMap any more after calling build_indexes()"); + auto reverse_map = m_map.flip_copy(); + reverse_map.sort_unique(); + m_map.sort_unique(); +#ifndef NDEBUG + m_valid = false; +#endif + return RelationsMapIndexes{std::move(m_map), std::move(reverse_map)}; + } + }; // class RelationsMapStash } // namespace index diff --git a/contrib/libosmium/osmium/io/detail/pbf_input_format.hpp b/contrib/libosmium/osmium/io/detail/pbf_input_format.hpp index be5f86097..74fc4f89d 100644 --- a/contrib/libosmium/osmium/io/detail/pbf_input_format.hpp +++ b/contrib/libosmium/osmium/io/detail/pbf_input_format.hpp @@ -33,7 +33,6 @@ DEALINGS IN THE SOFTWARE. */ -#include #include #include #include @@ -42,6 +41,7 @@ DEALINGS IN THE SOFTWARE. #include #include #include +#include #include #include diff --git a/contrib/libosmium/osmium/io/detail/pbf_output_format.hpp b/contrib/libosmium/osmium/io/detail/pbf_output_format.hpp index d431a6796..00b364f6d 100644 --- a/contrib/libosmium/osmium/io/detail/pbf_output_format.hpp +++ b/contrib/libosmium/osmium/io/detail/pbf_output_format.hpp @@ -33,7 +33,6 @@ DEALINGS IN THE SOFTWARE. */ -#include #include #include #include diff --git a/contrib/libosmium/osmium/io/detail/protobuf_tags.hpp b/contrib/libosmium/osmium/io/detail/protobuf_tags.hpp index 8bb2e56c6..58fff5694 100644 --- a/contrib/libosmium/osmium/io/detail/protobuf_tags.hpp +++ b/contrib/libosmium/osmium/io/detail/protobuf_tags.hpp @@ -169,4 +169,4 @@ namespace osmium { } // namespace osmium -#endif // OSMIUM_IO_DETAIL_PROTOBUF_TAGS_HPP +#endif // OSMIUM_IO_DETAIL_PROTOBUF_TAGS_HPP diff --git a/contrib/libosmium/osmium/io/detail/write_thread.hpp b/contrib/libosmium/osmium/io/detail/write_thread.hpp index 1e22155f4..1b0745184 100644 --- a/contrib/libosmium/osmium/io/detail/write_thread.hpp +++ b/contrib/libosmium/osmium/io/detail/write_thread.hpp @@ -33,11 +33,11 @@ DEALINGS IN THE SOFTWARE. */ -#include #include #include #include #include +#include #include #include diff --git a/contrib/libosmium/osmium/io/detail/xml_output_format.hpp b/contrib/libosmium/osmium/io/detail/xml_output_format.hpp index 44d68313c..130a3c734 100644 --- a/contrib/libosmium/osmium/io/detail/xml_output_format.hpp +++ b/contrib/libosmium/osmium/io/detail/xml_output_format.hpp @@ -33,7 +33,6 @@ DEALINGS IN THE SOFTWARE. */ -#include #include #include #include diff --git a/contrib/libosmium/osmium/osm/crc.hpp b/contrib/libosmium/osmium/osm/crc.hpp index 5de52ed18..c191db93a 100644 --- a/contrib/libosmium/osmium/osm/crc.hpp +++ b/contrib/libosmium/osmium/osm/crc.hpp @@ -246,4 +246,4 @@ namespace osmium { } // namespace osmium -#endif // OSMIUM_OSM_CRC +#endif // OSMIUM_OSM_CRC_HPP diff --git a/contrib/libosmium/osmium/relations/collector.hpp b/contrib/libosmium/osmium/relations/collector.hpp index bede76405..ee17d47b8 100644 --- a/contrib/libosmium/osmium/relations/collector.hpp +++ b/contrib/libosmium/osmium/relations/collector.hpp @@ -293,17 +293,6 @@ namespace osmium { void flush() { } - /** - * This removes all relations that have already been assembled - * from the m_relations vector. - */ - void clean_assembled_relations() { - m_relations.erase( - std::remove_if(m_relations.begin(), m_relations.end(), has_all_members()), - m_relations.end() - ); - } - const osmium::Relation& get_relation(size_t offset) const { assert(m_relations_buffer.committed() > offset); return m_relations_buffer.get(offset); diff --git a/contrib/libosmium/osmium/tags/filter.hpp b/contrib/libosmium/osmium/tags/filter.hpp index 22abb1430..e8f3da806 100644 --- a/contrib/libosmium/osmium/tags/filter.hpp +++ b/contrib/libosmium/osmium/tags/filter.hpp @@ -34,6 +34,7 @@ DEALINGS IN THE SOFTWARE. */ #include +#include #include #include #include @@ -49,27 +50,41 @@ namespace osmium { template struct match_key { - bool operator()(const TKey& rule_key, const char* tag_key) { + bool operator()(const TKey& rule_key, const char* tag_key) const { return rule_key == tag_key; } }; // struct match_key + template <> + struct match_key { + bool operator()(const std::string& rule_key, const char* tag_key) const { + return !std::strcmp(rule_key.c_str(), tag_key); + } + }; // struct match_key + struct match_key_prefix { - bool operator()(const std::string& rule_key, const char* tag_key) { + bool operator()(const std::string& rule_key, const char* tag_key) const { return rule_key.compare(0, std::string::npos, tag_key, 0, rule_key.size()) == 0; } }; // struct match_key_prefix template struct match_value { - bool operator()(const TValue& rule_value, const char* tag_value) { + bool operator()(const TValue& rule_value, const char* tag_value) const { return rule_value == tag_value; } }; // struct match_value + template <> + struct match_value { + bool operator()(const std::string& rule_value, const char* tag_value) const { + return !std::strcmp(rule_value.c_str(), tag_value); + } + }; // struct match_value + template <> struct match_value { - bool operator()(const bool, const char*) { + bool operator()(const bool, const char*) const noexcept { return true; } }; // struct match_value @@ -138,15 +153,19 @@ namespace osmium { /** * Return the number of rules in this filter. + * + * Complexity: Constant. */ - size_t count() const { - return m_rules.count(); + size_t count() const noexcept { + return m_rules.size(); } /** * Is this filter empty, ie are there no rules defined? + * + * Complexity: Constant. */ - bool empty() const { + bool empty() const noexcept { return m_rules.empty(); } diff --git a/contrib/libosmium/osmium/tags/regex_filter.hpp b/contrib/libosmium/osmium/tags/regex_filter.hpp index 72c3217e5..16aefe0cc 100644 --- a/contrib/libosmium/osmium/tags/regex_filter.hpp +++ b/contrib/libosmium/osmium/tags/regex_filter.hpp @@ -44,14 +44,14 @@ namespace osmium { template <> struct match_key { - bool operator()(const std::regex& rule_key, const char* tag_key) { + bool operator()(const std::regex& rule_key, const char* tag_key) const { return std::regex_match(tag_key, rule_key); } }; // struct match_key template <> struct match_value { - bool operator()(const std::regex& rule_value, const char* tag_value) { + bool operator()(const std::regex& rule_value, const char* tag_value) const { return std::regex_match(tag_value, rule_value); } }; // struct match_value diff --git a/contrib/libosmium/osmium/tags/taglist.hpp b/contrib/libosmium/osmium/tags/taglist.hpp index 30db9e99c..3f640d6b2 100644 --- a/contrib/libosmium/osmium/tags/taglist.hpp +++ b/contrib/libosmium/osmium/tags/taglist.hpp @@ -34,6 +34,7 @@ DEALINGS IN THE SOFTWARE. */ #include +#include #include // IWYU pragma: keep #include @@ -46,18 +47,18 @@ namespace osmium { namespace tags { template - inline bool match_any_of(const osmium::TagList& tag_list, TFilter&& filter) { - return std::any_of(tag_list.cbegin(), tag_list.cend(), std::forward(filter)); + inline bool match_any_of(const osmium::TagList& tag_list, const TFilter& filter) { + return std::any_of(tag_list.cbegin(), tag_list.cend(), std::cref(filter)); } template - inline bool match_all_of(const osmium::TagList& tag_list, TFilter&& filter) { - return std::all_of(tag_list.cbegin(), tag_list.cend(), std::forward(filter)); + inline bool match_all_of(const osmium::TagList& tag_list, const TFilter& filter) { + return std::all_of(tag_list.cbegin(), tag_list.cend(), std::cref(filter)); } template - inline bool match_none_of(const osmium::TagList& tag_list, TFilter&& filter) { - return std::none_of(tag_list.cbegin(), tag_list.cend(), std::forward(filter)); + inline bool match_none_of(const osmium::TagList& tag_list, const TFilter& filter) { + return std::none_of(tag_list.cbegin(), tag_list.cend(), std::cref(filter)); } } // namespace tags diff --git a/contrib/libosmium/osmium/thread/function_wrapper.hpp b/contrib/libosmium/osmium/thread/function_wrapper.hpp index 2ada084d3..eeb942542 100644 --- a/contrib/libosmium/osmium/thread/function_wrapper.hpp +++ b/contrib/libosmium/osmium/thread/function_wrapper.hpp @@ -33,8 +33,8 @@ DEALINGS IN THE SOFTWARE. */ -#include #include +#include namespace osmium { diff --git a/contrib/libosmium/osmium/thread/pool.hpp b/contrib/libosmium/osmium/thread/pool.hpp index 06d96f52a..a89b3e940 100644 --- a/contrib/libosmium/osmium/thread/pool.hpp +++ b/contrib/libosmium/osmium/thread/pool.hpp @@ -33,11 +33,11 @@ DEALINGS IN THE SOFTWARE. */ -#include #include #include #include #include +#include #include #include diff --git a/contrib/libosmium/osmium/thread/util.hpp b/contrib/libosmium/osmium/thread/util.hpp index 4eb59c9d1..53d16a19e 100644 --- a/contrib/libosmium/osmium/thread/util.hpp +++ b/contrib/libosmium/osmium/thread/util.hpp @@ -115,4 +115,4 @@ namespace osmium { } // namespace osmium -#endif // OSMIUM_THREAD_UTIL_HPP +#endif // OSMIUM_THREAD_UTIL_HPP diff --git a/contrib/libosmium/osmium/util/file.hpp b/contrib/libosmium/osmium/util/file.hpp index a26d88fc1..e4083c0d9 100644 --- a/contrib/libosmium/osmium/util/file.hpp +++ b/contrib/libosmium/osmium/util/file.hpp @@ -70,14 +70,14 @@ namespace osmium { // https://msdn.microsoft.com/en-us/library/dfbc2kec.aspx const auto size = ::_filelengthi64(fd); if (size == -1L) { - throw std::system_error(errno, std::system_category(), "_filelengthi64 failed"); + throw std::system_error{errno, std::system_category(), "Could not get file size"}; } return size_t(size); #else // Unix implementation struct stat s; if (::fstat(fd, &s) != 0) { - throw std::system_error(errno, std::system_category(), "fstat failed"); + throw std::system_error{errno, std::system_category(), "Could not get file size"}; } return size_t(s.st_size); #endif @@ -97,13 +97,13 @@ namespace osmium { // https://msdn.microsoft.com/en-us/library/14h5k7ff.aspx struct _stat64 s; if (::_stati64(name, &s) != 0) { - throw std::system_error(errno, std::system_category(), "_stati64 failed"); + throw std::system_error{errno, std::system_category(), std::string{"Could not get file size of file '"} + name + "'"}; } #else // Unix implementation struct stat s; if (::stat(name, &s) != 0) { - throw std::system_error(errno, std::system_category(), "stat failed"); + throw std::system_error{errno, std::system_category(), std::string{"Could not get file size of file '"} + name + "'"}; } #endif return size_t(s.st_size); @@ -136,7 +136,7 @@ namespace osmium { #else if (::ftruncate(fd, static_cast_with_assert(new_size)) != 0) { #endif - throw std::system_error(errno, std::system_category(), "resizing file failed"); + throw std::system_error{errno, std::system_category(), "Could not resize file"}; } } diff --git a/docs/usage.md b/docs/usage.md index 50fd1af42..39ec8e937 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -144,6 +144,3 @@ reimporting the database, at the cost of a * ``--keep-coastlines`` disables a hard-coded rule that would otherwise discard ``natural=coastline`` ways. - -* ``--exclude-invalid-polygon`` prevents osm2pgsql from attempting to form - valid polygons from invalid ones and just rejects the invalid ones. diff --git a/expire-tiles.cpp b/expire-tiles.cpp index 28c1f3bc7..11f7558a2 100644 --- a/expire-tiles.cpp +++ b/expire-tiles.cpp @@ -16,9 +16,9 @@ #include "expire-tiles.hpp" #include "options.hpp" -#include "geometry-builder.hpp" #include "reprojection.hpp" #include "table.hpp" +#include "wkb.hpp" #define EARTH_CIRCUMFERENCE 40075016.68 #define HALF_EARTH_CIRCUMFERENCE (EARTH_CIRCUMFERENCE / 2) @@ -333,74 +333,114 @@ int expire_tiles::from_bbox(double min_lon, double min_lat, double max_lon, doub return 0; } -void expire_tiles::from_nodes_line(const nodelist_t &nodes) -{ - if (maxzoom < 0 || nodes.empty()) - return; - if (nodes.size() == 1) { - from_bbox(nodes[0].x, nodes[0].y, nodes[0].x, nodes[0].y); - } else { - for (size_t i = 1; i < nodes.size(); ++i) - from_line(nodes[i - 1].x, nodes[i - 1].y, nodes[i].x, nodes[i].y); - } -} - -/* - * Calculate a bounding box from a list of nodes and expire all tiles within it - */ -void expire_tiles::from_nodes_poly(const nodelist_t &nodes, osmid_t osm_id) +void expire_tiles::from_wkb(const char *wkb, osmid_t osm_id) { - if (maxzoom < 0 || nodes.empty()) + if (maxzoom < 0) { return; - - double min_lon = nodes[0].x; - double min_lat = nodes[0].y; - double max_lon = nodes[0].x; - double max_lat = nodes[0].y; - - for (size_t i = 1; i < nodes.size(); ++i) { - if (nodes[i].x < min_lon) - min_lon = nodes[i].x; - if (nodes[i].y < min_lat) - min_lat = nodes[i].y; - if (nodes[i].x > max_lon) - max_lon = nodes[i].x; - if (nodes[i].y > max_lat) - max_lat = nodes[i].y; } - if (from_bbox(min_lon, min_lat, max_lon, max_lat)) { - /* Bounding box too big - just expire tiles on the line */ - fprintf(stderr, "\rLarge polygon (%.0f x %.0f metres, OSM ID %" PRIdOSMID ") - only expiring perimeter\n", max_lon - min_lon, max_lat - min_lat, osm_id); - from_nodes_line(nodes); + auto parse = ewkb::parser_t(wkb); + + switch (parse.read_header()) { + case ewkb::wkb_point: + from_wkb_point(&parse); + break; + case ewkb::wkb_line: + from_wkb_line(&parse); + break; + case ewkb::wkb_polygon: + from_wkb_polygon(&parse, osm_id); + break; + case ewkb::wkb_multi_line: { + auto num = parse.read_length(); + for (unsigned i = 0; i < num; ++i) { + parse.read_header(); + from_wkb_line(&parse); + } + break; + } + case ewkb::wkb_multi_polygon: { + auto num = parse.read_length(); + for (unsigned i = 0; i < num; ++i) { + parse.read_header(); + from_wkb_polygon(&parse, osm_id); + } + break; + } + default: + fprintf(stderr, "OSM id %" PRIdOSMID + ": Unknown geometry type, cannot expire.\n", + osm_id); } } -void expire_tiles::from_xnodes_poly(const multinodelist_t &xnodes, osmid_t osm_id) +void expire_tiles::from_wkb_point(ewkb::parser_t *wkb) { - for (multinodelist_t::const_iterator it = xnodes.begin(); it != xnodes.end(); ++it) - from_nodes_poly(*it, osm_id); + auto c = wkb->read_point(); + from_bbox(c.x, c.y, c.x, c.y); } -void expire_tiles::from_xnodes_line(const multinodelist_t &xnodes) +void expire_tiles::from_wkb_line(ewkb::parser_t *wkb) { - for (multinodelist_t::const_iterator it = xnodes.begin(); it != xnodes.end(); ++it) - from_nodes_line(*it); + auto sz = wkb->read_length(); + + if (sz == 0) { + return; + } + + if (sz == 1) { + from_wkb_point(wkb); + } else { + auto prev = wkb->read_point(); + for (size_t i = 1; i < sz; ++i) { + auto cur = wkb->read_point(); + from_line(prev.x, prev.y, cur.x, cur.y); + prev = cur; + } + } } -void expire_tiles::from_wkb(const char* wkb, osmid_t osm_id) +void expire_tiles::from_wkb_polygon(ewkb::parser_t *wkb, osmid_t osm_id) { - if (maxzoom < 0) return; - - multinodelist_t xnodes; - bool polygon; + auto num_rings = wkb->read_length(); + assert(num_rings > 0); + + auto start = wkb->save_pos(); + + auto num_pt = wkb->read_length(); + auto initpt = wkb->read_point(); + + osmium::geom::Coordinates min{initpt}, max{initpt}; + + for (size_t i = 1; i < num_pt; ++i) { + auto c = wkb->read_point(); + if (c.x < min.x) + min.x = c.x; + if (c.y < min.y) + min.y = c.y; + if (c.x > max.x) + max.x = c.x; + if (c.y > max.y) + max.y = c.y; + } - if (geometry_builder::parse_wkb(wkb, xnodes, &polygon) == 0) { - if (polygon) - from_xnodes_poly(xnodes, osm_id); - else - from_xnodes_line(xnodes); + if (from_bbox(min.x, min.y, max.x, max.y)) { + /* Bounding box too big - just expire tiles on the line */ + fprintf(stderr, + "\rLarge polygon (%.0f x %.0f metres, OSM ID %" PRIdOSMID + ") - only expiring perimeter\n", + max.x - min.x, max.y - min.y, osm_id); + wkb->rewind(start); + for (unsigned ring = 0; ring < num_rings; ++ring) { + from_wkb_line(wkb); + } + } else { + // ignore inner rings + for (unsigned ring = 1; ring < num_rings; ++ring) { + auto inum_pt = wkb->read_length(); + wkb->skip_points(inum_pt); + } } } @@ -424,8 +464,10 @@ int expire_tiles::from_db(table_t* table, osmid_t osm_id) { //dirty the stuff const char* wkb = nullptr; - while((wkb = wkbs.get_next())) - from_wkb(wkb, osm_id); + while ((wkb = wkbs.get_next())) { + auto binwkb = ewkb::parser_t::wkb_from_hex(wkb); + from_wkb(binwkb.c_str(), osm_id); + } //return how many rows were affected return wkbs.get_count(); diff --git a/expire-tiles.hpp b/expire-tiles.hpp index 840e4fd9b..1ef70395e 100644 --- a/expire-tiles.hpp +++ b/expire-tiles.hpp @@ -8,6 +8,9 @@ class reprojection; class table_t; class tile; +namespace ewkb { +class parser_t; +} struct expire_tiles { @@ -15,8 +18,6 @@ struct expire_tiles const std::shared_ptr &projection); int from_bbox(double min_lon, double min_lat, double max_lon, double max_lat); - void from_nodes_line(const nodelist_t &nodes); - void from_nodes_poly(const nodelist_t &nodes, osmid_t osm_id); void from_wkb(const char* wkb, osmid_t osm_id); int from_db(table_t* table, osmid_t osm_id); @@ -47,8 +48,10 @@ struct expire_tiles void expire_tile(int x, int y); int normalise_tile_x_coord(int x); void from_line(double lon_a, double lat_a, double lon_b, double lat_b); - void from_xnodes_poly(const multinodelist_t &xnodes, osmid_t osm_id); - void from_xnodes_line(const multinodelist_t &xnodes); + + void from_wkb_point(ewkb::parser_t *wkb); + void from_wkb_line(ewkb::parser_t *wkb); + void from_wkb_polygon(ewkb::parser_t *wkb, osmid_t osm_id); double tile_width; double max_bbox; diff --git a/geometry-builder.cpp b/geometry-builder.cpp deleted file mode 100644 index 4dea8c8ea..000000000 --- a/geometry-builder.cpp +++ /dev/null @@ -1,739 +0,0 @@ -/* -#----------------------------------------------------------------------------- -# Part of osm2pgsql utility -#----------------------------------------------------------------------------- -# By Artem Pavlenko, Copyright 2007 -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License -# as published by the Free Software Foundation; either version 2 -# of the License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software -# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -#----------------------------------------------------------------------------- -*/ - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#if defined(__CYGWIN__) -#define GEOS_INLINE -#endif - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -using namespace geos::geom; -using namespace geos::util; -using namespace geos::operation::linemerge; - -#include "geometry-builder.hpp" -#include "reprojection.hpp" - -typedef std::unique_ptr geom_ptr; -typedef std::unique_ptr coord_ptr; - -namespace { - -void coords2nodes(CoordinateSequence * coords, nodelist_t &nodes) -{ - size_t num_coords = coords->getSize(); - nodes.reserve(num_coords); - - for (size_t i = 0; i < num_coords; i++) { - Coordinate coord = coords->getAt(i); - nodes.emplace_back(coord.x, coord.y); - } -} - -coord_ptr nodes2coords(GeometryFactory &gf, const nodelist_t &nodes) -{ - coord_ptr coords(gf.getCoordinateSequenceFactory()->create(size_t(0), size_t(2))); - - for (const auto& nd: nodes) { - coords->add(Coordinate(nd.x, nd.y), 0); - } - - return coords; -} - -geom_ptr create_multi_line(GeometryFactory &gf, const multinodelist_t &xnodes) -{ - // XXX leaks memory if an exception is thrown - std::unique_ptr > lines(new std::vector); - lines->reserve(xnodes.size()); - - for (const auto& nodes: xnodes) { - auto coords = nodes2coords(gf, nodes); - if (coords->getSize() > 1) { - lines->push_back(gf.createLineString(coords.release())); - } - } - - return geom_ptr(gf.createMultiLineString(lines.release())); -} - -bool is_polygon_line(CoordinateSequence * coords) -{ - return (coords->getSize() >= 4) - && (coords->getAt(coords->getSize() - 1).equals2D(coords->getAt(0))); -} - -/** - * Reprojects given Linear Ring from target projection to spherical mercator. - * Caller takes ownership of return value. - */ -LinearRing* reproject_linearring(const LineString *ls, const reprojection *proj) -{ - auto *gf = ls->getFactory(); - coord_ptr coords(gf->getCoordinateSequenceFactory()->create(size_t(0), size_t(2))); - for (auto i : *(ls->getCoordinatesRO()->toVector())) { - Coordinate c(i.x, i.y); - proj->target_to_tile(&c.y, &c.x); - coords->add(c); - } - return gf->createLinearRing(coords.release()); -} - - -/** - * Computes area of given polygonal geometry. - * \return the area in projected units, or in EPSG 3857 if area reprojection is enabled - */ -double get_area(const geos::geom::Geometry *geom, reprojection *proj) -{ - // reprojection is not necessary, or has not been asked for. - if (!proj) { - return geom->getArea(); - } - - // MultiPolygon - return sum of individual areas - if (const auto *multi = dynamic_cast(geom)) { - return std::accumulate(multi->begin(), multi->end(), 0.0, - [=](double a, const Geometry *g) { - return a + get_area(g, proj); - }); - } - - const auto *poly = dynamic_cast(geom); - if (!poly) { - return 0.0; - } - - // standard polygon - reproject rings individually, then assemble polygon and - // compute area. - - const auto *ext = poly->getExteriorRing(); - std::unique_ptr projectedExt(reproject_linearring(ext, proj)); - auto nholes = poly->getNumInteriorRing(); - std::unique_ptr > projectedHoles(new std::vector); - for (std::size_t i=0; i < nholes; i++) { - auto* hole = poly->getInteriorRingN(i); - projectedHoles->push_back(reproject_linearring(hole, proj)); - } - const geom_ptr projectedPoly(poly->getFactory()->createPolygon(projectedExt.release(), projectedHoles.release())); - - return projectedPoly->getArea(); -} - - -struct polygondata -{ - std::unique_ptr polygon; - std::unique_ptr ring; - double area; - bool iscontained; - unsigned containedbyid; - - polygondata(std::unique_ptr p, LinearRing* r, double a) - : polygon(std::move(p)), ring(r), area(a), - iscontained(false), containedbyid(0) - {} -}; - -struct polygondata_comparearea { - bool operator()(const polygondata& lhs, const polygondata& rhs) { - return lhs.area > rhs.area; - } -}; - -} // anonymous namespace - - -void geometry_builder::pg_geom_t::set(const geos::geom::Geometry *g, bool poly, - reprojection *p) -{ - geos::io::WKBWriter writer(2, getMachineByteOrder(), true); - std::stringstream stream(std::ios_base::out); - writer.writeHEX(*g, stream); - geom = stream.str(); - - if (valid()) { - area = poly ? get_area(g, p) : 0; - polygon = poly; - } -} - -geom_ptr geometry_builder::create_simple_poly(GeometryFactory &gf, - std::unique_ptr coords) const -{ - std::unique_ptr shell(gf.createLinearRing(coords.release())); - std::unique_ptr > empty(new std::vector); - geom_ptr geom(gf.createPolygon(shell.release(), empty.release())); - - if (geom->isEmpty()) { - throw std::runtime_error("Excluding empty polygon."); - } - if (!geom->isValid()) { - if (excludepoly) { - throw std::runtime_error("Excluding broken polygon."); - } else { - geom = geom_ptr(geom->buffer(0)); - if (geom->isEmpty() || !geom->isValid()) { - throw std::runtime_error("Excluding unrecoverable broken polygon."); - } - } - } - geom->normalize(); // Fix direction of ring - - return geom; -} - -geometry_builder::pg_geom_t geometry_builder::get_wkb_simple(const nodelist_t &nodes, int polygon) const -{ - pg_geom_t wkb; - - try - { - GeometryFactory gf; - auto coords = nodes2coords(gf, nodes); - if (polygon && is_polygon_line(coords.get())) { - auto geom = create_simple_poly(gf, std::move(coords)); - wkb.set(geom.get(), true, projection); - } else { - if (coords->getSize() < 2) - throw std::runtime_error("Excluding degenerate line."); - geom_ptr geom(gf.createLineString(coords.release())); - wkb.set(geom.get(), false); - } - } - catch (const std::bad_alloc&) - { - std::cerr << std::endl << "Exception caught processing way. You are likelly running out of memory." << std::endl; - std::cerr << "Try in slim mode, using -s parameter." << std::endl; - } - catch (const std::runtime_error& e) - { - //std::cerr << std::endl << "Exception caught processing way: " << e.what() << std::endl; - } - catch (...) - { - std::cerr << std::endl << "Exception caught processing way" << std::endl; - } - - return wkb; -} - -geometry_builder::pg_geoms_t geometry_builder::get_wkb_split(const nodelist_t &nodes, int polygon, double split_at) const -{ - //TODO: use count to get some kind of hint of how much we should reserve? - pg_geoms_t wkbs; - - try - { - GeometryFactory gf; - auto coords = nodes2coords(gf, nodes); - - if (polygon && is_polygon_line(coords.get())) { - auto geom = create_simple_poly(gf, std::move(coords)); - wkbs.emplace_back(geom.get(), true, projection); - } else { - if (coords->getSize() < 2) - throw std::runtime_error("Excluding degenerate line."); - - double distance = 0; - std::unique_ptr segment(gf.getCoordinateSequenceFactory()->create((size_t)0, (size_t)2)); - segment->add(coords->getAt(0)); - for(size_t i=1; igetSize(); i++) { - const Coordinate this_pt = coords->getAt(i); - const Coordinate prev_pt = coords->getAt(i-1); - const double delta = this_pt.distance(prev_pt); - assert(!std::isnan(delta)); - // figure out if the addition of this point would take the total - // length of the line in `segment` over the `split_at` distance. - - if (distance + delta > split_at) { - const size_t splits = (size_t) std::floor((distance + delta) / split_at); - // use the splitting distance to split the current segment up - // into as many parts as necessary to keep each part below - // the `split_at` distance. - for (size_t j = 0; j < splits; ++j) { - double frac = (double(j + 1) * split_at - distance) / delta; - const Coordinate interpolated(frac * (this_pt.x - prev_pt.x) + prev_pt.x, - frac * (this_pt.y - prev_pt.y) + prev_pt.y); - segment->add(interpolated); - geom_ptr geom(gf.createLineString(segment.release())); - - wkbs.emplace_back(geom.get(), false); - - segment.reset(gf.getCoordinateSequenceFactory()->create((size_t)0, (size_t)2)); - segment->add(interpolated); - } - // reset the distance based on the final splitting point for - // the next iteration. - distance = segment->getAt(0).distance(this_pt); - - } else { - // if not split then just push this point onto the sequence - // being saved up. - distance += delta; - } - - // always add this point - segment->add(this_pt); - - // on the last iteration, close out the line. - if (i == coords->getSize()-1) { - geom_ptr geom(gf.createLineString(segment.release())); - - wkbs.emplace_back(geom.get(), false); - } - } - } - } - catch (const std::bad_alloc&) - { - std::cerr << std::endl << "Exception caught processing way. You are likely running out of memory." << std::endl; - std::cerr << "Try in slim mode, using -s parameter." << std::endl; - } - catch (const std::runtime_error& e) - { - //std::cerr << std::endl << "Exception caught processing way: " << e.what() << std::endl; - } - catch (...) - { - std::cerr << std::endl << "Exception caught processing way" << std::endl; - } - return wkbs; -} - -int geometry_builder::parse_wkb(const char* wkb, multinodelist_t &nodes, bool *polygon) { - GeometryFactory gf; - geos::io::WKBReader reader(gf); - - *polygon = false; - std::stringstream stream(wkb, std::ios_base::in); - geom_ptr geometry(reader.readHEX(stream)); - switch (geometry->getGeometryTypeId()) { - // Single geometries - case GEOS_POLYGON: - // Drop through - case GEOS_LINEARRING: - *polygon = true; - // Drop through - case GEOS_POINT: - // Drop through - case GEOS_LINESTRING: - { - nodes.push_back(nodelist_t()); - coord_ptr coords(geometry->getCoordinates()); - coords2nodes(coords.get(), nodes.back()); - break; - } - // Geometry collections - case GEOS_MULTIPOLYGON: - *polygon = true; - // Drop through - case GEOS_MULTIPOINT: - // Drop through - case GEOS_MULTILINESTRING: - { - auto gc = dynamic_cast(geometry.get()); - size_t num_geometries = gc->getNumGeometries(); - nodes.assign(num_geometries, nodelist_t()); - for (size_t i = 0; i < num_geometries; i++) { - const Geometry *subgeometry = gc->getGeometryN(i); - coord_ptr coords(subgeometry->getCoordinates()); - coords2nodes(coords.get(), nodes[i]); - } - break; - } - default: - std::cerr << std::endl << "unexpected object type while processing PostGIS data" << std::endl; - return -1; - } - - return 0; -} - -geometry_builder::pg_geoms_t geometry_builder::build_polygons(const multinodelist_t &xnodes, - bool enable_multi, osmid_t osm_id) const -{ - pg_geoms_t wkbs; - - try - { - GeometryFactory gf; - geom_ptr mline = create_multi_line(gf, xnodes); - - //geom_ptr noded (segment->Union(mline.get())); - LineMerger merger; - //merger.add(noded.get()); - merger.add(mline.get()); - std::unique_ptr> merged(merger.getMergedLineStrings()); - - // Procces ways into lines or simple polygon list - std::vector polys; - polys.reserve(merged->size()); - - for (auto *line: *merged) { - // stuff into unique pointer for auto-destruct - std::unique_ptr pline(line); - if (pline->getNumPoints() > 3 && pline->isClosed()) { - std::unique_ptr poly(gf.createPolygon(gf.createLinearRing(pline->getCoordinates()),0)); - double area = get_area(poly.get(), projection); - if (area > 0.0) { - polys.emplace_back(std::move(poly), - gf.createLinearRing(pline->getCoordinates()), - area); - } - } - } - - if (!polys.empty()) - { - std::sort(polys.begin(), polys.end(), polygondata_comparearea()); - - unsigned toplevelpolygons = 0; - int istoplevelafterall; - size_t totalpolys = polys.size(); - - geos::geom::prep::PreparedGeometryFactory pgf; - for (unsigned i=0 ;i < totalpolys; ++i) - { - if (polys[i].iscontained) continue; - toplevelpolygons++; - const geos::geom::prep::PreparedGeometry* preparedtoplevelpolygon = pgf.create(polys[i].polygon.get()); - - for (unsigned j=i+1; j < totalpolys; ++j) - { - // Does preparedtoplevelpolygon contain the smaller polygon[j]? - if (polys[j].containedbyid == 0 && preparedtoplevelpolygon->contains(polys[j].polygon.get())) - { - // are we in a [i] contains [k] contains [j] situation - // which would actually make j top level - istoplevelafterall = 0; - for (unsigned k=i+1; k < j; ++k) - { - if (polys[k].iscontained && polys[k].containedbyid == i && polys[k].polygon->contains(polys[j].polygon.get())) - { - istoplevelafterall = 1; - break; - } - } - if (istoplevelafterall == 0) - { - polys[j].iscontained = true; - polys[j].containedbyid = i; - } - } - } - pgf.destroy(preparedtoplevelpolygon); - } - // polys now is a list of polygons tagged with which ones are inside each other - - // List of polygons for multipolygon - std::unique_ptr> polygons(new std::vector); - - // For each top level polygon create a new polygon including any holes - for (unsigned i=0 ;i < totalpolys; ++i) - { - if (polys[i].iscontained) continue; - - // List of holes for this top level polygon - std::unique_ptr > interior(new std::vector); - for (unsigned j=i+1; j < totalpolys; ++j) - { - if (polys[j].iscontained && polys[j].containedbyid == i) - { - interior->push_back(polys[j].ring.release()); - } - } - - Polygon* poly(gf.createPolygon(polys[i].ring.release(), interior.release())); - poly->normalize(); - polygons->push_back(poly); - } - - // Make a multipolygon if required - if ((toplevelpolygons > 1) && enable_multi) - { - geom_ptr multipoly(gf.createMultiPolygon(polygons.release())); - - if (!multipoly->isEmpty()) { - if (!multipoly->isValid() && !excludepoly) { - multipoly = geom_ptr(multipoly->buffer(0)); - multipoly->normalize(); - if (!multipoly->isEmpty() && multipoly->isValid()) { - wkbs.emplace_back(multipoly.get(), true, projection); - } - } else { - multipoly->normalize(); - wkbs.emplace_back(multipoly.get(), true, projection); - } - } - } else { - for(unsigned i=0; iat(i)); - if (!poly->isEmpty()) { - if (!poly->isValid() && !excludepoly) { - poly = geom_ptr(poly->buffer(0)); - poly->normalize(); - if (!poly->isEmpty() && poly->isValid()) { - wkbs.emplace_back(poly.get(), true, projection); - } - } else { - poly->normalize(); - wkbs.emplace_back(poly.get(), true, projection); - } - } - } - } - } - }//TODO: don't show in message id when osm_id == -1 - catch (const std::exception& e) - { - std::cerr << std::endl << "Standard exception processing relation_id="<< osm_id << ": " << e.what() << std::endl; - } - catch (...) - { - std::cerr << std::endl << "Exception caught processing relation id=" << osm_id << std::endl; - } - - return wkbs; -} - -geometry_builder::pg_geom_t geometry_builder::build_multilines(const multinodelist_t &xnodes, osmid_t osm_id) const -{ - pg_geom_t wkb; - - try - { - GeometryFactory gf; - geom_ptr mline = create_multi_line(gf, xnodes); - - wkb.set(mline.get(), false); - }//TODO: don't show in message id when osm_id == -1 - catch (const std::exception& e) - { - std::cerr << std::endl << "Standard exception processing relation_id="<< osm_id << ": " << e.what() << std::endl; - } - catch (...) - { - std::cerr << std::endl << "Exception caught processing relation id=" << osm_id << std::endl; - } - return wkb; -} - -geometry_builder::pg_geoms_t geometry_builder::build_both(const multinodelist_t &xnodes, - int make_polygon, int enable_multi, - double split_at, osmid_t osm_id) const -{ - pg_geoms_t wkbs; - - try - { - GeometryFactory gf; - geom_ptr mline = create_multi_line(gf, xnodes); - //geom_ptr noded (segment->Union(mline.get())); - LineMerger merger; - //merger.add(noded.get()); - merger.add(mline.get()); - std::unique_ptr > merged(merger.getMergedLineStrings()); - - // Procces ways into lines or simple polygon list - std::vector polys; - polys.reserve(merged->size()); - - for (auto *line: *merged) { - // stuff into unique pointer to ensure auto-destruct - std::unique_ptr pline(line); - if (make_polygon && pline->getNumPoints() > 3 && pline->isClosed()) { - std::unique_ptr poly(gf.createPolygon(gf.createLinearRing(pline->getCoordinates()),0)); - double area = get_area(poly.get(), projection); - if (area > 0.0) { - polys.emplace_back(std::move(poly), - gf.createLinearRing(pline->getCoordinates()), - area); - } - } else { - double distance = 0; - std::unique_ptr segment; - segment = std::unique_ptr(gf.getCoordinateSequenceFactory()->create((size_t)0, (size_t)2)); - segment->add(pline->getCoordinateN(0)); - for(int j=1; j<(int)pline->getNumPoints(); ++j) { - segment->add(pline->getCoordinateN(j)); - distance += pline->getCoordinateN(j).distance(pline->getCoordinateN(j-1)); - if ((distance >= split_at) || (j == (int)pline->getNumPoints()-1)) { - geom_ptr geom = geom_ptr(gf.createLineString(segment.release())); - - wkbs.emplace_back(geom.get(), false); - - segment.reset(gf.getCoordinateSequenceFactory()->create((size_t)0, (size_t)2)); - distance=0; - segment->add(pline->getCoordinateN(j)); - } - } - } - } - - if (!polys.empty()) - { - std::sort(polys.begin(), polys.end(), polygondata_comparearea()); - - unsigned toplevelpolygons = 0; - int istoplevelafterall; - size_t totalpolys = polys.size(); - - geos::geom::prep::PreparedGeometryFactory pgf; - for (unsigned i=0 ;i < totalpolys; ++i) - { - if (polys[i].iscontained) continue; - toplevelpolygons++; - const geos::geom::prep::PreparedGeometry* preparedtoplevelpolygon = pgf.create(polys[i].polygon.get()); - - for (unsigned j=i+1; j < totalpolys; ++j) - { - // Does preparedtoplevelpolygon contain the smaller polygon[j]? - if (polys[j].containedbyid == 0 && preparedtoplevelpolygon->contains(polys[j].polygon.get())) - { - // are we in a [i] contains [k] contains [j] situation - // which would actually make j top level - istoplevelafterall = 0; - for (unsigned k=i+1; k < j; ++k) - { - if (polys[k].iscontained && polys[k].containedbyid == i && polys[k].polygon->contains(polys[j].polygon.get())) - { - istoplevelafterall = 1; - break; - } -#if 0 - else if (polys[k].polygon->intersects(polys[j].polygon) || polys[k].polygon->touches(polys[j].polygon)) - { - // FIXME: This code does not work as intended - // It should be setting the polys[k].ring in order to update this object - // but the value of polys[k].polygon calculated is normally NULL - - // Add polygon this polygon (j) to k since they intersect - // Mark ourselfs to be dropped (2), delete the original k - Geometry* polyunion = polys[k].polygon->Union(polys[j].polygon); - delete(polys[k].polygon); - polys[k].polygon = dynamic_cast(polyunion); - polys[j].iscontained = 2; // Drop - istoplevelafterall = 2; - break; - } -#endif - } - if (istoplevelafterall == 0) - { - polys[j].iscontained = true; - polys[j].containedbyid = i; - } - } - } - pgf.destroy(preparedtoplevelpolygon); - } - // polys now is a list of polygons tagged with which ones are inside each other - - // List of polygons for multipolygon - std::unique_ptr > polygons(new std::vector); - - // For each top level polygon create a new polygon including any holes - for (unsigned i=0 ;i < totalpolys; ++i) - { - if (polys[i].iscontained) continue; - - // List of holes for this top level polygon - std::unique_ptr > interior(new std::vector); - for (unsigned j=i+1; j < totalpolys; ++j) - { - if (polys[j].iscontained && polys[j].containedbyid == i) - { - interior->push_back(polys[j].ring.release()); - } - } - - Polygon* poly(gf.createPolygon(polys[i].ring.release(), interior.release())); - poly->normalize(); - polygons->push_back(poly); - } - - // Make a multipolygon if required - if ((toplevelpolygons > 1) && enable_multi) - { - geom_ptr multipoly(gf.createMultiPolygon(polygons.release())); - if (!multipoly->isValid() && !excludepoly) { - multipoly = geom_ptr(multipoly->buffer(0)); - } - multipoly->normalize(); - - if (!multipoly->isEmpty() && multipoly->isValid()) { - wkbs.emplace_back(multipoly.get(), true, projection); - } - } - else - { - for(unsigned i=0; iat(i)); - if (!poly->isValid() && !excludepoly) { - poly = geom_ptr(poly->buffer(0)); - poly->normalize(); - } - if (!poly->isEmpty() && poly->isValid()) { - wkbs.emplace_back(poly.get(), true, projection); - } - } - } - } - }//TODO: don't show in message id when osm_id == -1 - catch (const std::exception& e) - { - std::cerr << std::endl << "Standard exception processing relation id="<< osm_id << ": " << e.what() << std::endl; - } - catch (...) - { - std::cerr << std::endl << "Exception caught processing relation id=" << osm_id << std::endl; - } - - return wkbs; -} diff --git a/geometry-builder.hpp b/geometry-builder.hpp deleted file mode 100644 index 56457c173..000000000 --- a/geometry-builder.hpp +++ /dev/null @@ -1,114 +0,0 @@ -/* -#----------------------------------------------------------------------------- -# Part of osm2pgsql utility -#----------------------------------------------------------------------------- -# By Artem Pavlenko, Copyright 2007 -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License -# as published by the Free Software Foundation; either version 2 -# of the License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software -# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -#----------------------------------------------------------------------------- -*/ - -#ifndef GEOMETRY_BUILDER_H -#define GEOMETRY_BUILDER_H - -#include "osmtypes.hpp" - -#include -#include -#include - -namespace geos { namespace geom { -class Geometry; -class GeometryFactory; -class CoordinateSequence; -}} - -class reprojection; - -class geometry_builder -{ -public: - struct pg_geom_t - { - pg_geom_t(const std::string &geom_str, bool poly, double geom_area = 0) - : geom(geom_str), area(geom_area), polygon(poly) - {} - - /// Create an invalid geometry. - pg_geom_t() - : area(0), polygon(false) - {} - - pg_geom_t(const geos::geom::Geometry *g, bool poly, reprojection *p = nullptr) - { - set(g, poly, p); - } - - /** - * Set geometry from a Geos geometry. - */ - void set(const geos::geom::Geometry *geom, bool poly, reprojection *p = nullptr); - - - bool is_polygon() const - { - return polygon; - } - - bool valid() const - { - return !geom.empty(); - } - - std::string geom; - double area; - bool polygon; - }; - - typedef std::vector pg_geoms_t; - - static int parse_wkb(const char *wkb, multinodelist_t &nodes, bool *polygon); - pg_geom_t get_wkb_simple(const nodelist_t &nodes, int polygon) const; - pg_geoms_t get_wkb_split(const nodelist_t &nodes, int polygon, double split_at) const; - pg_geoms_t build_both(const multinodelist_t &xnodes, int make_polygon, - int enable_multi, double split_at, osmid_t osm_id = -1) const; - pg_geoms_t build_polygons(const multinodelist_t &xnodes, bool enable_multi, osmid_t osm_id = -1) const; - /** Output relation as a multiline. - * - * Used by gazetteer only. - */ - pg_geom_t build_multilines(const multinodelist_t &xnodes, osmid_t osm_id) const; - - void set_exclude_broken_polygon(bool exclude) - { - excludepoly = exclude; - } - - void set_reprojection(reprojection *r) - { - projection = r; - } - - -private: - std::unique_ptr - create_simple_poly(geos::geom::GeometryFactory &gf, - std::unique_ptr coords) const; - - bool excludepoly = false; - reprojection *projection = nullptr; -}; - -#endif diff --git a/geometry-processor.cpp b/geometry-processor.cpp index a5191828d..729293d4a 100644 --- a/geometry-processor.cpp +++ b/geometry-processor.cpp @@ -14,16 +14,16 @@ std::shared_ptr geometry_processor::create(const std::string &type, const options_t *options) { std::shared_ptr ptr; - int srid = options->projection->target_srs(); if (type == "point") { - ptr = std::make_shared(srid); + ptr = std::make_shared(options->projection); } else if (type == "line") { - ptr = std::make_shared(srid); + ptr = std::make_shared(options->projection); } else if (type == "polygon") { - ptr = std::make_shared(srid, options->enable_multi); + ptr = std::make_shared(options->projection, + options->enable_multi); } else { throw std::runtime_error((boost::format("Unable to construct geometry processor " @@ -38,8 +38,7 @@ geometry_processor::geometry_processor(int srid, const std::string &type, unsign : m_srid(srid), m_type(type), m_interests(interests) { } -geometry_processor::~geometry_processor() { -} +geometry_processor::~geometry_processor() = default; int geometry_processor::srid() const { return m_srid; @@ -57,85 +56,39 @@ bool geometry_processor::interests(unsigned int interested) const { return (interested & m_interests) == interested; } -geometry_builder::pg_geom_t geometry_processor::process_node(double, double) { - return geometry_builder::pg_geom_t(); -} - -geometry_builder::pg_geom_t geometry_processor::process_way(const nodelist_t &) { - return geometry_builder::pg_geom_t(); -} - -geometry_builder::pg_geoms_t geometry_processor::process_relation(const multinodelist_t &) { - return geometry_builder::pg_geoms_t(); +geometry_processor::wkb_t +geometry_processor::process_node(osmium::Location const &) +{ + return wkb_t(); } -way_helper::way_helper() +geometry_processor::wkb_t geometry_processor::process_way(osmium::Way const &) { + return wkb_t(); } -way_helper::~way_helper() + +geometry_processor::wkbs_t +geometry_processor::process_relation(osmium::Relation const &, + osmium::memory::Buffer const &) { + return wkbs_t(); } -size_t way_helper::set(osmium::WayNodeList const &node_ids, - middle_query_t const *mid, reprojection const *proj) -{ - node_cache.clear(); - mid->nodes_get_list(node_cache, node_ids, proj); - // equivalent to returning node_count for complete ways, different for partial - // extracts - return node_cache.size(); -} relation_helper::relation_helper() : data(1024, osmium::memory::Buffer::auto_grow::yes) {} -relation_helper::~relation_helper() = default; - -size_t relation_helper::set(osmium::RelationMemberList const &member_list, middle_t const *mid) +size_t relation_helper::set(osmium::Relation const &rel, middle_t const *mid) { // cleanup - input_way_ids.clear(); data.clear(); roles.clear(); - //grab the way members' ids - size_t num_input = member_list.size(); - input_way_ids.reserve(num_input); - roles.reserve(num_input); - for (auto const &member : member_list) { - if (member.type() == osmium::item_type::way) { - input_way_ids.push_back(member.ref()); - roles.push_back(member.role()); - } - } - - //if we didn't end up using any we'll bail - if (input_way_ids.empty()) - return 0; - - //get the nodes of the ways - auto num_ways = mid->ways_get_list(input_way_ids, data); - - //grab the roles of each way - if (num_ways < input_way_ids.size()) { - size_t memberpos = 0; - size_t waypos = 0; - for (auto const &w : data.select()) { - while (memberpos < input_way_ids.size()) { - if (input_way_ids[memberpos] == w.id()) { - roles[waypos] = roles[memberpos]; - ++memberpos; - break; - } - ++memberpos; - } - ++waypos; - } - roles.resize(num_ways); - } + // get the nodes and roles of the ways + auto num_ways = mid->rel_way_members_get(rel, &roles, data); - //mark the ends of each so whoever uses them will know where they end.. + // mark the ends of each so whoever uses them will know where they end.. superseded.resize(num_ways); return num_ways; @@ -147,21 +100,15 @@ multitaglist_t relation_helper::get_filtered_tags(tagtransform *transform, expor size_t i = 0; for (auto const &w : data.select()) { - transform->filter_tags(w, nullptr, nullptr, el, filtered[i++]); + transform->filter_tags(w, nullptr, nullptr, el, filtered[++i]); } return filtered; } -multinodelist_t relation_helper::get_nodes(middle_t const *mid, - reprojection const *proj) const +void relation_helper::add_way_locations(middle_t const *mid) { - multinodelist_t nodes(roles.size()); - - size_t i = 0; - for (auto const &w : data.select()) { - mid->nodes_get_list(nodes[i++], w.nodes(), proj); + for (auto &w : data.select()) { + mid->nodes_get_list(&(w.nodes())); } - - return nodes; } diff --git a/geometry-processor.hpp b/geometry-processor.hpp index efb876bad..479b81914 100644 --- a/geometry-processor.hpp +++ b/geometry-processor.hpp @@ -8,7 +8,7 @@ #include -#include "geometry-builder.hpp" +#include "osmium-builder.hpp" #include "osmtypes.hpp" #include "tagtransform.hpp" @@ -18,6 +18,8 @@ struct options_t; class reprojection; struct geometry_processor { + using wkb_t = geom::osmium_builder_t::wkb_t; + using wkbs_t = geom::osmium_builder_t::wkbs_t; // factory method for creating various types of geometry processors either by name or by geometry column type static std::shared_ptr create(const std::string &type, const options_t *options); @@ -45,17 +47,18 @@ struct geometry_processor { // process a node, optionally returning a WKB string describing // geometry to be inserted into the table. - virtual geometry_builder::pg_geom_t process_node(double lat, double lon); + virtual wkb_t process_node(osmium::Location const &loc); // process a way // position data and optionally returning WKB-encoded geometry // for insertion into the table. - virtual geometry_builder::pg_geom_t process_way(const nodelist_t &nodes); + virtual wkb_t process_way(osmium::Way const &way); // process a way, taking a middle query object to get way and // node position data. optionally returns an array of WKB-encoded geometry // for insertion into the table. - virtual geometry_builder::pg_geoms_t process_relation(const multinodelist_t &nodes); + virtual wkbs_t process_relation(osmium::Relation const &rel, + osmium::memory::Buffer const &ways); // returns the SRID of the output geometry. int srid() const; @@ -75,36 +78,19 @@ struct geometry_processor { }; -//various bits for continuous processing of ways -struct way_helper -{ - way_helper(); - ~way_helper(); - size_t set(osmium::WayNodeList const &node_ids, middle_query_t const *mid, - reprojection const *proj); - - nodelist_t node_cache; -}; - //various bits for continuous processing of members of relations -struct relation_helper +class relation_helper { +public: relation_helper(); - ~relation_helper(); - size_t set(osmium::RelationMemberList const &member_list, middle_t const *mid); - multitaglist_t get_filtered_tags(tagtransform *transform, export_list const &el) const; - multinodelist_t get_nodes(middle_t const *mid, - reprojection const *proj) const; - osmium::memory::ItemIteratorRange way_iterator() const - { return data.select(); } + size_t set(osmium::Relation const &rel, middle_t const *mid); + multitaglist_t get_filtered_tags(tagtransform *transform, export_list const &el) const; + void add_way_locations(middle_t const *mid); rolelist_t roles; std::vector superseded; osmium::memory::Buffer data; - -private: - idlist_t input_way_ids; }; #endif /* GEOMETRY_PROCESSOR_HPP */ diff --git a/middle-pgsql.cpp b/middle-pgsql.cpp index f3f3731d0..147ad53be 100644 --- a/middle-pgsql.cpp +++ b/middle-pgsql.cpp @@ -286,90 +286,60 @@ void middle_pgsql_t::local_nodes_set(osmium::Node const &node) } } -// This should be made more efficient by using an IN(ARRAY[]) construct */ -size_t middle_pgsql_t::local_nodes_get_list(nodelist_t &out, - osmium::WayNodeList const &nds, - reprojection const *proj) const +size_t middle_pgsql_t::local_nodes_get_list(osmium::WayNodeList *nodes) const { - assert(out.empty()); + size_t count = 0; + std::string buffer("{"); - char tmp[16]; - - char *tmp2 = static_cast(malloc(sizeof(char) * nds.size() * 16)); - if (tmp2 == nullptr) { - return 0; // failed to allocate memory, return - } - - // create a list of ids in tmp2 to query the database - sprintf(tmp2, "{"); - int countDB = 0; - out.reserve(nds.size()); - for (auto const &n : nds) { - // Check cache first */ + // get nodes where possible from cache, + // at the same time build a list for querying missing nodes from DB + size_t pos = 0; + for (auto &n : *nodes) { auto loc = cache->get(n.ref()); if (loc.valid()) { - out.push_back(proj->reproject(loc)); - continue; + n.set_location(loc); + ++count; + } else { + buffer += std::to_string(n.ref()); + buffer += ','; } - - countDB++; - // Mark nodes as needing to be fetched from the DB - out.emplace_back(NAN, NAN); - - snprintf(tmp, sizeof(tmp), "%" PRIdOSMID ",", n.ref()); - strncat(tmp2, tmp, sizeof(char) * (nds.size() * 16 - 2)); + ++pos; } - // replace last , with } to complete list of ids - tmp2[strlen(tmp2) - 1] = '}'; - if (countDB == 0) { - free(tmp2); - return nds.size(); // All ids where in cache, so nothing more to do + if (count == pos) { + return count; // all ids found in cache, nothing more to do } + // get any remaining nodes from the DB + buffer[buffer.size() - 1] = '}'; + pgsql_endCopy(node_table); PGconn *sql_conn = node_table->sql_conn; char const *paramValues[1]; - paramValues[0] = tmp2; + paramValues[0] = buffer.c_str(); PGresult *res = pgsql_execPrepared(sql_conn, "get_node_list", 1, paramValues, PGRES_TUPLES_OK); - int countPG = PQntuples(res); - - // store the pg results in a hashmap and telling it how many we expect - std::unordered_map pg_nodes(countPG); - - for (int i = 0; i < countPG; i++) { - osmid_t id = strtoosmid(PQgetvalue(res, i, 0), nullptr, 10); - osmium::Location n((int)strtol(PQgetvalue(res, i, 2), nullptr, 10), - (int)strtol(PQgetvalue(res, i, 1), nullptr, 10)); + auto countPG = PQntuples(res); - pg_nodes.emplace(id, n); + std::unordered_map locs; + for (int i = 0; i < countPG; ++i) { + locs.emplace(strtoosmid(PQgetvalue(res, i, 0), nullptr, 10), + osmium::Location((int)strtol(PQgetvalue(res, i, 2), nullptr, 10), + (int)strtol(PQgetvalue(res, i, 1), nullptr, 10))); } - PQclear(res); - free(tmp2); - - // If some of the nodes in the way don't exist, the returning list has holes. - // Merge the two lists removing any holes. - size_t wrtidx = 0; - for (size_t i = 0; i < nds.size(); ++i) { - if (std::isnan(out[i].x)) { - auto found = pg_nodes.find(nds[i].ref()); - if (found != pg_nodes.end()) { - out[wrtidx] = proj->reproject(found->second); - ++wrtidx; - } - } else { - if (wrtidx < i) - out[wrtidx] = out[i]; - ++wrtidx; + for (auto &n : *nodes) { + auto el = locs.find(n.ref()); + if (el != locs.end()) { + n.set_location(el->second); + ++count; } + } - out.resize(wrtidx, osmium::geom::Coordinates(NAN, NAN)); - return wrtidx; + return count; } void middle_pgsql_t::nodes_set(osmium::Node const &node) @@ -383,13 +353,11 @@ void middle_pgsql_t::nodes_set(osmium::Node const &node) } } -size_t middle_pgsql_t::nodes_get_list(nodelist_t &out, - osmium::WayNodeList const &nds, - reprojection const *proj) const +size_t middle_pgsql_t::nodes_get_list(osmium::WayNodeList *nodes) const { return (out_options->flat_node_cache_enabled) - ? persistent_cache->get_list(out, nds, proj) - : local_nodes_get_list(out, nds, proj); + ? persistent_cache->get_list(nodes) + : local_nodes_get_list(nodes); } void middle_pgsql_t::local_nodes_delete(osmid_t osm_id) @@ -525,30 +493,33 @@ bool middle_pgsql_t::ways_get(osmid_t id, osmium::memory::Buffer &buffer) const return true; } -size_t middle_pgsql_t::ways_get_list(const idlist_t &ids, osmium::memory::Buffer &buffer) const +size_t middle_pgsql_t::rel_way_members_get(osmium::Relation const &rel, + rolelist_t *roles, + osmium::memory::Buffer &buffer) const { - if (ids.empty()) - return 0; - char tmp[16]; - std::unique_ptr tmp2(new (std::nothrow) char[ids.size() * 16]); char const *paramValues[1]; - if (tmp2 == nullptr) return 0; //failed to allocate memory, return */ + // create a list of ids in tmp2 to query the database + std::string tmp2("{"); + for (auto const &m : rel.members()) { + if (m.type() == osmium::item_type::way) { + snprintf(tmp, sizeof(tmp), "%" PRIdOSMID ",", m.ref()); + tmp2.append(tmp); + } + } - // create a list of ids in tmp2 to query the database */ - sprintf(tmp2.get(), "{"); - for (auto id : ids) { - snprintf(tmp, sizeof(tmp), "%" PRIdOSMID ",", id); - strncat(tmp2.get(), tmp, sizeof(char)*(ids.size()*16 - 2)); + if (tmp2.length() == 1) { + return 0; // no ways found } - tmp2[strlen(tmp2.get()) - 1] = '}'; // replace last , with } to complete list of ids*/ + // replace last , with } to complete list of ids + tmp2[tmp2.length() - 1] = '}'; pgsql_endCopy(way_table); PGconn *sql_conn = way_table->sql_conn; - paramValues[0] = tmp2.get(); + paramValues[0] = tmp2.c_str(); PGresult *res = pgsql_execPrepared(sql_conn, "get_way_list", 1, paramValues, PGRES_TUPLES_OK); int countPG = PQntuples(res); @@ -558,22 +529,27 @@ size_t middle_pgsql_t::ways_get_list(const idlist_t &ids, osmium::memory::Buffer wayidspg.push_back(strtoosmid(PQgetvalue(res, i, 0), nullptr, 10)); } - // Match the list of ways coming from postgres in a different order // back to the list of ways given by the caller */ - int outres = 0; - for (auto id : ids) { + size_t outres = 0; + for (auto const &m : rel.members()) { + if (m.type() != osmium::item_type::way) { + continue; + } for (int j = 0; j < countPG; j++) { - if (id == wayidspg[j]) { + if (m.ref() == wayidspg[j]) { { osmium::builder::WayBuilder builder(buffer); - builder.set_id(id); + builder.set_id(m.ref()); pgsql_parse_nodes(PQgetvalue(res, j, 1), buffer, builder); pgsql_parse_tags(PQgetvalue(res, j, 2), buffer, builder); } buffer.commit(); + if (roles) { + roles->emplace_back(m.role()); + } outres++; break; } diff --git a/middle-pgsql.hpp b/middle-pgsql.hpp index d9f63e29b..40186df9c 100644 --- a/middle-pgsql.hpp +++ b/middle-pgsql.hpp @@ -27,14 +27,14 @@ struct middle_pgsql_t : public slim_middle_t { void commit(void) override; void nodes_set(osmium::Node const &node) override; - size_t nodes_get_list(nodelist_t &out, osmium::WayNodeList const &nds, - reprojection const *proj) const override; + size_t nodes_get_list(osmium::WayNodeList *nodes) const override; void nodes_delete(osmid_t id) override; void node_changed(osmid_t id) override; void ways_set(osmium::Way const &way) override; bool ways_get(osmid_t id, osmium::memory::Buffer &buffer) const override; - size_t ways_get_list(const idlist_t &ids, osmium::memory::Buffer &buffer) const override; + size_t rel_way_members_get(osmium::Relation const &rel, rolelist_t *roles, + osmium::memory::Buffer &buffer) const override; void ways_delete(osmid_t id) override; void way_changed(osmid_t id) override; @@ -88,8 +88,7 @@ struct middle_pgsql_t : public slim_middle_t { */ void connect(table_desc& table); void local_nodes_set(osmium::Node const &node); - size_t local_nodes_get_list(nodelist_t &out, osmium::WayNodeList const &nds, - reprojection const *proj) const; + size_t local_nodes_get_list(osmium::WayNodeList *nodes) const; void local_nodes_delete(osmid_t osm_id); std::vector tables; diff --git a/middle-ram.cpp b/middle-ram.cpp index 8d593915b..97ea1a88f 100644 --- a/middle-ram.cpp +++ b/middle-ram.cpp @@ -48,18 +48,19 @@ void middle_ram_t::relations_set(osmium::Relation const &rel) rels.set(rel.id(), new ramRel(rel, out_options->extra_attributes)); } -size_t middle_ram_t::nodes_get_list(nodelist_t &out, - osmium::WayNodeList const &nds, - reprojection const *proj) const +size_t middle_ram_t::nodes_get_list(osmium::WayNodeList *nodes) const { - for (auto const &in : nds) { - auto loc = cache->get(in.ref()); + size_t count = 0; + + for (auto &n : *nodes) { + auto loc = cache->get(n.ref()); + n.set_location(loc); if (loc.valid()) { - out.push_back(proj->reproject(loc)); + ++count; } } - return out.size(); + return count; } void middle_ram_t::iterate_relations(pending_processor& pf) @@ -117,15 +118,16 @@ bool middle_ram_t::ways_get(osmid_t id, osmium::memory::Buffer &buffer) const return true; } -size_t middle_ram_t::ways_get_list(const idlist_t &ids, osmium::memory::Buffer &buffer) const +size_t middle_ram_t::rel_way_members_get(osmium::Relation const &rel, + rolelist_t *roles, + osmium::memory::Buffer &buffer) const { - if (ids.empty()) { - return 0; - } - size_t count = 0; - for (auto const id: ids) { - if (ways_get(id, buffer)) { + for (auto const &m : rel.members()) { + if (m.type() == osmium::item_type::way && ways_get(m.ref(), buffer)) { + if (roles) { + roles->emplace_back(m.role()); + } ++count; } } diff --git a/middle-ram.hpp b/middle-ram.hpp index ca9e58296..67982552a 100644 --- a/middle-ram.hpp +++ b/middle-ram.hpp @@ -91,14 +91,14 @@ struct middle_ram_t : public middle_t { void commit(void) override; void nodes_set(osmium::Node const &node) override; - size_t nodes_get_list(nodelist_t &out, osmium::WayNodeList const &nds, - reprojection const *proj) const override; + size_t nodes_get_list(osmium::WayNodeList *nodes) const override; int nodes_delete(osmid_t id); int node_changed(osmid_t id); void ways_set(osmium::Way const &way) override; bool ways_get(osmid_t id, osmium::memory::Buffer &buffer) const override; - size_t ways_get_list(idlist_t const &ids, osmium::memory::Buffer &buffer) const override; + size_t rel_way_members_get(osmium::Relation const &rel, rolelist_t *roles, + osmium::memory::Buffer &buffer) const override; int ways_delete(osmid_t id); int way_changed(osmid_t id); diff --git a/middle.hpp b/middle.hpp index 7b5f76ffa..0e6a6ab2b 100644 --- a/middle.hpp +++ b/middle.hpp @@ -23,9 +23,12 @@ struct options_t; struct middle_query_t { virtual ~middle_query_t() {} - virtual size_t nodes_get_list(nodelist_t &out, - osmium::WayNodeList const &nds, - reprojection const *proj) const = 0; + /** + * Retrives node locations for the given node list. + * + * The locations are saved directly in the input list. + */ + virtual size_t nodes_get_list(osmium::WayNodeList *nodes) const = 0; /** * Retrives a single way from the ways storage @@ -40,7 +43,17 @@ struct middle_query_t { */ virtual bool ways_get(osmid_t id, osmium::memory::Buffer &buffer) const = 0; - virtual size_t ways_get_list(idlist_t const &ids, osmium::memory::Buffer &buffer) const = 0; + /** + * Retrives the way members of a relation and stores them in + * the given osmium buffer. + * + * \param rel Relation to get the members for. + * \param[out] roles Roles for the ways that where retrived. + * \param[out] buffer Buffer where to store the members in. + */ + virtual size_t + rel_way_members_get(osmium::Relation const &rel, rolelist_t *roles, + osmium::memory::Buffer &buffer) const = 0; /** * Retrives a single relation from the relation storage diff --git a/node-persistent-cache.cpp b/node-persistent-cache.cpp index 8d962646c..45f96de58 100644 --- a/node-persistent-cache.cpp +++ b/node-persistent-cache.cpp @@ -2,7 +2,6 @@ #include "node-persistent-cache.hpp" #include "options.hpp" -#include "reprojection.hpp" void node_persistent_cache::set(osmid_t id, const osmium::Location &coord) { @@ -25,30 +24,27 @@ osmium::Location node_persistent_cache::get(osmid_t id) return osmium::Location(); } -size_t node_persistent_cache::get_list(nodelist_t &out, - osmium::WayNodeList const &nds, - reprojection const *proj) +size_t node_persistent_cache::get_list(osmium::WayNodeList *nodes) { - assert(nds.empty()); - out.reserve(nds.size()); + size_t count = 0; - for (auto const &n : nds) { - try { - /* Check cache first */ - auto loc = m_ram_cache->get(n.ref()); - if (!loc.valid() && n.ref() >= 0) { + for (auto &n : *nodes) { + auto loc = m_ram_cache->get(n.ref()); + /* Check cache first */ + if (!loc.valid() && n.ref() >= 0) { + try { loc = m_index->get( - static_cast(n.ref())); - } - if (loc.valid()) { - auto coord = proj->reproject(loc); - out.emplace_back(coord.x, coord.y); + static_cast(n.ref())); + } catch (osmium::not_found const &) { } - } catch (osmium::not_found const &) { + } + n.set_location(loc); + if (loc.valid()) { + ++count; } } - return out.size(); + return count; } node_persistent_cache::node_persistent_cache( diff --git a/node-persistent-cache.hpp b/node-persistent-cache.hpp index e0b10b6e4..1902d963f 100644 --- a/node-persistent-cache.hpp +++ b/node-persistent-cache.hpp @@ -21,8 +21,7 @@ class node_persistent_cache void set(osmid_t id, osmium::Location const &coord); osmium::Location get(osmid_t id); - size_t get_list(nodelist_t &out, osmium::WayNodeList const &nds, - reprojection const *proj); + size_t get_list(osmium::WayNodeList *nodes); private: // Dense node cache for unsigned IDs only diff --git a/options.cpp b/options.cpp index 9d032c761..7aef5189f 100644 --- a/options.cpp +++ b/options.cpp @@ -61,7 +61,6 @@ namespace {"drop", 0, 0, 206}, {"unlogged", 0, 0, 207}, {"flat-nodes",1,0,209}, - {"exclude-invalid-polygon",0,0,210}, {"tag-transform-script",1,0,212}, {"reproject-area",0,0,213}, {0, 0, 0, 0} @@ -204,7 +203,6 @@ namespace -K|--keep-coastlines Keep coastline data rather than filtering it out.\n\ By default natural=coastline tagged data will be discarded\n\ because renderers usually have shape files for them.\n\ - --exclude-invalid-polygon do not attempt to recover invalid geometries.\n\ --reproject-area compute area column using spherical mercator coordinates.\n\ -h|--help Help information.\n\ -v|--verbose Verbose output.\n"); @@ -278,7 +276,7 @@ options_t::options_t() alloc_chunkwise(ALLOC_SPARSE), #endif droptemp(false), unlogged(false), hstore_match_only(false), - flat_node_cache_enabled(false), excludepoly(false), reproject_area(false), + flat_node_cache_enabled(false), reproject_area(false), flat_node_file(boost::none), tag_transform_script(boost::none), tag_transform_node_func(boost::none), tag_transform_way_func(boost::none), tag_transform_rel_func(boost::none), tag_transform_rel_mem_func(boost::none), @@ -453,9 +451,6 @@ options_t::options_t(int argc, char *argv[]): options_t() flat_node_cache_enabled = true; flat_node_file = optarg; break; - case 210: - excludepoly = true; - break; case 211: enable_hstore_index = true; break; diff --git a/options.hpp b/options.hpp index 8b1490410..6865a0cc8 100644 --- a/options.hpp +++ b/options.hpp @@ -71,7 +71,6 @@ struct options_t { bool unlogged; ///< use unlogged tables where possible bool hstore_match_only; ///< only copy rows that match an explicitly listed key bool flat_node_cache_enabled; - bool excludepoly; bool reproject_area; boost::optional flat_node_file; /** diff --git a/osmdata.cpp b/osmdata.cpp index b713e2c8c..d9e03f2f6 100644 --- a/osmdata.cpp +++ b/osmdata.cpp @@ -45,9 +45,9 @@ int osmdata_t::node_add(osmium::Node const &node) return status; } -int osmdata_t::way_add(osmium::Way const &way) +int osmdata_t::way_add(osmium::Way *way) { - mid->ways_set(way); + mid->ways_set(*way); int status = 0; for (auto& out: outs) { @@ -84,20 +84,19 @@ int osmdata_t::node_modify(osmium::Node const &node) return status; } -int osmdata_t::way_modify(osmium::Way const &way) +int osmdata_t::way_modify(osmium::Way *way) { - idlist_t nodes(way.nodes()); slim_middle_t *slim = dynamic_cast(mid.get()); - slim->ways_delete(way.id()); - slim->ways_set(way); + slim->ways_delete(way->id()); + slim->ways_set(*way); int status = 0; for (auto& out: outs) { status |= out->way_modify(way); } - slim->way_changed(way.id()); + slim->way_changed(way->id()); return status; } diff --git a/osmdata.hpp b/osmdata.hpp index 945ca626e..8759262d9 100644 --- a/osmdata.hpp +++ b/osmdata.hpp @@ -28,11 +28,11 @@ class osmdata_t { void stop(); int node_add(osmium::Node const &node); - int way_add(osmium::Way const &way); + int way_add(osmium::Way *way); int relation_add(osmium::Relation const &rel); int node_modify(osmium::Node const &node); - int way_modify(osmium::Way const &way); + int way_modify(osmium::Way *way); int relation_modify(osmium::Relation const &rel); int node_delete(osmid_t id); diff --git a/osmium-builder.cpp b/osmium-builder.cpp new file mode 100644 index 000000000..5d1ec506a --- /dev/null +++ b/osmium-builder.cpp @@ -0,0 +1,402 @@ +#include +#include +#include +#include + +#include + +#include "osmium-builder.hpp" + +namespace { + +inline double distance(osmium::geom::Coordinates p1, + osmium::geom::Coordinates p2) +{ + double dx = p1.x - p2.x; + double dy = p1.y - p2.y; + return std::sqrt(dx * dx + dy * dy); +} + +inline osmium::geom::Coordinates interpolate(osmium::geom::Coordinates p1, + osmium::geom::Coordinates p2, + double frac) +{ + return osmium::geom::Coordinates(frac * (p1.x - p2.x) + p2.x, + frac * (p1.y - p2.y) + p2.y); +} + +template +inline void add_nodes_to_builder(osmium::builder::WayNodeListBuilder &builder, + ITERATOR const &begin, ITERATOR const &end, + bool skip_first) +{ + auto it = begin; + if (skip_first) { + ++it; + } + + while (it != end) { + if (it->location().valid()) { + builder.add_node_ref(*it); + } + ++it; + } +} + +} // name space + +namespace geom { + +osmium_builder_t::wkb_t +osmium_builder_t::get_wkb_node(osmium::Location const &loc) const +{ + return m_writer.make_point(m_proj->reproject(loc)); +} + +osmium_builder_t::wkbs_t +osmium_builder_t::get_wkb_line(osmium::WayNodeList const &nodes, bool do_split) +{ + wkbs_t ret; + + double const split_at = m_proj->target_latlon() ? 1 : 100 * 1000; + + double dist = 0; + osmium::geom::Coordinates prev_pt; + m_writer.linestring_start(); + size_t curlen = 0; + + for (auto const &node : nodes) { + if (!node.location().valid()) + continue; + + auto const this_pt = m_proj->reproject(node.location()); + if (prev_pt.valid()) { + if (prev_pt == this_pt) { + continue; + } + + if (do_split) { + double const delta = distance(prev_pt, this_pt); + + // figure out if the addition of this point would take the total + // length of the line in `segment` over the `split_at` distance. + + if (dist + delta > split_at) { + size_t const splits = + (size_t)std::floor((dist + delta) / split_at); + // use the splitting distance to split the current segment up + // into as many parts as necessary to keep each part below + // the `split_at` distance. + osmium::geom::Coordinates ipoint; + for (size_t j = 0; j < splits; ++j) { + double const frac = + ((double)(j + 1) * split_at - dist) / delta; + ipoint = interpolate(this_pt, prev_pt, frac); + m_writer.add_location(ipoint); + ret.push_back(m_writer.linestring_finish(curlen + 1)); + // start a new segment + m_writer.linestring_start(); + m_writer.add_location(ipoint); + curlen = 1; + } + // reset the distance based on the final splitting point for + // the next iteration. + if (this_pt == ipoint) { + dist = 0; + m_writer.linestring_start(); + curlen = 0; + } else { + dist = distance(this_pt, ipoint); + } + } else { + dist += delta; + } + } + } + + m_writer.add_location(this_pt); + ++curlen; + + prev_pt = this_pt; + } + + if (curlen > 1) { + ret.push_back(m_writer.linestring_finish(curlen)); + } + + return ret; +} + +osmium_builder_t::wkb_t +osmium_builder_t::get_wkb_polygon(osmium::Way const &way) +{ + osmium::area::AssemblerConfig area_config; + area_config.ignore_invalid_locations = true; + osmium::area::GeomAssembler assembler{area_config}; + + m_buffer.clear(); + if (!assembler(way, m_buffer)) { + return wkb_t(); + } + + auto wkbs = create_polygons(m_buffer.get(0)); + + return wkbs.empty() ? wkb_t() : wkbs[0]; +} + +osmium_builder_t::wkbs_t +osmium_builder_t::get_wkb_multipolygon(osmium::Relation const &rel, + osmium::memory::Buffer const &ways) +{ + wkbs_t ret; + osmium::area::AssemblerConfig area_config; + area_config.ignore_invalid_locations = true; + osmium::area::GeomAssembler assembler{area_config}; + + m_buffer.clear(); + if (assembler(rel, ways, m_buffer)) { + if (m_build_multigeoms) { + ret.push_back(create_multipolygon(m_buffer.get(0))); + } else { + ret = create_polygons(m_buffer.get(0)); + } + } + + return ret; +} + +osmium_builder_t::wkbs_t +osmium_builder_t::get_wkb_multiline(osmium::memory::Buffer const &ways, + bool do_split) +{ + // make a list of all endpoints + using endpoint_t = std::tuple; + std::vector endpoints; + // and a list of way connections + enum lmt : size_t + { + NOCONN = -1UL + }; + std::vector> conns; + + // initialise the two lists + for (auto const &w : ways.select()) { + if (w.nodes().size() > 1) { + endpoints.emplace_back(w.nodes().front().ref(), conns.size(), true); + endpoints.emplace_back(w.nodes().back().ref(), conns.size(), false); + conns.emplace_back(NOCONN, &w, NOCONN); + } + } + // sort by node id + std::sort(endpoints.begin(), endpoints.end()); + // now fill the connection list based on the sorted list + { + endpoint_t const *prev = nullptr; + for (auto const &pt : endpoints) { + if (prev) { + if (std::get<0>(*prev) == std::get<0>(pt)) { + auto previd = std::get<1>(*prev); + auto ptid = std::get<1>(pt); + if (std::get<2>(*prev)) { + std::get<0>(conns[previd]) = ptid; + } else { + std::get<2>(conns[previd]) = ptid; + } + if (std::get<2>(pt)) { + std::get<0>(conns[ptid]) = previd; + } else { + std::get<2>(conns[ptid]) = previd; + } + prev = nullptr; + continue; + } + } + + prev = &pt; + } + } + + wkbs_t ret; + + size_t done_ways = 0; + size_t todo_ways = conns.size(); + for (size_t i = 0; i < todo_ways; ++i) { + if (!std::get<1>(conns[i]) || (std::get<0>(conns[i]) != NOCONN && + std::get<2>(conns[i]) != NOCONN)) { + continue; // way already done or not the beginning of a segment + } + + m_buffer.clear(); + osmium::builder::WayNodeListBuilder wnl_builder(m_buffer); + size_t prev = NOCONN; + size_t cur = i; + + do { + auto &conn = conns[cur]; + assert(std::get<1>(conn)); + auto &nl = std::get<1>(conn)->nodes(); + bool skip_first = prev != NOCONN; + bool forward = std::get<0>(conn) == prev; + prev = cur; + // add way nodes + if (forward) { + add_nodes_to_builder(wnl_builder, nl.cbegin(), nl.cend(), + skip_first); + cur = std::get<2>(conn); + } else { + add_nodes_to_builder(wnl_builder, nl.crbegin(), nl.crend(), + skip_first); + cur = std::get<0>(conn); + } + // mark way as done + std::get<1>(conns[prev]) = nullptr; + ++done_ways; + } while (cur != NOCONN); + + // found a line end, create the wkbs + m_buffer.commit(); + auto linewkbs = + get_wkb_line(m_buffer.get(0), do_split); + std::move(linewkbs.begin(), linewkbs.end(), + std::inserter(ret, ret.end())); + } + + if (done_ways < todo_ways) { + // oh dear, there must be circular ways without an end + // need to do the same shebang again + for (size_t i = 0; i < todo_ways; ++i) { + if (!std::get<1>(conns[i])) { + continue; // way already done + } + + m_buffer.clear(); + osmium::builder::WayNodeListBuilder wnl_builder(m_buffer); + size_t prev = std::get<0>(conns[i]); + size_t cur = i; + bool skip_first = false; + + do { + auto &conn = conns[cur]; + assert(std::get<1>(conn)); + auto &nl = std::get<1>(conn)->nodes(); + bool forward = std::get<0>(conn) == prev; + prev = cur; + if (forward) { + // add way forwards + add_nodes_to_builder(wnl_builder, nl.cbegin(), nl.cend(), + skip_first); + cur = std::get<2>(conn); + } else { + // add way backwards + add_nodes_to_builder(wnl_builder, nl.crbegin(), nl.crend(), + skip_first); + cur = std::get<0>(conn); + } + // mark way as done + std::get<1>(conns[prev]) = nullptr; + skip_first = true; + } while (cur != i); + + // found a line end, create the wkbs + m_buffer.commit(); + auto linewkbs = + get_wkb_line(m_buffer.get(0), do_split); + std::move(linewkbs.begin(), linewkbs.end(), + std::inserter(ret, ret.end())); + } + } + + if (!do_split && !ret.empty()) { + auto num_lines = ret.size(); + m_writer.multilinestring_start(); + for (auto const &line : ret) { + m_writer.add_sub_geometry(line); + } + ret.clear(); + ret.push_back(m_writer.multilinestring_finish(num_lines)); + } + + return ret; +} + +size_t osmium_builder_t::add_mp_points(const osmium::NodeRefList &nodes) +{ + size_t num_points = 0; + osmium::Location last_location; + for (const osmium::NodeRef &node_ref : nodes) { + if (node_ref.location().valid() && + last_location != node_ref.location()) { + last_location = node_ref.location(); + m_writer.add_location(m_proj->reproject(last_location)); + ++num_points; + } + } + + return num_points; +} + +osmium_builder_t::wkb_t +osmium_builder_t::create_multipolygon(osmium::Area const &area) +{ + wkb_t ret; + + auto polys = create_polygons(area); + + switch (polys.size()) { + case 0: + break; //nothing + case 1: + ret = polys[0]; + break; + default: + m_writer.multipolygon_start(); + for (auto const &p : polys) { + m_writer.add_sub_geometry(p); + } + ret = m_writer.multipolygon_finish(polys.size()); + break; + } + + return ret; +} + +osmium_builder_t::wkbs_t +osmium_builder_t::create_polygons(osmium::Area const &area) +{ + wkbs_t ret; + + try { + size_t num_rings = 0; + + for (auto it = area.cbegin(); it != area.cend(); ++it) { + if (it->type() == osmium::item_type::outer_ring) { + auto &ring = static_cast(*it); + if (num_rings > 0) { + ret.push_back(m_writer.polygon_finish(num_rings)); + num_rings = 0; + } + m_writer.polygon_start(); + m_writer.polygon_ring_start(); + auto num_points = add_mp_points(ring); + m_writer.polygon_ring_finish(num_points); + ++num_rings; + } else if (it->type() == osmium::item_type::inner_ring) { + auto &ring = static_cast(*it); + m_writer.polygon_ring_start(); + auto num_points = add_mp_points(ring); + m_writer.polygon_ring_finish(num_points); + ++num_rings; + } + } + + if (num_rings > 0) { + ret.push_back(m_writer.polygon_finish(num_rings)); + } + + } catch (osmium::geometry_error &e) { /* ignored */ + } + + return ret; +} + +} // name space diff --git a/osmium-builder.hpp b/osmium-builder.hpp new file mode 100644 index 000000000..ce6ba349a --- /dev/null +++ b/osmium-builder.hpp @@ -0,0 +1,51 @@ +#ifndef OSMIUM_BUILDER_H +#define OSMIUM_BUILDER_H + +#include +#include +#include + +#include +#include + +#include "reprojection.hpp" +#include "wkb.hpp" + +namespace geom { + +class osmium_builder_t +{ +public: + typedef std::string wkb_t; + typedef std::vector wkbs_t; + + explicit osmium_builder_t(std::shared_ptr const &proj, + bool build_multigeoms) + : m_proj(proj), m_buffer(1024, osmium::memory::Buffer::auto_grow::yes), + m_writer(m_proj->target_srs()), m_build_multigeoms(build_multigeoms) + { + } + + wkb_t get_wkb_node(osmium::Location const &loc) const; + wkbs_t get_wkb_line(osmium::WayNodeList const &way, bool do_split); + wkb_t get_wkb_polygon(osmium::Way const &way); + + wkbs_t get_wkb_multipolygon(osmium::Relation const &rel, + osmium::memory::Buffer const &ways); + wkbs_t get_wkb_multiline(osmium::memory::Buffer const &ways, bool split); + +private: + wkb_t create_multipolygon(osmium::Area const &area); + wkbs_t create_polygons(osmium::Area const &area); + size_t add_mp_points(const osmium::NodeRefList &nodes); + + std::shared_ptr m_proj; + // internal buffer for creating areas + osmium::memory::Buffer m_buffer; + ewkb::writer_t m_writer; + bool m_build_multigeoms; +}; + +} // namespace + +#endif // OSMIUM_BUILDER_H diff --git a/osmtypes.hpp b/osmtypes.hpp index 6c830ef4e..14760dc6f 100644 --- a/osmtypes.hpp +++ b/osmtypes.hpp @@ -22,9 +22,6 @@ typedef int64_t osmid_t; #define PRIdOSMID PRId64 #define POSTGRES_OSMID_TYPE "int8" -typedef std::vector nodelist_t; -typedef std::vector multinodelist_t; - struct member { osmium::item_type type; osmid_t id; diff --git a/output-gazetteer.cpp b/output-gazetteer.cpp index 8125d6013..4deb42d97 100644 --- a/output-gazetteer.cpp +++ b/output-gazetteer.cpp @@ -2,13 +2,13 @@ #include #include -#include "osmtypes.hpp" #include "middle.hpp" -#include "pgsql.hpp" -#include "reprojection.hpp" -#include "output-gazetteer.hpp" #include "options.hpp" +#include "osmtypes.hpp" +#include "output-gazetteer.hpp" +#include "pgsql.hpp" #include "util.hpp" +#include "wkb.hpp" #include #include @@ -37,7 +37,6 @@ #define CREATE_PLACE_ID_INDEX \ "CREATE INDEX place_id_idx ON place USING BTREE (osm_type, osm_id) %s %s" - void place_tag_processor::clear() { // set members to sane defaults @@ -486,9 +485,8 @@ void place_tag_processor::copy_out(osmium::OSMObject const &o, } buffer += "\t"; } - // geometry - buffer += srid_str; - buffer += geom; + // add the geometry - encoding it to hex along the way + ewkb::writer_t::write_as_hex(buffer, geom); buffer += '\n'; } } @@ -611,7 +609,6 @@ int output_gazetteer_t::connect() { int output_gazetteer_t::start() { int srid = m_options.projection->target_srs(); - builder.set_exclude_broken_polygon(m_options.excludepoly); places.srid_str = (boost::format("SRID=%1%;") % srid).str(); @@ -675,34 +672,42 @@ int output_gazetteer_t::process_node(osmium::Node const &node) /* Are we interested in this item? */ if (places.has_data()) { - auto c = reproj->reproject(node.location()); - std::string wkt = (point_fmt % c.x % c.y).str(); - places.copy_out(node, wkt, buffer); + auto wkb = m_builder.get_wkb_node(node.location()); + places.copy_out(node, wkb, buffer); flush_place_buffer(); } return 0; } -int output_gazetteer_t::process_way(osmium::Way const &way) +int output_gazetteer_t::process_way(osmium::Way *way) { - places.process_tags(way); + places.process_tags(*way); if (m_options.append) - delete_unused_classes('W', way.id()); + delete_unused_classes('W', way->id()); /* Are we interested in this item? */ if (places.has_data()) { /* Fetch the node details */ - nodelist_t nodes; - m_mid->nodes_get_list(nodes, way.nodes(), reproj.get()); + m_mid->nodes_get_list(&(way->nodes())); /* Get the geometry of the object */ - auto geom = builder.get_wkb_simple(nodes, 1); - if (geom.valid()) { - places.copy_out(way, geom.geom, buffer); - flush_place_buffer(); + geom::osmium_builder_t::wkb_t geom; + if (way->is_closed()) { + geom = m_builder.get_wkb_polygon(*way); } + if (geom.empty()) { + auto wkbs = m_builder.get_wkb_line(way->nodes(), false); + if (wkbs.empty()) { + return 0; + } + + geom = wkbs[0]; + } + + places.copy_out(*way, geom, buffer); + flush_place_buffer(); } return 0; @@ -736,45 +741,27 @@ int output_gazetteer_t::process_relation(osmium::Relation const &rel) return 0; /* get the boundary path (ways) */ - idlist_t xid2; - for (const auto& member: rel.members()) { - /* only interested in ways */ - if (member.type() == osmium::item_type::way) - xid2.push_back(member.ref()); - } + osmium_buffer.clear(); + auto num_ways = m_mid->rel_way_members_get(rel, nullptr, osmium_buffer); - if (xid2.empty()) { + if (num_ways == 0) { if (m_options.append) delete_unused_full('R', rel.id()); return 0; } - osmium_buffer.clear(); - auto num_ways = m_mid->ways_get_list(xid2, osmium_buffer); - multinodelist_t xnodes(num_ways); - size_t i = 0; - for (auto const &w : osmium_buffer.select()) { - m_mid->nodes_get_list(xnodes[i++], w.nodes(), reproj.get()); + for (auto &w : osmium_buffer.select()) { + m_mid->nodes_get_list(&(w.nodes())); } - if (!is_waterway) { - auto geoms = builder.build_both(xnodes, 1, 1, 1000000, rel.id()); - for (const auto& geom: geoms) { - if (geom.is_polygon()) { - places.copy_out(rel, geom.geom, buffer); - flush_place_buffer(); - } else { - /* add_polygon_error('R', id, "boundary", "adminitrative", &names, countrycode, wkt); */ - } - } - } else { - /* waterways result in multilinestrings */ - auto geom = builder.build_multilines(xnodes, rel.id()); - if (geom.valid()) { - places.copy_out(rel, geom.geom, buffer); - flush_place_buffer(); - } + auto geoms = is_waterway + ? m_builder.get_wkb_multiline(osmium_buffer, false) + : m_builder.get_wkb_multipolygon(rel, osmium_buffer); + + if (!geoms.empty()) { + places.copy_out(rel, geoms[0], buffer); + flush_place_buffer(); } return 0; diff --git a/output-gazetteer.hpp b/output-gazetteer.hpp index e90a08f5d..57cd82d98 100644 --- a/output-gazetteer.hpp +++ b/output-gazetteer.hpp @@ -8,7 +8,7 @@ #include #include -#include "geometry-builder.hpp" +#include "osmium-builder.hpp" #include "osmtypes.hpp" #include "output.hpp" #include "pgsql.hpp" @@ -130,33 +130,22 @@ class place_tag_processor class output_gazetteer_t : public output_t { public: - output_gazetteer_t(const middle_query_t* mid_, const options_t &options_) - : output_t(mid_, options_), - Connection(NULL), - ConnectionDelete(NULL), - ConnectionError(NULL), - copy_active(false), - reproj(options_.projection), - single_fmt("%1%"), - point_fmt("POINT(%.15g %.15g)"), + output_gazetteer_t(const middle_query_t *mid_, const options_t &options_) + : output_t(mid_, options_), Connection(NULL), ConnectionDelete(NULL), + ConnectionError(NULL), copy_active(false), + m_builder(options_.projection, true), single_fmt("%1%"), osmium_buffer(PLACE_BUFFER_SIZE, osmium::memory::Buffer::auto_grow::yes) { buffer.reserve(PLACE_BUFFER_SIZE); } - output_gazetteer_t(const output_gazetteer_t& other) - : output_t(other.m_mid, other.m_options), - Connection(NULL), - ConnectionDelete(NULL), - ConnectionError(NULL), - copy_active(false), - reproj(other.reproj), - single_fmt(other.single_fmt), - point_fmt(other.point_fmt), + output_gazetteer_t(const output_gazetteer_t &other) + : output_t(other.m_mid, other.m_options), Connection(NULL), + ConnectionDelete(NULL), ConnectionError(NULL), copy_active(false), + m_builder(other.m_options.projection, true), single_fmt(other.single_fmt), osmium_buffer(PLACE_BUFFER_SIZE, osmium::memory::Buffer::auto_grow::yes) { buffer.reserve(PLACE_BUFFER_SIZE); - builder.set_exclude_broken_polygon(m_options.excludepoly); connect(); } @@ -184,7 +173,7 @@ class output_gazetteer_t : public output_t { return process_node(node); } - int way_add(osmium::Way const &way) override + int way_add(osmium::Way *way) override { return process_way(way); } @@ -199,7 +188,7 @@ class output_gazetteer_t : public output_t { return process_node(node); } - int way_modify(osmium::Way const &way) override + int way_modify(osmium::Way *way) override { return process_way(way); } @@ -234,7 +223,7 @@ class output_gazetteer_t : public output_t { void delete_unused_classes(char osm_type, osmid_t osm_id); void delete_place(char osm_type, osmid_t osm_id); int process_node(osmium::Node const &node); - int process_way(osmium::Way const &way); + int process_way(osmium::Way *way); int process_relation(osmium::Relation const &rel); int connect(); @@ -267,14 +256,11 @@ class output_gazetteer_t : public output_t { std::string buffer; place_tag_processor places; - geometry_builder builder; - - std::shared_ptr reproj; + geom::osmium_builder_t m_builder; // string formatters // Need to be part of the class, so we have one per thread. boost::format single_fmt; - boost::format point_fmt; osmium::memory::Buffer osmium_buffer; }; diff --git a/output-multi.cpp b/output-multi.cpp index 6df394c36..263b0f625 100644 --- a/output-multi.cpp +++ b/output-multi.cpp @@ -1,12 +1,12 @@ #include "output-multi.hpp" -#include "taginfo_impl.hpp" +#include "expire-tiles.hpp" +#include "id-tracker.hpp" +#include "middle.hpp" +#include "options.hpp" #include "table.hpp" +#include "taginfo_impl.hpp" #include "tagtransform.hpp" -#include "options.hpp" -#include "middle.hpp" -#include "id-tracker.hpp" -#include "geometry-builder.hpp" -#include "expire-tiles.hpp" +#include "wkb.hpp" #include #include @@ -33,7 +33,8 @@ output_multi_t::output_multi_t(const std::string &name, ways_done_tracker(new id_tracker()), m_expire(m_options.expire_tiles_zoom, m_options.expire_tiles_max_bbox, m_options.projection), - buffer(1024, osmium::memory::Buffer::auto_grow::yes) + buffer(1024, osmium::memory::Buffer::auto_grow::yes), + m_way_area(m_export_list->has_column(m_osm_type, "way_area")) { } @@ -49,7 +50,8 @@ output_multi_t::output_multi_t(const output_multi_t &other) ways_done_tracker(other.ways_done_tracker), m_expire(m_options.expire_tiles_zoom, m_options.expire_tiles_max_bbox, m_options.projection), - buffer(1024, osmium::memory::Buffer::auto_grow::yes) + buffer(1024, osmium::memory::Buffer::auto_grow::yes), + m_way_area(other.m_way_area) { } @@ -117,7 +119,7 @@ int output_multi_t::pending_way(osmid_t id, int exists) { buffer.clear(); if (m_mid->ways_get(id, buffer)) { // Output the way - ret = reprocess_way(buffer.get(0), exists); + ret = reprocess_way(&buffer.get(0), exists); } return ret; @@ -194,8 +196,8 @@ int output_multi_t::node_add(osmium::Node const &node) return 0; } -int output_multi_t::way_add(osmium::Way const &way) { - if (m_processor->interests(geometry_processor::interest_way) && way.nodes().size() > 1) { +int output_multi_t::way_add(osmium::Way *way) { + if (m_processor->interests(geometry_processor::interest_way) && way->nodes().size() > 1) { return process_way(way); } return 0; @@ -224,10 +226,10 @@ int output_multi_t::node_modify(osmium::Node const &node) return 0; } -int output_multi_t::way_modify(osmium::Way const &way) { +int output_multi_t::way_modify(osmium::Way *way) { if (m_processor->interests(geometry_processor::interest_way)) { // TODO - need to know it's a way? - delete_from_output(way.id()); + delete_from_output(way->id()); // TODO: need to mark any relations using it - depends on what // type of output this is... delegate to the geometry processor?? @@ -282,68 +284,63 @@ int output_multi_t::process_node(osmium::Node const &node) auto filter = m_tagtransform->filter_tags(node, 0, 0, *m_export_list.get(), outtags, true); if (!filter) { - auto c = m_proj->reproject(node.location()); // grab its geom - auto geom = m_processor->process_node(c.y, c.x); - if (geom.valid()) { - m_expire.from_bbox(c.x, c.y, c.x, c.y); - copy_node_to_table(node.id(), geom.geom, outtags); + auto geom = m_processor->process_node(node.location()); + if (!geom.empty()) { + m_expire.from_wkb(geom.c_str(), node.id()); + copy_node_to_table(node.id(), geom, outtags); } } return 0; } -int output_multi_t::reprocess_way(osmium::Way const &way, bool exists) +int output_multi_t::reprocess_way(osmium::Way *way, bool exists) { //if the way could exist already we have to make the relation pending and reprocess it later //but only if we actually care about relations if(m_processor->interests(geometry_processor::interest_relation) && exists) { - way_delete(way.id()); - const std::vector rel_ids = m_mid->relations_using_way(way.id()); + way_delete(way->id()); + const std::vector rel_ids = + m_mid->relations_using_way(way->id()); for (std::vector::const_iterator itr = rel_ids.begin(); itr != rel_ids.end(); ++itr) { rels_pending_tracker.mark(*itr); } } //check if we are keeping this way - int polygon = 0; taglist_t outtags; - unsigned int filter = m_tagtransform->filter_tags(way, &polygon, 0, - *m_export_list.get(), outtags, true); + unsigned int filter = m_tagtransform->filter_tags( + *way, 0, 0, *m_export_list.get(), outtags, true); if (!filter) { - //grab its geom - nodelist_t nodes; - m_mid->nodes_get_list(nodes, way.nodes(), m_proj.get()); - auto geom = m_processor->process_way(nodes); - if (geom.valid()) { - copy_to_table(way.id(), geom, outtags, polygon); + m_mid->nodes_get_list(&(way->nodes())); + auto geom = m_processor->process_way(*way); + if (!geom.empty()) { + copy_to_table(way->id(), geom, outtags); } } return 0; } -int output_multi_t::process_way(osmium::Way const &way) { +int output_multi_t::process_way(osmium::Way *way) { //check if we are keeping this way - int polygon = 0, roads = 0; taglist_t outtags; - auto filter = m_tagtransform->filter_tags(way, &polygon, &roads, - *m_export_list.get(), outtags, true); + auto filter = m_tagtransform->filter_tags(*way, 0, 0, *m_export_list.get(), outtags, true); if (!filter) { //get the geom from the middle - if (m_way_helper.set(way.nodes(), m_mid, m_proj.get()) < 1) + if (m_mid->nodes_get_list(&(way->nodes())) < 1) return 0; //grab its geom - auto geom = m_processor->process_way(m_way_helper.node_cache); + auto geom = m_processor->process_way(*way); - if (geom.valid()) { + if (!geom.empty()) { //if we are also interested in relations we need to mark //this way pending just in case it shows up in one if (m_processor->interests(geometry_processor::interest_relation)) { - ways_pending_tracker.mark(way.id()); + ways_pending_tracker.mark(way->id()); } else { // We wouldn't be interested in this as a relation, so no need to mark it pending. // TODO: Does this imply anything for non-multipolygon relations? - copy_to_table(way.id(), geom, outtags, polygon); + copy_to_table(way->id(), geom, outtags); } } } @@ -365,7 +362,7 @@ int output_multi_t::process_relation(osmium::Relation const &rel, if (!filter) { //TODO: move this into geometry processor, figure a way to come back for tag transform //grab ways/nodes of the members in the relation, bail if none were used - if (m_relation_helper.set(rel.members(), (middle_t*)m_mid) < 1) + if (m_relation_helper.set(rel, (middle_t *)m_mid) < 1) return 0; //filter the tags on each member because we got them from the middle @@ -384,7 +381,7 @@ int output_multi_t::process_relation(osmium::Relation const &rel, //if the export list did not include the type tag. //TODO: find a less hacky way to do the matching/superseded and tag copying stuff without //all this trickery - int make_boundary, make_polygon = 1; + int make_boundary, make_polygon; taglist_t outtags; filter = m_tagtransform->filter_rel_member_tags(rel_outtags, filtered, m_relation_helper.roles, &m_relation_helper.superseded.front(), @@ -392,23 +389,17 @@ int output_multi_t::process_relation(osmium::Relation const &rel, *m_export_list.get(), outtags, true); if (!filter) { - auto nodes = - m_relation_helper.get_nodes((middle_t *)m_mid, m_proj.get()); - auto geoms = m_processor->process_relation(nodes); + m_relation_helper.add_way_locations((middle_t *)m_mid); + auto geoms = + m_processor->process_relation(rel, m_relation_helper.data); for (const auto geom : geoms) { - // TODO: we actually have the nodes in the m_relation_helper and - // could use them - // instead of having to reparse the wkb in the expiry code - m_expire.from_wkb(geom.geom.c_str(), -rel.id()); - // what part of the code relies on relation members getting negative - // ids? - copy_to_table(-rel.id(), geom, outtags, make_polygon); + copy_to_table(-rel.id(), geom, outtags); } //TODO: should this loop be inside the if above just in case? //take a look at each member to see if its superseded (tags on it matched the tags on the relation) size_t i = 0; - for (auto const &w : m_relation_helper.way_iterator()) { + for (auto const &w : m_relation_helper.data.select()) { //tags matched so we are keeping this one with this relation if (m_relation_helper.superseded[i]) { //just in case it wasnt previously with this relation we get rid of them @@ -426,7 +417,9 @@ int output_multi_t::process_relation(osmium::Relation const &rel, return 0; } -void output_multi_t::copy_node_to_table(osmid_t id, const std::string &geom, taglist_t &tags) { +void output_multi_t::copy_node_to_table(osmid_t id, std::string const &geom, + taglist_t &tags) +{ m_table->write_row(id, tags, geom); } @@ -439,25 +432,23 @@ void output_multi_t::copy_node_to_table(osmid_t id, const std::string &geom, tag * * \pre geom must be valid. */ -void output_multi_t::copy_to_table(const osmid_t id, const geometry_builder::pg_geom_t &geom, taglist_t &tags, int polygon) { - if (geom.is_polygon()) { +void output_multi_t::copy_to_table(const osmid_t id, + geometry_processor::wkb_t const &geom, + taglist_t &tags) +{ + // XXX really should depend on expected output type + if (m_way_area) { // It's a polygon table (implied by it turning into a poly), // and it got formed into a polygon, so expire as a polygon and write the geom - m_expire.from_nodes_poly(m_way_helper.node_cache, id); - if (geom.area > 0.0) { - char tmp[32]; - snprintf(tmp, sizeof(tmp), "%g", geom.area); - tags.push_override(tag_t("way_area", tmp)); - } - m_table->write_row(id, tags, geom.geom); - } else { - // Linestring - if (!polygon) { - // non-polygons are okay - m_expire.from_nodes_line(m_way_helper.node_cache); - m_table->write_row(id, tags, geom.geom); - } + auto area = + ewkb::parser_t(geom).get_area(); + char tmp[32]; + snprintf(tmp, sizeof(tmp), "%g", area); + tags.push_override(tag_t("way_area", tmp)); } + + m_expire.from_wkb(geom.c_str(), id); + m_table->write_row(id, tags, geom); } void output_multi_t::delete_from_output(osmid_t id) { diff --git a/output-multi.hpp b/output-multi.hpp index 4e093959b..5ff89616a 100644 --- a/output-multi.hpp +++ b/output-multi.hpp @@ -45,11 +45,11 @@ class output_multi_t : public output_t { int pending_relation(osmid_t id, int exists) override; int node_add(osmium::Node const &node) override; - int way_add(osmium::Way const &way) override; + int way_add(osmium::Way *way) override; int relation_add(osmium::Relation const &rel) override; int node_modify(osmium::Node const &node) override; - int way_modify(osmium::Way const &way) override; + int way_modify(osmium::Way *way) override; int relation_modify(osmium::Relation const &rel) override; int node_delete(osmid_t id) override; @@ -65,11 +65,12 @@ class output_multi_t : public output_t { void delete_from_output(osmid_t id); int process_node(osmium::Node const &node); - int process_way(osmium::Way const &way); - int reprocess_way(osmium::Way const &way, bool exists); + int process_way(osmium::Way *way); + int reprocess_way(osmium::Way *way, bool exists); int process_relation(osmium::Relation const &rel, bool exists, bool pending=false); void copy_node_to_table(osmid_t id, const std::string &geom, taglist_t &tags); - void copy_to_table(const osmid_t id, const geometry_builder::pg_geom_t &geom, taglist_t &tags, int polygon); + void copy_to_table(const osmid_t id, geometry_processor::wkb_t const &geom, + taglist_t &tags); std::unique_ptr m_tagtransform; std::unique_ptr m_export_list; @@ -80,9 +81,9 @@ class output_multi_t : public output_t { id_tracker ways_pending_tracker, rels_pending_tracker; std::shared_ptr ways_done_tracker; expire_tiles m_expire; - way_helper m_way_helper; relation_helper m_relation_helper; osmium::memory::Buffer buffer; + bool m_way_area; }; #endif diff --git a/output-null.cpp b/output-null.cpp index e29b6fad7..81d571537 100644 --- a/output-null.cpp +++ b/output-null.cpp @@ -33,7 +33,7 @@ int output_null_t::pending_relation(osmid_t, int) { int output_null_t::node_add(osmium::Node const &) { return 0; } -int output_null_t::way_add(osmium::Way const &) { +int output_null_t::way_add(osmium::Way *) { return 0; } @@ -55,7 +55,7 @@ int output_null_t::relation_delete(osmid_t) { int output_null_t::node_modify(osmium::Node const &) { return 0; } -int output_null_t::way_modify(osmium::Way const &) { +int output_null_t::way_modify(osmium::Way *) { return 0; } diff --git a/output-null.hpp b/output-null.hpp index d56eaf2d9..14f5eb68c 100644 --- a/output-null.hpp +++ b/output-null.hpp @@ -26,11 +26,11 @@ class output_null_t : public output_t { int pending_relation(osmid_t id, int exists) override; int node_add(osmium::Node const &node) override; - int way_add(osmium::Way const &way) override; + int way_add(osmium::Way *way) override; int relation_add(osmium::Relation const &rel) override; int node_modify(osmium::Node const &node) override; - int way_modify(osmium::Way const &way) override; + int way_modify(osmium::Way *way) override; int relation_modify(osmium::Relation const &rel) override; int node_delete(osmid_t id) override; diff --git a/output-pgsql.cpp b/output-pgsql.cpp index 77e69c921..7e2326fd6 100644 --- a/output-pgsql.cpp +++ b/output-pgsql.cpp @@ -38,6 +38,7 @@ #include "tagtransform.hpp" #include "util.hpp" #include "wildcmp.hpp" +#include "wkb.hpp" /* make the diagnostic information work with older versions of * boost - the function signature changed at version 1.54. @@ -70,127 +71,38 @@ E4C1421D5BF24D06053E7DF4940 212696 Oswald Road \N \N \N \N \N \N minor \N \N \N \N \N \N \N 0102000020E610000004000000467D923B6C22D5BFA359D93EE4DF4940B3976DA7AD11D5BF84BBB376DBDF4940997FF44D9A06D5BF4223D8B8FEDF49404D158C4AEA04D 5BF5BB39597FCDF4940 */ -int output_pgsql_t::pgsql_out_way(osmid_t id, taglist_t &outtags, - const nodelist_t &nodes, - int polygon, int roads) +void output_pgsql_t::pgsql_out_way(osmium::Way const &way, taglist_t *tags, + bool polygon, bool roads) { - /* Split long ways after around 1 degree or 100km */ - double split_at; - if (m_options.projection->target_latlon()) - split_at = 1; - else - split_at = 100 * 1000; - - char tmp[32]; - auto wkbs = builder.get_wkb_split(nodes, polygon, split_at); - for (const auto& wkb: wkbs) { - /* FIXME: there should be a better way to detect polygons */ - if (wkb.is_polygon()) { - expire.from_nodes_poly(nodes, id); - if ((wkb.area > 0.0) && m_enable_way_area) { - snprintf(tmp, sizeof(tmp), "%g", wkb.area); - outtags.push_override(tag_t("way_area", tmp)); + if (polygon && way.is_closed()) { + auto wkb = m_builder.get_wkb_polygon(way); + if (!wkb.empty()) { + expire.from_wkb(wkb.c_str(), way.id()); + if (m_enable_way_area) { + char tmp[32]; + auto const area = + m_options.reproject_area + ? ewkb::parser_t(wkb).get_area( + m_options.projection.get()) + : ewkb::parser_t(wkb) + .get_area(); + snprintf(tmp, sizeof(tmp), "%g", area); + tags->push_override(tag_t("way_area", tmp)); } - m_tables[t_poly]->write_row(id, outtags, wkb.geom); - } else { - expire.from_nodes_line(nodes); - m_tables[t_line]->write_row(id, outtags, wkb.geom); - if (roads) - m_tables[t_roads]->write_row(id, outtags, wkb.geom); + m_tables[t_poly]->write_row(way.id(), *tags, wkb); } - } - - return 0; -} - -int output_pgsql_t::pgsql_out_relation(osmid_t id, const taglist_t &rel_tags, - const multinodelist_t &xnodes, const multitaglist_t & xtags, - const idlist_t &xid, const rolelist_t &xrole, - bool pending) -{ - if (xnodes.empty()) - return 0; - - int roads = 0; - int make_polygon = 0; - int make_boundary = 0; - double split_at; - - std::vector members_superseded(xnodes.size(), 0); - taglist_t outtags; - - //if its a route relation make_boundary and make_polygon will be false otherwise one or the other will be true - if (m_tagtransform->filter_rel_member_tags(rel_tags, xtags, xrole, - &(members_superseded[0]), &make_boundary, &make_polygon, &roads, - *m_export_list.get(), outtags)) { - return 0; - } - - /* Split long linear ways after around 1 degree or 100km (polygons not effected) */ - if (m_options.projection->target_latlon()) - split_at = 1; - else - split_at = 100 * 1000; - - //this will either make lines or polygons (unless the lines arent a ring or are less than 3 pts) depending on the tag transform above - //TODO: pick one or the other based on which we expect to care about - auto wkbs = builder.build_both(xnodes, make_polygon, m_options.enable_multi, split_at, id); - - if (wkbs.empty()) { - return 0; - } - - char tmp[32]; - for (const auto& wkb: wkbs) { - expire.from_wkb(wkb.geom.c_str(), -id); - /* FIXME: there should be a better way to detect polygons */ - if (wkb.is_polygon()) { - if ((wkb.area > 0.0) && m_enable_way_area) { - snprintf(tmp, sizeof(tmp), "%g", wkb.area); - outtags.push_override(tag_t("way_area", tmp)); - } - m_tables[t_poly]->write_row(-id, outtags, wkb.geom); - } else { - m_tables[t_line]->write_row(-id, outtags, wkb.geom); - if (roads) - m_tables[t_roads]->write_row(-id, outtags, wkb.geom); - } - } - - /* Tagtransform will have marked those member ways of the relation that - * have fully been dealt with as part of the multi-polygon entry. - * Set them in the database as done and delete their entry to not - * have duplicates */ - //dont do this when working with pending relations as its not needed - if (make_polygon) { - for (size_t i=0; i < xid.size(); i++) { - if (members_superseded[i]) { - pgsql_delete_way_from_output(xid[i]); - if(!pending) - ways_done_tracker->mark(xid[i]); + } else { + for (auto const &wkb : m_builder.get_wkb_line(way.nodes(), true)) { + expire.from_wkb(wkb.c_str(), way.id()); + m_tables[t_line]->write_row(way.id(), *tags, wkb); + if (roads) { + m_tables[t_roads]->write_row(way.id(), *tags, wkb); } } - } - // If the tag transform said the polygon looked like a boundary we want to make that as well - // If we are making a boundary then also try adding any relations which form complete rings - // The linear variants will have already been processed above - if (make_boundary) { - wkbs = builder.build_polygons(xnodes, m_options.enable_multi, id); - for (const auto& wkb: wkbs) { - expire.from_wkb(wkb.geom.c_str(), -id); - if ((wkb.area > 0.0) && m_enable_way_area) { - snprintf(tmp, sizeof(tmp), "%g", wkb.area); - outtags.push_override(tag_t("way_area", tmp)); - } - m_tables[t_poly]->write_row(-id, outtags, wkb.geom); - } } - - return 0; } - void output_pgsql_t::enqueue_ways(pending_queue_t &job_queue, osmid_t id, size_t output_id, size_t& added) { osmid_t const prev = ways_pending_tracker.last_returned(); if (id_tracker::is_valid(prev) && prev >= id) { @@ -252,9 +164,11 @@ int output_pgsql_t::pending_way(osmid_t id, int exists) { auto &way = buffer.get(0); if (!m_tagtransform->filter_tags(way, &polygon, &roads, *m_export_list.get(), outtags)) { - nodelist_t nodes; - m_mid->nodes_get_list(nodes, way.nodes(), reproj.get()); - return pgsql_out_way(id, outtags, nodes, polygon, roads); + auto nnodes = m_mid->nodes_get_list(&(way.nodes())); + if (nnodes > 1) { + pgsql_out_way(way, &outtags, polygon, roads); + return 1; + } } } @@ -305,8 +219,13 @@ int output_pgsql_t::pending_relation(osmid_t id, int exists) { // might be relocated when more data is added. rels_buffer.clear(); if (m_mid->relations_get(id, rels_buffer)) { + // If the flag says this object may exist already, delete it first. + if (exists) { + pgsql_delete_relation_from_output(id); + } + auto const &rel = rels_buffer.get(0); - return pgsql_process_relation(rel, exists, true); + return pgsql_process_relation(rel, true); } return 0; @@ -355,33 +274,34 @@ int output_pgsql_t::node_add(osmium::Node const &node) *m_export_list.get(), outtags)) return 1; - auto c = reproj->reproject(node.location()); - expire.from_bbox(c.x, c.y, c.x, c.y); - m_tables[t_point]->write_node(node.id(), outtags, c.y, c.x); + auto wkb = m_builder.get_wkb_node(node.location()); + expire.from_wkb(wkb.c_str(), node.id()); + m_tables[t_point]->write_row(node.id(), outtags, wkb); return 0; } -int output_pgsql_t::way_add(osmium::Way const &way) +int output_pgsql_t::way_add(osmium::Way *way) { int polygon = 0; int roads = 0; taglist_t outtags; /* Check whether the way is: (1) Exportable, (2) Maybe a polygon */ - auto filter = m_tagtransform->filter_tags(way, &polygon, &roads, + auto filter = m_tagtransform->filter_tags(*way, &polygon, &roads, *m_export_list.get(), outtags); /* If this isn't a polygon then it can not be part of a multipolygon Hence only polygons are "pending" */ - if (!filter && polygon) { ways_pending_tracker.mark(way.id()); } + if (!filter && polygon) { ways_pending_tracker.mark(way->id()); } if( !polygon && !filter ) { /* Get actual node data and generate output */ - nodelist_t nodes; - m_mid->nodes_get_list(nodes, way.nodes(), reproj.get()); - pgsql_out_way(way.id(), outtags, nodes, polygon, roads); + auto nnodes = m_mid->nodes_get_list(&(way->nodes())); + if (nnodes > 1) { + pgsql_out_way(*way, &outtags, polygon, roads); + } } return 0; } @@ -389,59 +309,118 @@ int output_pgsql_t::way_add(osmium::Way const &way) /* This is the workhorse of pgsql_add_relation, split out because it is used as the callback for iterate relations */ int output_pgsql_t::pgsql_process_relation(osmium::Relation const &rel, - bool exists, bool pending) + bool pending) { - /* If the flag says this object may exist already, delete it first */ - if(exists) - pgsql_delete_relation_from_output(rel.id()); + taglist_t prefiltered_tags; + if (m_tagtransform->filter_tags(rel, nullptr, nullptr, *m_export_list.get(), + prefiltered_tags)) { + return 1; + } - taglist_t outtags; - if (m_tagtransform->filter_tags(rel, nullptr, nullptr, *m_export_list.get(), outtags)) - return 1; - - idlist_t xid2; - for (auto const &m : rel.members()) - { - /* Need to handle more than just ways... */ - if (m.type() == osmium::item_type::way) { - xid2.push_back(m.ref()); + idlist_t xid2; + for (auto const &m : rel.members()) { + /* Need to handle more than just ways... */ + if (m.type() == osmium::item_type::way) { + xid2.push_back(m.ref()); + } } + buffer.clear(); + rolelist_t xrole; + auto num_ways = m_mid->rel_way_members_get(rel, &xrole, buffer); + + if (num_ways == 0) + return 0; + + multitaglist_t xtags(num_ways, taglist_t()); + + size_t i = 0; + for (auto const &w : buffer.select()) { + assert(i < num_ways); + + //filter the tags on this member because we got it from the middle + //and since the middle is no longer tied to the output it no longer + //shares any kind of tag transform and therefore all original tags + //will come back and need to be filtered by individual outputs before + //using these ways + m_tagtransform->filter_tags(w, nullptr, nullptr, *m_export_list.get(), + xtags[i]); + //TODO: if the filter says that this member is now not interesting we + //should decrement the count and remove his nodes and tags etc. for + //now we'll just keep him with no tags so he will get filtered later + ++i; } - buffer.clear(); - auto num_ways = m_mid->ways_get_list(xid2, buffer); - multitaglist_t xtags(num_ways, taglist_t()); - rolelist_t xrole(num_ways); - multinodelist_t xnodes(num_ways, nodelist_t()); - idlist_t xid; - - size_t i = 0; - for (auto const &w : buffer.select()) { - assert(i < num_ways); - xid.push_back(w.id()); - m_mid->nodes_get_list(xnodes[i], w.nodes(), reproj.get()); - - //filter the tags on this member because we got it from the middle - //and since the middle is no longer tied to the output it no longer - //shares any kind of tag transform and therefore all original tags - //will come back and need to be filtered by individual outputs before - //using these ways - m_tagtransform->filter_tags(w, nullptr, nullptr, *m_export_list.get(), xtags[i]); - //TODO: if the filter says that this member is now not interesting we - //should decrement the count and remove his nodes and tags etc. for - //now we'll just keep him with no tags so he will get filtered later - for (auto const &member : rel.members()) { - if (member.ref() == w.id() && member.type() == osmium::item_type::way) { - xrole[i] = member.role(); - break; - } + int roads = 0; + int make_polygon = 0; + int make_boundary = 0; + std::vector members_superseded(num_ways, 0); + taglist_t outtags; + + // If it's a route relation make_boundary and make_polygon will be false + // otherwise one or the other will be true. + if (m_tagtransform->filter_rel_member_tags( + prefiltered_tags, xtags, xrole, &(members_superseded[0]), + &make_boundary, &make_polygon, &roads, *m_export_list.get(), + outtags)) { + return 0; + } + + for (auto &w : buffer.select()) { + m_mid->nodes_get_list(&(w.nodes())); + } + + // linear features and boundaries + // Needs to be done before the polygon treatment below because + // for boundaries the way_area tag may be added. + if (!make_polygon) { + auto wkbs = m_builder.get_wkb_multiline(buffer, true); + for (auto const &wkb : wkbs) { + expire.from_wkb(wkb.c_str(), -rel.id()); + m_tables[t_line]->write_row(-rel.id(), outtags, wkb); + if (roads) + m_tables[t_roads]->write_row(-rel.id(), outtags, wkb); } - ++i; } - /* At some point we might want to consider storing the retrieved data in the members, rather than as separate arrays */ - pgsql_out_relation(rel.id(), outtags, xnodes, xtags, xid, xrole, pending); + // multipolygons and boundaries + if (make_boundary || make_polygon) { + auto wkbs = m_builder.get_wkb_multipolygon(rel, buffer); + + char tmp[32]; + for (auto const &wkb : wkbs) { + expire.from_wkb(wkb.c_str(), -rel.id()); + if (m_enable_way_area) { + auto const area = + m_options.reproject_area + ? ewkb::parser_t(wkb).get_area( + m_options.projection.get()) + : ewkb::parser_t(wkb) + .get_area(); + snprintf(tmp, sizeof(tmp), "%g", area); + outtags.push_override(tag_t("way_area", tmp)); + } + m_tables[t_poly]->write_row(-rel.id(), outtags, wkb); + } + + /* Tagtransform will have marked those member ways of the relation that + * have fully been dealt with as part of the multi-polygon entry. + * Set them in the database as done and delete their entry to not + * have duplicates */ + if (make_polygon) { + size_t j = 0; + for (auto &w : buffer.select()) { + if (members_superseded[j]) { + pgsql_delete_way_from_output(w.id()); + // When working with pending relations this is not needed. + if (!pending) { + ways_done_tracker->mark(w.id()); + } + } + ++j; + } + } + } return 0; } @@ -460,7 +439,7 @@ int output_pgsql_t::relation_add(osmium::Relation const &rel) return 0; } - return pgsql_process_relation(rel, 0); + return pgsql_process_relation(rel, false); } /* Delete is easy, just remove all traces of this object. We don't need to @@ -545,14 +524,14 @@ int output_pgsql_t::node_modify(osmium::Node const &node) return 0; } -int output_pgsql_t::way_modify(osmium::Way const &way) +int output_pgsql_t::way_modify(osmium::Way *way) { if( !m_options.slim ) { fprintf( stderr, "Cannot apply diffs unless in slim mode\n" ); util::exit_nicely(); } - way_delete(way.id()); + way_delete(way->id()); way_add(way); return 0; @@ -588,17 +567,13 @@ std::shared_ptr output_pgsql_t::clone(const middle_query_t* cloned_mid return std::shared_ptr(clone); } -output_pgsql_t::output_pgsql_t(const middle_query_t* mid, const options_t &o) - : output_t(mid, o), - expire(o.expire_tiles_zoom, o.expire_tiles_max_bbox, o.projection), - ways_done_tracker(new id_tracker()), - buffer(32768, osmium::memory::Buffer::auto_grow::yes), - rels_buffer(1024, osmium::memory::Buffer::auto_grow::yes) +output_pgsql_t::output_pgsql_t(const middle_query_t *mid, const options_t &o) +: output_t(mid, o), m_builder(o.projection, o.enable_multi), + expire(o.expire_tiles_zoom, o.expire_tiles_max_bbox, o.projection), + ways_done_tracker(new id_tracker()), + buffer(32768, osmium::memory::Buffer::auto_grow::yes), + rels_buffer(1024, osmium::memory::Buffer::auto_grow::yes) { - reproj = m_options.projection; - builder.set_exclude_broken_polygon(m_options.excludepoly); - if (m_options.reproject_area) builder.set_reprojection(reproj.get()); - m_export_list.reset(new export_list()); m_enable_way_area = read_style_file( m_options.style, m_export_list.get() ); @@ -650,31 +625,29 @@ output_pgsql_t::output_pgsql_t(const middle_query_t* mid, const options_t &o) //tremble in awe of this massive constructor! seriously we are trying to avoid passing an //options object because we want to make use of the table_t in output_mutli_t which could //have a different tablespace/hstores/etc per table - m_tables.push_back(std::shared_ptr( - new table_t( - m_options.database_options.conninfo(), name, type, columns, m_options.hstore_columns, - reproj->target_srs(), - m_options.append, m_options.slim, m_options.droptemp, m_options.hstore_mode, - m_options.enable_hstore_index, m_options.tblsmain_data, m_options.tblsmain_index - ) - )); + m_tables.push_back(std::shared_ptr(new table_t( + m_options.database_options.conninfo(), name, type, columns, + m_options.hstore_columns, m_options.projection->target_srs(), + m_options.append, m_options.slim, m_options.droptemp, + m_options.hstore_mode, m_options.enable_hstore_index, + m_options.tblsmain_data, m_options.tblsmain_index))); } } -output_pgsql_t::output_pgsql_t(const output_pgsql_t& other): - output_t(other.m_mid, other.m_options), m_tagtransform(new tagtransform(&m_options)), m_enable_way_area(other.m_enable_way_area), - m_export_list(new export_list(*other.m_export_list)), - expire(m_options.expire_tiles_zoom, m_options.expire_tiles_max_bbox, - m_options.projection), - reproj(other.reproj), - //NOTE: we need to know which ways were used by relations so each thread - //must have a copy of the original marked done ways, its read only so its ok - ways_done_tracker(other.ways_done_tracker), - buffer(1024, osmium::memory::Buffer::auto_grow::yes), - rels_buffer(1024, osmium::memory::Buffer::auto_grow::yes) +output_pgsql_t::output_pgsql_t(const output_pgsql_t &other) +: output_t(other.m_mid, other.m_options), + m_tagtransform(new tagtransform(&m_options)), + m_enable_way_area(other.m_enable_way_area), + m_export_list(new export_list(*other.m_export_list)), + m_builder(m_options.projection, other.m_options.enable_multi), + expire(m_options.expire_tiles_zoom, m_options.expire_tiles_max_bbox, + m_options.projection), + //NOTE: we need to know which ways were used by relations so each thread + //must have a copy of the original marked done ways, its read only so its ok + ways_done_tracker(other.ways_done_tracker), + buffer(1024, osmium::memory::Buffer::auto_grow::yes), + rels_buffer(1024, osmium::memory::Buffer::auto_grow::yes) { - builder.set_exclude_broken_polygon(m_options.excludepoly); - if (m_options.reproject_area) builder.set_reprojection(reproj.get()); for(std::vector >::const_iterator t = other.m_tables.begin(); t != other.m_tables.end(); ++t) { //copy constructor will just connect to the already there table m_tables.push_back(std::shared_ptr(new table_t(**t))); diff --git a/output-pgsql.hpp b/output-pgsql.hpp index d80e991ae..561fb908e 100644 --- a/output-pgsql.hpp +++ b/output-pgsql.hpp @@ -6,13 +6,12 @@ #ifndef OUTPUT_PGSQL_H #define OUTPUT_PGSQL_H -#include "output.hpp" -#include "tagtransform.hpp" -#include "geometry-builder.hpp" -#include "reprojection.hpp" #include "expire-tiles.hpp" #include "id-tracker.hpp" +#include "osmium-builder.hpp" +#include "output.hpp" #include "table.hpp" +#include "tagtransform.hpp" #include #include @@ -40,11 +39,11 @@ class output_pgsql_t : public output_t { int pending_relation(osmid_t id, int exists) override; int node_add(osmium::Node const &node) override; - int way_add(osmium::Way const &way) override; + int way_add(osmium::Way *way) override; int relation_add(osmium::Relation const &rel) override; int node_modify(osmium::Node const &node) override; - int way_modify(osmium::Way const &way) override; + int way_modify(osmium::Way *way) override; int relation_modify(osmium::Relation const &rel) override; int node_delete(osmid_t id) override; @@ -57,14 +56,9 @@ class output_pgsql_t : public output_t { void merge_expire_trees(output_t *other) override; protected: - int pgsql_out_way(osmid_t id, taglist_t &tags, const nodelist_t &nodes, - int polygons, int roads); - int pgsql_out_relation(osmid_t id, const taglist_t &rel_tags, - const multinodelist_t &xnodes, const multitaglist_t & xtags, - const idlist_t &xid, const rolelist_t &xrole, - bool pending); - int pgsql_process_relation(osmium::Relation const &rel, - bool exists, bool pending=false); + void pgsql_out_way(osmium::Way const &way, taglist_t *tags, bool polygon, + bool roads); + int pgsql_process_relation(osmium::Relation const &rel, bool pending); int pgsql_delete_way_from_output(osmid_t osm_id); int pgsql_delete_relation_from_output(osmid_t osm_id); @@ -77,11 +71,9 @@ class output_pgsql_t : public output_t { std::unique_ptr m_export_list; - geometry_builder builder; + geom::osmium_builder_t m_builder; expire_tiles expire; - std::shared_ptr reproj; - id_tracker ways_pending_tracker, rels_pending_tracker; std::shared_ptr ways_done_tracker; osmium::memory::Buffer buffer; diff --git a/output.hpp b/output.hpp index 681fe7e7b..19c02f7c3 100644 --- a/output.hpp +++ b/output.hpp @@ -50,11 +50,11 @@ class output_t : public boost::noncopyable { virtual int pending_relation(osmid_t id, int exists) = 0; virtual int node_add(osmium::Node const &node) = 0; - virtual int way_add(osmium::Way const &way) = 0; + virtual int way_add(osmium::Way *way) = 0; virtual int relation_add(osmium::Relation const &rel) = 0; virtual int node_modify(osmium::Node const &node) = 0; - virtual int way_modify(osmium::Way const &way) = 0; + virtual int way_modify(osmium::Way *way) = 0; virtual int relation_modify(osmium::Relation const &rel) = 0; virtual int node_delete(osmid_t id) = 0; diff --git a/parse-osmium.cpp b/parse-osmium.cpp index 46d1be579..b404f3185 100644 --- a/parse-osmium.cpp +++ b/parse-osmium.cpp @@ -129,7 +129,7 @@ void parse_osmium_t::stream_file(const std::string &filename, const std::string reader.close(); } -void parse_osmium_t::node(osmium::Node& node) +void parse_osmium_t::node(osmium::Node const &node) { if (node.deleted()) { m_data->node_delete(node.id()); @@ -163,15 +163,15 @@ void parse_osmium_t::way(osmium::Way& way) m_data->way_delete(way.id()); } else { if (m_append) { - m_data->way_modify(way); + m_data->way_modify(&way); } else { - m_data->way_add(way); + m_data->way_add(&way); } } m_stats.add_way(way.id()); } -void parse_osmium_t::relation(osmium::Relation& rel) +void parse_osmium_t::relation(osmium::Relation const &rel) { if (rel.deleted()) { m_data->relation_delete(rel.id()); diff --git a/parse-osmium.hpp b/parse-osmium.hpp index be1bc3af6..4b1e23aea 100644 --- a/parse-osmium.hpp +++ b/parse-osmium.hpp @@ -113,9 +113,9 @@ class parse_osmium_t: public osmium::handler::Handler void stream_file(const std::string &filename, const std::string &fmt); - void node(osmium::Node& node); + void node(osmium::Node const &node); void way(osmium::Way& way); - void relation(osmium::Relation& rel); + void relation(osmium::Relation const &rel); parse_stats_t const &stats() const { diff --git a/processor-line.cpp b/processor-line.cpp index bc1b23aa4..3c971039c 100644 --- a/processor-line.cpp +++ b/processor-line.cpp @@ -1,19 +1,23 @@ #include "processor-line.hpp" -processor_line::processor_line(int srid) : geometry_processor(srid, "LINESTRING", interest_way | interest_relation ) +processor_line::processor_line(std::shared_ptr const &proj) +: geometry_processor(proj->target_srs(), "LINESTRING", + interest_way | interest_relation), + m_builder(proj, false) { } -processor_line::~processor_line() +geometry_processor::wkb_t processor_line::process_way(osmium::Way const &way) { -} + auto wkbs = m_builder.get_wkb_line(way.nodes(), false); -geometry_builder::pg_geom_t processor_line::process_way(const nodelist_t &nodes) -{ - return builder.get_wkb_simple(nodes, false); + return wkbs.empty() ? wkb_t() : wkbs[0]; } -geometry_builder::pg_geoms_t processor_line::process_relation(const multinodelist_t &nodes) +geometry_processor::wkbs_t +processor_line::process_relation(osmium::Relation const &, + osmium::memory::Buffer const &ways) { - return builder.build_both(nodes, false, false, 1000000); + // XXX are multilines really acceptable? + return m_builder.get_wkb_multiline(ways, false); } diff --git a/processor-line.hpp b/processor-line.hpp index 1a011b658..883290241 100644 --- a/processor-line.hpp +++ b/processor-line.hpp @@ -3,15 +3,17 @@ #include "geometry-processor.hpp" -struct processor_line : public geometry_processor { - processor_line(int srid); - virtual ~processor_line(); +class processor_line : public geometry_processor +{ +public: + processor_line(std::shared_ptr const &proj); - geometry_builder::pg_geom_t process_way(const nodelist_t &nodes); - geometry_builder::pg_geoms_t process_relation(const multinodelist_t &nodes); + wkb_t process_way(osmium::Way const &way) override; + wkbs_t process_relation(osmium::Relation const &rel, + osmium::memory::Buffer const &ways) override; private: - geometry_builder builder; + geom::osmium_builder_t m_builder; }; #endif /* PROCESSOR_LINE_HPP */ diff --git a/processor-point.cpp b/processor-point.cpp index d40472536..faf66b327 100644 --- a/processor-point.cpp +++ b/processor-point.cpp @@ -5,14 +5,14 @@ #include "processor-point.hpp" #include "util.hpp" -processor_point::processor_point(int srid) - : geometry_processor(srid, "POINT", interest_node) { -} - -processor_point::~processor_point() { +processor_point::processor_point(std::shared_ptr const &proj) +: geometry_processor(proj->target_srs(), "POINT", interest_node), + m_builder(proj, false) +{ } -geometry_builder::pg_geom_t processor_point::process_node(double lat, double lon) +geometry_processor::wkb_t +processor_point::process_node(osmium::Location const &loc) { - return geometry_builder::pg_geom_t((boost::format("POINT(%.15g %.15g)") % lon % lat).str(), false); + return m_builder.get_wkb_node(loc); } diff --git a/processor-point.hpp b/processor-point.hpp index 6cdf58ea9..c265bbc64 100644 --- a/processor-point.hpp +++ b/processor-point.hpp @@ -3,11 +3,15 @@ #include "geometry-processor.hpp" -struct processor_point : public geometry_processor { - processor_point(int srid); - virtual ~processor_point(); +class processor_point : public geometry_processor +{ +public: + processor_point(std::shared_ptr const &proj); - geometry_builder::pg_geom_t process_node(double lat, double lon); + wkb_t process_node(osmium::Location const &loc) override; + +private: + geom::osmium_builder_t m_builder; }; #endif /* PROCESSOR_POINT_HPP */ diff --git a/processor-polygon.cpp b/processor-polygon.cpp index 73c339219..35cf5e195 100644 --- a/processor-polygon.cpp +++ b/processor-polygon.cpp @@ -1,19 +1,21 @@ #include "processor-polygon.hpp" -processor_polygon::processor_polygon(int srid, bool enable_multi) : geometry_processor(srid, "GEOMETRY", interest_way | interest_relation), enable_multi(enable_multi) +processor_polygon::processor_polygon(std::shared_ptr const &proj, + bool enable_multi) +: geometry_processor(proj->target_srs(), "GEOMETRY", + interest_way | interest_relation), + m_builder(proj, enable_multi) { } -processor_polygon::~processor_polygon() +geometry_processor::wkb_t processor_polygon::process_way(osmium::Way const &way) { + return m_builder.get_wkb_polygon(way); } -geometry_builder::pg_geom_t processor_polygon::process_way(const nodelist_t &nodes) +geometry_processor::wkbs_t +processor_polygon::process_relation(osmium::Relation const &rel, + osmium::memory::Buffer const &ways) { - return builder.get_wkb_simple(nodes, true); -} - -geometry_builder::pg_geoms_t processor_polygon::process_relation(const multinodelist_t &nodes) -{ - return builder.build_polygons(nodes, enable_multi, -1); + return m_builder.get_wkb_multipolygon(rel, ways); } diff --git a/processor-polygon.hpp b/processor-polygon.hpp index 6d562ebfe..d2273e412 100644 --- a/processor-polygon.hpp +++ b/processor-polygon.hpp @@ -3,16 +3,18 @@ #include "geometry-processor.hpp" -struct processor_polygon : public geometry_processor { - processor_polygon(int srid, bool enable_multi); - virtual ~processor_polygon(); +class processor_polygon : public geometry_processor +{ +public: + processor_polygon(std::shared_ptr const &proj, + bool enable_multi); - geometry_builder::pg_geom_t process_way(const nodelist_t &nodes); - geometry_builder::pg_geoms_t process_relation(const multinodelist_t &nodes); + wkb_t process_way(osmium::Way const &nodes) override; + wkbs_t process_relation(osmium::Relation const &rel, + osmium::memory::Buffer const &ways) override; private: - bool enable_multi; - geometry_builder builder; + geom::osmium_builder_t m_builder; }; #endif /* PROCESSOR_POLYGON_HPP */ diff --git a/table.cpp b/table.cpp index 6196e40d4..3ef4d7ea0 100644 --- a/table.cpp +++ b/table.cpp @@ -1,8 +1,3 @@ -#include "table.hpp" -#include "options.hpp" -#include "util.hpp" -#include "taginfo.hpp" - #include #include #include @@ -10,6 +5,12 @@ #include #include +#include "options.hpp" +#include "table.hpp" +#include "taginfo.hpp" +#include "util.hpp" +#include "wkb.hpp" + using std::string; typedef boost::format fmt; @@ -32,7 +33,6 @@ table_t::table_t(const string& conninfo, const string& name, const string& type, //we use these a lot, so instead of constantly allocating them we predefine these single_fmt = fmt("%1%"); - point_fmt = fmt("POINT(%.15g %.15g)"); del_fmt = fmt("DELETE FROM %1% WHERE osm_id = %2%"); } @@ -40,7 +40,7 @@ table_t::table_t(const table_t& other): conninfo(other.conninfo), name(other.name), type(other.type), sql_conn(nullptr), copyMode(false), buffer(), srid(other.srid), append(other.append), slim(other.slim), drop_temp(other.drop_temp), hstore_mode(other.hstore_mode), enable_hstore_index(other.enable_hstore_index), columns(other.columns), hstore_columns(other.hstore_columns), copystr(other.copystr), table_space(other.table_space), - table_space_index(other.table_space_index), single_fmt(other.single_fmt), point_fmt(other.point_fmt), del_fmt(other.del_fmt) + table_space_index(other.table_space_index), single_fmt(other.single_fmt), del_fmt(other.del_fmt) { // if the other table has already started, then we want to execute // the same stuff to get into the same state. but if it hasn't, then @@ -287,18 +287,13 @@ void table_t::stop_copy() copyMode = false; } -void table_t::write_node(const osmid_t id, const taglist_t &tags, double lat, double lon) -{ - write_row(id, tags, (point_fmt % lon % lat).str()); -} - void table_t::delete_row(const osmid_t id) { stop_copy(); pgsql_exec_simple(sql_conn, PGRES_COMMAND_OK, (del_fmt % name % id).str()); } -void table_t::write_row(const osmid_t id, const taglist_t &tags, const std::string &geom) +void table_t::write_row(osmid_t id, taglist_t const &tags, std::string const &geom) { //add the osm id buffer.append((single_fmt % id).str()); @@ -311,7 +306,7 @@ void table_t::write_row(const osmid_t id, const taglist_t &tags, const std::stri used.assign(tags.size(), false); //get the regular columns' values - write_columns(tags, buffer, hstore_mode == HSTORE_NORM?&used:nullptr); + write_columns(tags, buffer, hstore_mode == HSTORE_NORM ? &used : nullptr); //get the hstore columns' values write_hstore_columns(tags, buffer); @@ -320,25 +315,20 @@ void table_t::write_row(const osmid_t id, const taglist_t &tags, const std::stri if (hstore_mode != HSTORE_NONE) write_tags_column(tags, buffer, used); - //give the geometry an srid - buffer.append("SRID="); - buffer.append(srid); - buffer.push_back(';'); - //add the geometry - buffer.append(geom); + //add the geometry - encoding it to hex along the way + ewkb::writer_t::write_as_hex(buffer, geom); + //we need \n because we are copying from stdin buffer.push_back('\n'); //tell the db we are copying if for some reason we arent already - if (!copyMode) - { + if (!copyMode) { pgsql_exec_simple(sql_conn, PGRES_COPY_IN, copystr); copyMode = true; } //send all the data to postgres - if(buffer.length() > BUFFER_SEND_SIZE) - { + if (buffer.length() > BUFFER_SEND_SIZE) { pgsql_CopyData(name.c_str(), sql_conn, buffer); buffer.clear(); } diff --git a/table.hpp b/table.hpp index f4b2423f4..497cd1b69 100644 --- a/table.hpp +++ b/table.hpp @@ -31,8 +31,7 @@ class table_t void begin(); void commit(); - void write_row(const osmid_t id, const taglist_t &tags, const std::string &geom); - void write_node(const osmid_t id, const taglist_t &tags, double lat, double lon); + void write_row(osmid_t id, taglist_t const &tags, std::string const &geom); void delete_row(const osmid_t id); std::string const& get_name(); @@ -108,7 +107,7 @@ class table_t boost::optional table_space; boost::optional table_space_index; - boost::format single_fmt, point_fmt, del_fmt; + boost::format single_fmt, del_fmt; }; #endif diff --git a/taginfo.cpp b/taginfo.cpp index 1a2ed8d05..f30a87f52 100644 --- a/taginfo.cpp +++ b/taginfo.cpp @@ -70,6 +70,22 @@ const std::vector &export_list::get(osmium::item_type id) const { } } +bool export_list::has_column(osmium::item_type id, char const *name) const +{ + auto idx = item_type_to_nwr_index(id); + if (idx >= exportList.size()) { + return false; + } + + for (auto const &info : exportList[idx]) { + if (info.name == name) { + return true; + } + } + + return false; +} + columns_t export_list::normal_columns(osmium::item_type id) const { columns_t columns; diff --git a/taginfo_impl.hpp b/taginfo_impl.hpp index cf2e554af..c6313bef7 100644 --- a/taginfo_impl.hpp +++ b/taginfo_impl.hpp @@ -42,6 +42,7 @@ struct export_list { const std::vector &get(osmium::item_type id) const; columns_t normal_columns(osmium::item_type id) const; + bool has_column(osmium::item_type id, char const *name) const; std::vector > exportList; /* Indexed osmium nwr index */ }; diff --git a/tests/middle-tests.cpp b/tests/middle-tests.cpp index bb856435c..f495e4179 100644 --- a/tests/middle-tests.cpp +++ b/tests/middle-tests.cpp @@ -23,18 +23,18 @@ std::shared_ptr proj(reprojection::create_projection(PROJ_LATLONG)); #define ALLOWED_ERROR 10e-9 -bool node_okay(osmium::geom::Coordinates node, osmium::Node const &expected) +bool node_okay(osmium::Location loc, osmium::Node const &expected) { - if ((node.y > expected.location().lat() + ALLOWED_ERROR) || - (node.y < expected.location().lat() - ALLOWED_ERROR)) { + if ((loc.lat() > expected.location().lat() + ALLOWED_ERROR) || + (loc.lat() < expected.location().lat() - ALLOWED_ERROR)) { std::cerr << "ERROR: Node should have lat=" << expected.location().lat() - << ", but got back " << node.y << " from middle.\n"; + << ", but got back " << loc.lat() << " from middle.\n"; return false; } - if ((node.x > expected.location().lon() + ALLOWED_ERROR) || - (node.x < expected.location().lon() - ALLOWED_ERROR)) { + if ((loc.lon() > expected.location().lon() + ALLOWED_ERROR) || + (loc.lon() < expected.location().lon() - ALLOWED_ERROR)) { std::cerr << "ERROR: Node should have lon=" << expected.location().lon() - << ", but got back " << node.x << " from middle.\n"; + << ", but got back " << loc.lon() << " from middle.\n"; return false; } return true; @@ -57,25 +57,19 @@ int test_node_set(middle_t *mid) buffer.clear(); auto const &node = buffer.get(add_node(1234, 12.3456789, 98.7654321)); - auto const &way = buffer.get(way_with_nodes({node.id()})); - nodelist_t nodes; + auto &way = buffer.get(way_with_nodes({node.id()})); // set the node mid->nodes_set(node); // get it back - if (mid->nodes_get_list(nodes, way.nodes(), proj.get()) != - way.nodes().size()) { + if (mid->nodes_get_list(&(way.nodes())) != way.nodes().size()) { std::cerr << "ERROR: Unable to get node list.\n"; return 1; } - if (nodes.size() != way.nodes().size()) { - std::cerr << "ERROR: Mismatch in returned node list size.\n"; - return 1; - } // check that it's the same - if (!node_okay(nodes[0], node)) { + if (!node_okay(way.nodes()[0].location(), node)) { return 1; } @@ -137,23 +131,16 @@ int test_nodes_comprehensive_set(middle_t *mid) ids.push_back(node.id()); } - auto const &way = buffer.get(way_with_nodes(ids)); + auto &way = buffer.get(way_with_nodes(ids)); - nodelist_t nodes; - if (mid->nodes_get_list(nodes, way.nodes(), proj.get()) != ids.size()) { + if (mid->nodes_get_list(&(way.nodes())) != ids.size()) { std::cerr << "ERROR: Unable to get node list.\n"; return 1; } - if (nodes.size() != ids.size()) { - std::cerr << "ERROR: Mismatch in returned node list size.\n"; - return 1; - } - - for (size_t i = 0; i < nodes.size(); ++i) - { + for (size_t i = 0; i < ids.size(); ++i) { auto const &node = buffer.get(expected_nodes[i]); - if (!node_okay(nodes[i], node)) { + if (!node_okay(way.nodes()[i].location(), node)) { return 1; } } @@ -214,13 +201,33 @@ int test_way_set(middle_t *mid) mid->commit(); // get it back - idlist_t ways; - ways.push_back(way_id); + osmium::memory::Buffer relbuf(4096, osmium::memory::Buffer::auto_grow::yes); + { + using namespace osmium::builder::attr; + osmium::builder::add_relation( + relbuf, _id(123), _member(osmium::item_type::node, 132), + _member(osmium::item_type::way, way_id, "outer")); + } + + auto const &rel = relbuf.get(0); + auto buf_pos = buffer.committed(); - size_t way_count = mid->ways_get_list(ways, buffer); + rolelist_t roles; + size_t way_count = mid->rel_way_members_get(rel, &roles, buffer); if (way_count != 1) { std::cerr << "ERROR: Unable to get way list.\n"; return 1; } - auto const &way = buffer.get(buf_pos); + if (roles.size() != 1) { + std::cerr << "Bad length of role ist. Expected 1, got " << roles.size() + << ".\n"; + return 1; + } + + if (strcmp(roles[0], "outer") != 0) { + std::cerr << "Bad role. Expected 'outer', got '" << roles[0] << "'.\n"; + return 1; + } + + auto &way = buffer.get(buf_pos); // check that it's the same if (way.nodes().size() != nds.size()) { std::cerr << "ERROR: Way should have " << nds.size() << " nodes, but got back " @@ -232,17 +239,18 @@ int test_way_set(middle_t *mid) << way.id() << " from middle.\n"; return 1; } - nodelist_t xnodes; - mid->nodes_get_list(xnodes, way.nodes(), proj.get()); + mid->nodes_get_list(&(way.nodes())); for (size_t i = 0; i < nds.size(); ++i) { - if (xnodes[i].x != lon) { + if (way.nodes()[i].location().lon() != lon) { std::cerr << "ERROR: Way node should have lon=" << lon - << ", but got back " << xnodes[i].x << " from middle.\n"; + << ", but got back " << way.nodes()[i].location().lon() + << " from middle.\n"; return 1; } - if (xnodes[i].y != lat) { + if (way.nodes()[i].location().lat() != lat) { std::cerr << "ERROR: Way node should have lat=" << lat - << ", but got back " << xnodes[i].y << " from middle.\n"; + << ", but got back " << way.nodes()[i].location().lat() + << " from middle.\n"; return 1; } } diff --git a/tests/mockups.hpp b/tests/mockups.hpp index 1efed9c3f..f5ab58e1f 100644 --- a/tests/mockups.hpp +++ b/tests/mockups.hpp @@ -15,15 +15,15 @@ struct dummy_middle_t : public middle_t { void commit(void) override { } void nodes_set(osmium::Node const &) override {} - size_t nodes_get_list(nodelist_t &, osmium::WayNodeList const &, - reprojection const *) const override - { - return 0; - } + size_t nodes_get_list(osmium::WayNodeList *) const override { return 0; } void ways_set(osmium::Way const &) override { } bool ways_get(osmid_t, osmium::memory::Buffer &) const override { return true; } - size_t ways_get_list(idlist_t const &, osmium::memory::Buffer &) const override { return 0; } + size_t rel_way_members_get(osmium::Relation const &, rolelist_t *, + osmium::memory::Buffer &) const override + { + return 0; + } void relations_set(osmium::Relation const &) override { } bool relations_get(osmid_t, osmium::memory::Buffer &) const override { return 0; } @@ -52,15 +52,15 @@ struct dummy_slim_middle_t : public slim_middle_t { void commit(void) override { } void nodes_set(osmium::Node const &) override {} - size_t nodes_get_list(nodelist_t &, osmium::WayNodeList const &, - reprojection const *) const override - { - return 0; - } + size_t nodes_get_list(osmium::WayNodeList *) const override { return 0; } void ways_set(osmium::Way const &) override { } bool ways_get(osmid_t, osmium::memory::Buffer &) const override { return true; } - size_t ways_get_list(idlist_t const &, osmium::memory::Buffer &) const override { return 0; } + size_t rel_way_members_get(osmium::Relation const &, rolelist_t *, + osmium::memory::Buffer &) const override + { + return 0; + } void relations_set(osmium::Relation const &) override { } bool relations_get(osmid_t, osmium::memory::Buffer &) const override { return 0; } diff --git a/tests/regression-test.py b/tests/regression-test.py index b93a8e3d9..12fbe6407 100755 --- a/tests/regression-test.py +++ b/tests/regression-test.py @@ -19,27 +19,27 @@ #**************************************************************** sql_test_statements=[ ( 0, 'Basic point count', 'SELECT count(*) FROM planet_osm_point;', 1342 ), - ( 1, 'Basic line count', 'SELECT count(*) FROM planet_osm_line;', 3300 ), + ( 1, 'Basic line count', 'SELECT count(*) FROM planet_osm_line;', 3231 ), ( 2, 'Basic road count', 'SELECT count(*) FROM planet_osm_roads;', 375 ), - ( 3, 'Basic polygon count', 'SELECT count(*) FROM planet_osm_polygon;', 4128 ), - ( 4, 'Basic latlon line count', 'SELECT count(*) FROM planet_osm_line;', 3298 ), + ( 3, 'Basic polygon count', 'SELECT count(*) FROM planet_osm_polygon;', 4127 ), + ( 4, 'Basic latlon line count', 'SELECT count(*) FROM planet_osm_line;', 3229 ), ( 5, 'Basic latlon road count', 'SELECT count(*) FROM planet_osm_roads;', 374 ), ( 6, 'Basic post-diff point count', 'SELECT count(*) FROM planet_osm_point;', 1457 ), - ( 7, 'Basic post-diff line count', 'SELECT count(*) FROM planet_osm_line;', 3344 ), - ( 8, 'Basic post-diff road count', 'SELECT count(*) FROM planet_osm_roads;', 381 ), - ( 9, 'Basic post-diff polygon count', 'SELECT count(*) FROM planet_osm_polygon;', 4275 ), + ( 7, 'Basic post-diff line count', 'SELECT count(*) FROM planet_osm_line;', 3274 ), + ( 8, 'Basic post-diff road count', 'SELECT count(*) FROM planet_osm_roads;', 380 ), + ( 9, 'Basic post-diff polygon count', 'SELECT count(*) FROM planet_osm_polygon;', 4274 ), ( 10, 'Absence of nodes table', 'SELECT count(*) FROM pg_tables WHERE tablename = \'planet_osm_nodes\'', 0), ( 11, 'Absence of way table', 'SELECT count(*) FROM pg_tables WHERE tablename = \'planet_osm_ways\'', 0), ( 12, 'Absence of rel line', 'SELECT count(*) FROM pg_tables WHERE tablename = \'planet_osm_rels\'', 0), - ( 13, 'Basic polygon area', 'SELECT round(sum(cast(ST_Area(way) as numeric)),0) FROM planet_osm_polygon;', 1223800703), - ( 14, 'Gazetteer place count', 'SELECT count(*) FROM place', 2837), + ( 13, 'Basic polygon area', 'SELECT round(sum(cast(ST_Area(way) as numeric)),0) FROM planet_osm_polygon;', 1210958566), + ( 14, 'Gazetteer place count', 'SELECT count(*) FROM place', 2836), ( 15, 'Gazetteer place node count', 'SELECT count(*) FROM place WHERE osm_type = \'N\'', 759), ( 16, 'Gazetteer place way count', 'SELECT count(*) FROM place WHERE osm_type = \'W\'', 2059), - ( 17, 'Gazetteer place rel count', 'SELECT count(*) FROM place WHERE osm_type = \'R\'', 19), - ( 18, 'Gazetteer post-diff place count', 'SELECT count(*) FROM place', 2878), + ( 17, 'Gazetteer place rel count', 'SELECT count(*) FROM place WHERE osm_type = \'R\'', 18), + ( 18, 'Gazetteer post-diff place count', 'SELECT count(*) FROM place', 2877), ( 19, 'Gazetteer post-diff place node count', 'SELECT count(*) FROM place WHERE osm_type = \'N\'', 764), ( 20, 'Gazetteer post-diff place way count', 'SELECT count(*) FROM place WHERE osm_type = \'W\'', 2095), - ( 21, 'Gazetteer post-diff place rel count', 'SELECT count(*) FROM place WHERE osm_type = \'R\'', 19), + ( 21, 'Gazetteer post-diff place rel count', 'SELECT count(*) FROM place WHERE osm_type = \'R\'', 18), ( 22, 'Gazetteer housenumber count', 'SELECT count(*) FROM place WHERE housenumber is not null', 199), ( 23, 'Gazetteer post-diff housenumber count count', 'SELECT count(*) FROM place WHERE housenumber is not null', 199), ( 24, 'Gazetteer isin count', 'SELECT count(*) FROM place WHERE isin is not null', 239), @@ -95,19 +95,19 @@ ( 50, 'Multipolygon nested outer ways. Both outer and inner ways are from multiple ways (multigeometry)', 'SELECT ST_NumGeometries(way) FROM planet_osm_polygon WHERE osm_id = -7 and landuse = \'farmland\' and name = \'Name_rel15\'', 2), ( 51, 'Basic hstore point count', 'SELECT count(*) FROM planet_osm_point;', 1360 ), - ( 52, 'Basic hstore line count', 'SELECT count(*) FROM planet_osm_line;', 3323 ), + ( 52, 'Basic hstore line count', 'SELECT count(*) FROM planet_osm_line;', 3254 ), ( 53, 'Basic hstore road count', 'SELECT count(*) FROM planet_osm_roads;', 375 ), - ( 54, 'Basic hstore polygon count', 'SELECT count(*) FROM planet_osm_polygon;', 4128 ), + ( 54, 'Basic hstore polygon count', 'SELECT count(*) FROM planet_osm_polygon;', 4127 ), ( 55, 'Basic post-diff point count', 'SELECT count(*) FROM planet_osm_point;', 1475 ), - ( 56, 'Basic post-diff line count', 'SELECT count(*) FROM planet_osm_line;', 3367 ), - ( 57, 'Basic post-diff road count', 'SELECT count(*) FROM planet_osm_roads;', 381 ), - ( 58, 'Basic post-diff polygon count', 'SELECT count(*) FROM planet_osm_polygon;', 4275 ), + ( 56, 'Basic post-diff line count', 'SELECT count(*) FROM planet_osm_line;', 3297 ), + ( 57, 'Basic post-diff road count', 'SELECT count(*) FROM planet_osm_roads;', 380 ), + ( 58, 'Basic post-diff polygon count', 'SELECT count(*) FROM planet_osm_polygon;', 4274 ), ( 59, 'Extra hstore full tags point count', 'SELECT count(*) FROM planet_osm_point WHERE tags ? \'osm_user\' and tags ? \'osm_version\' and tags ? \'osm_uid\' and tags ? \'osm_changeset\'', 1360), ( 60, 'Extra hstore full tags line count', - 'SELECT count(*) FROM planet_osm_line WHERE tags ? \'osm_user\' and tags ? \'osm_version\' and tags ? \'osm_uid\' and tags ? \'osm_changeset\'', 3323), + 'SELECT count(*) FROM planet_osm_line WHERE tags ? \'osm_user\' and tags ? \'osm_version\' and tags ? \'osm_uid\' and tags ? \'osm_changeset\'', 3254), ( 61, 'Extra hstore full tags polygon count', - 'SELECT count(*) FROM planet_osm_polygon WHERE tags ? \'osm_user\' and tags ? \'osm_version\' and tags ? \'osm_uid\' and tags ? \'osm_changeset\'', 4128), + 'SELECT count(*) FROM planet_osm_polygon WHERE tags ? \'osm_user\' and tags ? \'osm_version\' and tags ? \'osm_uid\' and tags ? \'osm_changeset\'', 4127), ( 62, 'Multipolygon copying of tags from outer with extra tags on relation', 'SELECT round(ST_Area(way)) FROM planet_osm_polygon WHERE osm_id = -22', 20878), ( 63, 'Multipolygon copying of tags from outer with extra tags on relation (abscence of way)', @@ -166,16 +166,16 @@ 'SELECT round(ST_Area(way)) FROM planet_osm_polygon WHERE osm_id = -35 and "natural" = \'water\'', 15737), ( 90, 'Multipolygon tags on relation two outer diff remove way from relation (presence of single way)', 'SELECT round(ST_Area(way)) FROM planet_osm_polygon WHERE osm_id = 102 and "natural" = \'water\'', 12994), - ( 91, 'Basic line length', 'SELECT round(sum(ST_Length(way))) FROM planet_osm_line;', 4269394), + ( 91, 'Basic line length', 'SELECT round(sum(ST_Length(way))) FROM planet_osm_line;', 4211350), ( 92, 'Basic line length', 'SELECT round(sum(ST_Length(way))) FROM planet_osm_roads;', 2032023), ( 93, 'Basic number of hstore points tags', 'SELECT sum(array_length(akeys(tags),1)) FROM planet_osm_point;', 4228), ( 94, 'Basic number of hstore roads tags', 'SELECT sum(array_length(akeys(tags),1)) FROM planet_osm_roads;', 2317), - ( 95, 'Basic number of hstore lines tags', 'SELECT sum(array_length(akeys(tags),1)) FROM planet_osm_line;', 11134), - ( 96, 'Basic number of hstore polygons tags', 'SELECT sum(array_length(akeys(tags),1)) FROM planet_osm_polygon;', 9541), + ( 95, 'Basic number of hstore lines tags', 'SELECT sum(array_length(akeys(tags),1)) FROM planet_osm_line;', 10387), + ( 96, 'Basic number of hstore polygons tags', 'SELECT sum(array_length(akeys(tags),1)) FROM planet_osm_polygon;', 9531), ( 97, 'Diff import number of hstore points tags', 'SELECT sum(array_length(akeys(tags),1)) FROM planet_osm_point;', 4352), - ( 98, 'Diff import number of hstore roads tags', 'SELECT sum(array_length(akeys(tags),1)) FROM planet_osm_roads;', 2341), - ( 99, 'Diff import number of hstore lines tags', 'SELECT sum(array_length(akeys(tags),1)) FROM planet_osm_line;', 11257), - ( 100, 'Diff import number of hstore polygons tags', 'SELECT sum(array_length(akeys(tags),1)) FROM planet_osm_polygon;', 9835), + ( 98, 'Diff import number of hstore roads tags', 'SELECT sum(array_length(akeys(tags),1)) FROM planet_osm_roads;', 2336), + ( 99, 'Diff import number of hstore lines tags', 'SELECT sum(array_length(akeys(tags),1)) FROM planet_osm_line;', 10505), + ( 100, 'Diff import number of hstore polygons tags', 'SELECT sum(array_length(akeys(tags),1)) FROM planet_osm_polygon;', 9825), #**** Tests to check if inner polygon appears when outer tags change after initially identicall inner and outer way tags in a multi-polygon **** #**** These tests are currently broken and noted in trac ticket #2853 **** ( 101, 'Multipolygon identical tags on inner and outer (presence of relation)', diff --git a/tests/test-options-parse.cpp b/tests/test-options-parse.cpp index 3b280c560..b90ecff6f 100644 --- a/tests/test-options-parse.cpp +++ b/tests/test-options-parse.cpp @@ -265,7 +265,6 @@ void test_random_perms() add_arg_or_not("--extra-attributes", args, options.extra_attributes); add_arg_or_not("--multi-geometry", args, options.enable_multi); add_arg_or_not("--keep-coastlines", args, options.keep_coastlines); - add_arg_or_not("--exclude-invalid-polygon", args, options.excludepoly); //add the input file args.push_back("tests/liechtenstein-2013-08-03.osm.pbf"); diff --git a/tests/test-output-multi-line.cpp b/tests/test-output-multi-line.cpp index 57f0767f5..9208655b7 100644 --- a/tests/test-output-multi-line.cpp +++ b/tests/test-output-multi-line.cpp @@ -56,7 +56,7 @@ int main(int argc, char *argv[]) { // start a new connection to run tests on db->check_count(1, "select count(*) from pg_catalog.pg_class where relname = 'foobar_highways'"); - db->check_count(2752, "select count(*) from foobar_highways"); + db->check_count(2753, "select count(*) from foobar_highways"); //check that we have the right spread db->check_count(13, "select count(*) from foobar_highways where highway='bridleway'"); @@ -65,7 +65,7 @@ int main(int argc, char *argv[]) { db->check_count(249, "select count(*) from foobar_highways where highway='footway'"); db->check_count(18, "select count(*) from foobar_highways where highway='living_street'"); db->check_count(171, "select count(*) from foobar_highways where highway='path'"); - db->check_count(5, "select count(*) from foobar_highways where highway='pedestrian'"); + db->check_count(6, "select count(*) from foobar_highways where highway='pedestrian'"); db->check_count(81, "select count(*) from foobar_highways where highway='primary'"); db->check_count(842, "select count(*) from foobar_highways where highway='residential'"); db->check_count(3, "select count(*) from foobar_highways where highway='road'"); diff --git a/tests/test-output-pgsql-tablespace.cpp b/tests/test-output-pgsql-tablespace.cpp index 090209e6c..1d897fd28 100644 --- a/tests/test-output-pgsql-tablespace.cpp +++ b/tests/test-output-pgsql-tablespace.cpp @@ -89,9 +89,9 @@ void test_regression_simple() { db->assert_has_table("osm2pgsql_test_roads"); db->check_count(1342, "SELECT count(*) FROM osm2pgsql_test_point"); - db->check_count(3300, "SELECT count(*) FROM osm2pgsql_test_line"); + db->check_count(3231, "SELECT count(*) FROM osm2pgsql_test_line"); db->check_count( 375, "SELECT count(*) FROM osm2pgsql_test_roads"); - db->check_count(4128, "SELECT count(*) FROM osm2pgsql_test_polygon"); + db->check_count(4127, "SELECT count(*) FROM osm2pgsql_test_polygon"); } } // anonymous namespace diff --git a/tests/test-output-pgsql-validgeom.cpp b/tests/test-output-pgsql-validgeom.cpp index 44da9fcd7..14056fb86 100644 --- a/tests/test-output-pgsql-validgeom.cpp +++ b/tests/test-output-pgsql-validgeom.cpp @@ -83,7 +83,7 @@ void test_z_order() { db->assert_has_table("osm2pgsql_test_polygon"); db->assert_has_table("osm2pgsql_test_roads"); - db->check_count(6, "SELECT COUNT(*) FROM osm2pgsql_test_polygon"); + db->check_count(10, "SELECT COUNT(*) FROM osm2pgsql_test_polygon"); db->check_count(0, "SELECT COUNT(*) FROM osm2pgsql_test_polygon WHERE NOT ST_IsValid(way)"); db->check_count(0, "SELECT COUNT(*) FROM osm2pgsql_test_polygon WHERE ST_IsEmpty(way)"); } diff --git a/tests/test-output-pgsql.cpp b/tests/test-output-pgsql.cpp index c5aad142d..98092f714 100644 --- a/tests/test-output-pgsql.cpp +++ b/tests/test-output-pgsql.cpp @@ -88,9 +88,9 @@ void test_regression_simple() { db->assert_has_table("osm2pgsql_test_roads"); db->check_count(1342, "SELECT count(*) FROM osm2pgsql_test_point"); - db->check_count(3300, "SELECT count(*) FROM osm2pgsql_test_line"); + db->check_count(3231, "SELECT count(*) FROM osm2pgsql_test_line"); db->check_count( 375, "SELECT count(*) FROM osm2pgsql_test_roads"); - db->check_count(4128, "SELECT count(*) FROM osm2pgsql_test_polygon"); + db->check_count(4127, "SELECT count(*) FROM osm2pgsql_test_polygon"); // Check size of lines db->check_number(1696.04, "SELECT ST_Length(way) FROM osm2pgsql_test_line WHERE osm_id = 44822682"); @@ -140,9 +140,9 @@ void test_latlong() { db->assert_has_table("osm2pgsql_test_roads"); db->check_count(1342, "SELECT count(*) FROM osm2pgsql_test_point"); - db->check_count(3298, "SELECT count(*) FROM osm2pgsql_test_line"); + db->check_count(3229, "SELECT count(*) FROM osm2pgsql_test_line"); db->check_count(374, "SELECT count(*) FROM osm2pgsql_test_roads"); - db->check_count(4128, "SELECT count(*) FROM osm2pgsql_test_polygon"); + db->check_count(4127, "SELECT count(*) FROM osm2pgsql_test_polygon"); // Check size of lines db->check_number(0.0105343, "SELECT ST_Length(way) FROM osm2pgsql_test_line WHERE osm_id = 44822682"); @@ -277,9 +277,9 @@ void test_clone() { db->assert_has_table("osm2pgsql_test_roads"); db->check_count(1342, "SELECT count(*) FROM osm2pgsql_test_point"); - db->check_count(3300, "SELECT count(*) FROM osm2pgsql_test_line"); + db->check_count(3231, "SELECT count(*) FROM osm2pgsql_test_line"); db->check_count( 375, "SELECT count(*) FROM osm2pgsql_test_roads"); - db->check_count(4128, "SELECT count(*) FROM osm2pgsql_test_polygon"); + db->check_count(4127, "SELECT count(*) FROM osm2pgsql_test_polygon"); } } // anonymous namespace diff --git a/tests/test-parse-diff.cpp b/tests/test-parse-diff.cpp index 9d751c9b6..c3d98a43e 100644 --- a/tests/test-parse-diff.cpp +++ b/tests/test-parse-diff.cpp @@ -48,8 +48,8 @@ struct test_output_t : public dummy_output_t { return 0; } - int way_add(osmium::Way const &w) override { - assert(w.id() > 0); + int way_add(osmium::Way *w) override { + assert(w->id() > 0); ++way.added; return 0; } @@ -65,7 +65,7 @@ struct test_output_t : public dummy_output_t { ++node.modified; return 0; } - int way_modify(osmium::Way const &) override { + int way_modify(osmium::Way *) override { ++way.modified; return 0; } diff --git a/tests/test-parse-xml2.cpp b/tests/test-parse-xml2.cpp index 70862375b..e9db2ac96 100644 --- a/tests/test-parse-xml2.cpp +++ b/tests/test-parse-xml2.cpp @@ -50,12 +50,12 @@ struct test_output_t : public output_null_t { return 0; } - int way_add(osmium::Way const &way) override { - assert(way.id() > 0); - sum_ids += (unsigned) way.id(); + int way_add(osmium::Way *way) override { + assert(way->id() > 0); + sum_ids += (unsigned) way->id(); num_ways += 1; - assert(way.nodes().size() >= 0); - num_nds += uint64_t(way.nodes().size()); + assert(way->nodes().size() >= 0); + num_nds += uint64_t(way->nodes().size()); return 0; } diff --git a/wkb.hpp b/wkb.hpp new file mode 100644 index 000000000..64896c9af --- /dev/null +++ b/wkb.hpp @@ -0,0 +1,371 @@ +#ifndef OSM2PGSQL_WKB_HPP +#define OSM2PGSQL_WKB_HPP + +#include +#include +#include +#include + +#include +#include + +namespace ewkb { + +enum geometry_type : uint32_t +{ + wkb_point = 1, + wkb_line = 2, + wkb_polygon = 3, + wkb_multi_point = 4, + wkb_multi_line = 5, + wkb_multi_polygon = 6, + wkb_collection = 7, + + wkb_srid = 0x20000000 // SRID-presence flag (EWKB) +}; + +enum wkb_byte_order_type_t : uint8_t +{ +#if __BYTE_ORDER == __LITTLE_ENDIAN + Endian = 1 // Little Endian +#else + Endian = 0, // Big Endian +#endif +}; + +/** + * Writer for EWKB data suitable for postgres. + * + * Code has been largely derived from osmium::geom::WKBFactoryImpl. + */ +class writer_t +{ + + std::string m_data; + int m_srid; + + size_t m_geometry_size_offset = 0; + size_t m_multigeometry_size_offset = 0; + size_t m_ring_size_offset = 0; + + size_t header(std::string &str, geometry_type type, bool add_length) const + { + str_push(str, Endian); + str_push(str, type | wkb_srid); + str_push(str, m_srid); + + const size_t offset = str.size(); + if (add_length) { + str_push(str, static_cast(0)); + } + return offset; + } + + void set_size(const size_t offset, const size_t size) + { + uint32_t s = static_cast(size); + std::copy_n(reinterpret_cast(&s), sizeof(uint32_t), + &m_data[offset]); + } + + template + inline static void str_push(std::string &str, T data) + { + str.append(reinterpret_cast(&data), sizeof(T)); + } + +public: + inline static void write_as_hex(std::string &out, std::string const &wkb) + { + static char const *lookup_hex = "0123456789ABCDEF"; + + for (char c : wkb) { + out += lookup_hex[(c >> 4) & 0xf]; + out += lookup_hex[c & 0xf]; + } + } + + explicit writer_t(int srid) : m_srid(srid) {} + + void add_sub_geometry(std::string const &part) { m_data.append(part); } + + void add_location(const osmium::geom::Coordinates &xy) + { + str_push(m_data, xy.x); + str_push(m_data, xy.y); + } + + /* Point */ + + std::string make_point(const osmium::geom::Coordinates &xy) const + { + std::string data; + header(data, wkb_point, false); + str_push(data, xy.x); + str_push(data, xy.y); + + return data; + } + + /* LineString */ + + void linestring_start() + { + m_geometry_size_offset = header(m_data, wkb_line, true); + } + + std::string linestring_finish(size_t num_points) + { + set_size(m_geometry_size_offset, num_points); + std::string data; + + using std::swap; + swap(data, m_data); + + return data; + } + + /* MultiLineString */ + + void multilinestring_start() + { + m_multigeometry_size_offset = header(m_data, wkb_multi_line, true); + } + + void multilinestring_line_finish(size_t num_points) + { + set_size(m_geometry_size_offset, num_points); + } + + std::string multilinestring_finish(size_t num_lines) + { + set_size(m_multigeometry_size_offset, num_lines); + std::string data; + + using std::swap; + swap(data, m_data); + + return data; + } + + /* Polygon */ + + void polygon_start() + { + m_geometry_size_offset = header(m_data, wkb_polygon, true); + } + + void polygon_ring_start() + { + m_ring_size_offset = m_data.size(); + str_push(m_data, static_cast(0)); + } + + void polygon_ring_finish(size_t num_points) + { + set_size(m_ring_size_offset, num_points); + } + + std::string polygon_finish(size_t num_rings) + { + set_size(m_geometry_size_offset, num_rings); + std::string data; + + using std::swap; + swap(data, m_data); + + return data; + } + + /* MultiPolygon */ + + void multipolygon_start() + { + m_multigeometry_size_offset = header(m_data, wkb_multi_polygon, true); + } + + void multipolygon_polygon_finish(size_t num_rings) + { + set_size(m_geometry_size_offset, num_rings); + } + + std::string multipolygon_finish(size_t num_polygons) + { + set_size(m_multigeometry_size_offset, num_polygons); + std::string data; + + using std::swap; + swap(data, m_data); + + return data; + } +}; + +/** + * Class that allows to iterate over the elements of a ewkb geometry. + * + * Note: this class assumes that the wkb was created by ewkb::writer_t. + * It implements the exact opposite decoding. + */ +class parser_t +{ +public: + inline static std::string wkb_from_hex(std::string const &wkb) + { + std::string out; + + bool front = true; + char outc; + for (char c : wkb) { + c -= 48; + if (c > 9) { + c -= 7; + } + if (front) { + outc = char(c << 4); + front = false; + } else { + out += outc | c; + front = true; + } + } + + if (out[0] != Endian) + throw std::runtime_error( +#if __BYTE_ORDER == __LITTLE_ENDIAN + "Geometries in the database are returned in big-endian byte order. " +#else + "Geometries in the database are returned in little-endian byte order. " +#endif + "osm2pgsql can only process geometries in native byte order." + ); + + return out; + } + + explicit parser_t(char const *wkb) : m_wkb(wkb), m_pos(0) {} + explicit parser_t(std::string const &wkb) : m_wkb(wkb.c_str()), m_pos(0) {} + + size_t save_pos() const { return m_pos; } + void rewind(size_t pos) { m_pos = pos; } + + int read_header() + { + m_pos += sizeof(uint8_t); // skip endianess + + auto type = read_data(); + + m_pos += sizeof(int); // skip srid + + return type & 0xff; + } + + uint32_t read_length() { return read_data(); } + + osmium::geom::Coordinates read_point() + { + auto x = read_data(); + auto y = read_data(); + + return osmium::geom::Coordinates(x, y); + } + + void skip_points(size_t num) { m_pos += sizeof(double) * 2 * num; } + + template + double get_area(PROJ *proj = nullptr) + { + double total = 0; + + auto type = read_header(); + + if (type == wkb_polygon) { + total = get_polygon_area(proj); + } else if (type == wkb_multi_polygon) { + auto num_poly = read_length(); + for (unsigned i = 0; i < num_poly; ++i) { + auto ptype = read_header(); + (void)ptype; + assert(ptype == wkb_polygon); + + total += get_polygon_area(proj); + } + } + + return total; + } + +private: + template + double get_polygon_area(PROJ *proj) + { + auto num_rings = read_length(); + assert(num_rings > 0); + + double total = get_ring_area(proj); + + for (unsigned i = 1; i < num_rings; ++i) { + total -= get_ring_area(proj); + } + + return total; + } + + template + double get_ring_area(PROJ *proj) + { + // Algorithm borrowed from + // http://stackoverflow.com/questions/451426/how-do-i-calculate-the-area-of-a-2d-polygon + // XXX numerically not stable (useless for latlon) + auto num_pts = read_length(); + assert(num_pts > 3); + + double total = 0; + + auto prev = read_point(); + proj->target_to_tile(&prev.y, &prev.x); + for (unsigned i = 1; i < num_pts; ++i) { + auto cur = read_point(); + proj->target_to_tile(&cur.y, &cur.x); + total += prev.x * cur.y - cur.x * prev.y; + prev = cur; + } + + return std::abs(total) * 0.5; + } + + double get_ring_area(osmium::geom::IdentityProjection *) + { + // Algorithm borrowed from + // http://stackoverflow.com/questions/451426/how-do-i-calculate-the-area-of-a-2d-polygon + auto num_pts = read_length(); + assert(num_pts > 3); + + double total = 0; + + auto prev = read_point(); + for (unsigned i = 1; i < num_pts; ++i) { + auto cur = read_point(); + total += prev.x * cur.y - cur.x * prev.y; + prev = cur; + } + + return std::abs(total) * 0.5; + } + + template + T read_data() + { + auto *data = reinterpret_cast(m_wkb + m_pos); + m_pos += sizeof(T); + + return *data; + } + + char const *m_wkb; + size_t m_pos; +}; + +} // namespace + +#endif // OSM2PGSQL_WKB_HPP