From 552e4bc0b874989878ea529193d106eeaed2e7e5 Mon Sep 17 00:00:00 2001 From: Jochen Topf Date: Fri, 20 Nov 2020 16:05:35 +0100 Subject: [PATCH] Add support for projection using PROJ library >6.1 This commit is adding projection support using the PROJ library from 6.1 onwards. The CMake config will prefer using the old API, simply because I don't know how to check for the new API without it trying to use the new API with version 5-6.0 which don't have everything we need. The code will only work from 6.1 onwards. There is a new CMake cache variable USE_PROJ_LIB which can be set to: * "4": Use version 4 API. Fail CMake if it is not available. * "6": Use version 6 API. Fail CMake if it is not available. * "off": Disable PROJ support (resulting binary will only have WGS84 (4326) and Web Mercator (3857) support) * "auto": Use version 4 API if available, otherwise try version 6 API or fall back to no PROJ support. Note that we have an issue with multithreading using the PROJ library, because we use it potentially from multiple threads. The information I can find about this seems to indicate that this is only a problem for error reporting, because of the error handling using a global errno-type variable. So we can't detect failed transformations reliably. But we don't care about those anyway in the code we have, so I believe this is probably not a huge problem. This affects the old and the new code, so this isn't something new. The API version 6 allows to do the multithreading correctly using a "context" which this commit does use. But the context is still shared between threads, because we are starting the threads after the reprojections have been initialized. So all of this is technically not correct, but we'll probably get away with it for the time being. We need larger refactorings in the geometry code anyway which we want to tackle soon and hopefully can resolve this correctly then. See #922 --- .github/actions/build-and-test/action.yml | 3 + .github/workflows/ci.yml | 55 ++++++++-- CMakeLists.txt | 47 ++++++-- README.md | 17 +++ src/CMakeLists.txt | 2 + src/options.cpp | 1 + src/reprojection-generic-none.cpp | 3 + src/reprojection-generic-proj4.cpp | 4 + src/reprojection-generic-proj6.cpp | 127 ++++++++++++++++++++++ src/reprojection.hpp | 7 +- tests/test-reprojection.cpp | 8 +- 11 files changed, 247 insertions(+), 27 deletions(-) create mode 100644 src/reprojection-generic-proj6.cpp diff --git a/.github/actions/build-and-test/action.yml b/.github/actions/build-and-test/action.yml index 214088061..e7e901732 100644 --- a/.github/actions/build-and-test/action.yml +++ b/.github/actions/build-and-test/action.yml @@ -22,6 +22,9 @@ runs: else CMAKE_OPTIONS="$CMAKE_OPTIONS -DWITH_LUAJIT=${LUAJIT_OPTION}" fi + if [ -n "$USE_PROJ_LIB" ]; then + CMAKE_OPTIONS="$CMAKE_OPTIONS -DUSE_PROJ_LIB=$USE_PROJ_LIB" + fi if [ -n "$CPP_VERSION" ]; then CMAKE_OPTIONS="$CMAKE_OPTIONS -DCMAKE_CXX_STANDARD=$CPP_VERSION" fi diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0f44ed212..90a332eeb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -36,7 +36,7 @@ jobs: PGHOST: /tmp - ubuntu-pg93-gcc5: + ubuntu16-pg93-gcc5: runs-on: ubuntu-16.04 env: @@ -53,7 +53,7 @@ jobs: - uses: ./.github/actions/ubuntu-prerequisites - uses: ./.github/actions/build-and-test - ubuntu-pg94-clang6: + ubuntu16-pg94-clang6: runs-on: ubuntu-16.04 env: @@ -70,7 +70,7 @@ jobs: - uses: ./.github/actions/ubuntu-prerequisites - uses: ./.github/actions/build-and-test - ubuntu-pg95-gcc7-jit: + ubuntu18-pg95-gcc7-jit: runs-on: ubuntu-18.04 env: @@ -87,7 +87,7 @@ jobs: - uses: ./.github/actions/ubuntu-prerequisites - uses: ./.github/actions/build-and-test - ubuntu-pg96-clang8-jit: + ubuntu18-pg96-clang8-jit: runs-on: ubuntu-18.04 env: @@ -104,7 +104,7 @@ jobs: - uses: ./.github/actions/ubuntu-prerequisites - uses: ./.github/actions/build-and-test - ubuntu-pg10-gcc9: + ubuntu18-pg10-gcc9: runs-on: ubuntu-18.04 env: @@ -122,7 +122,7 @@ jobs: - uses: ./.github/actions/build-and-test - ubuntu-pg11-clang9: + ubuntu18-pg11-clang9: runs-on: ubuntu-18.04 env: @@ -139,7 +139,7 @@ jobs: - uses: ./.github/actions/ubuntu-prerequisites - uses: ./.github/actions/build-and-test - ubuntu-pg12-gcc10-jit: + ubuntu20-pg12-gcc10-jit: runs-on: ubuntu-20.04 env: @@ -157,9 +157,8 @@ jobs: - uses: ./.github/actions/build-and-test - ubuntu-pg13-clang10-jit: + ubuntu20-pg13-clang10-jit: runs-on: ubuntu-20.04 - continue-on-error: true env: CC: clang-10 @@ -175,7 +174,43 @@ jobs: - uses: ./.github/actions/ubuntu-prerequisites - uses: ./.github/actions/build-and-test - ubuntu-pg12-gcc10-release: + ubuntu20-pg13-clang10-proj6: + runs-on: ubuntu-20.04 + + env: + CC: clang-10 + CXX: clang++-10 + LUA_VERSION: 5.3 + LUAJIT_OPTION: OFF + POSTGRESQL_VERSION: 13 + POSTGIS_VERSION: 3 + CPPVERSION: 14 + USE_PROJ_LIB: 6 + + steps: + - uses: actions/checkout@v2 + - uses: ./.github/actions/ubuntu-prerequisites + - uses: ./.github/actions/build-and-test + + ubuntu20-pg13-clang10-noproj: + runs-on: ubuntu-20.04 + + env: + CC: clang-10 + CXX: clang++-10 + LUA_VERSION: 5.3 + LUAJIT_OPTION: OFF + POSTGRESQL_VERSION: 13 + POSTGIS_VERSION: 3 + CPPVERSION: 14 + USE_PROJ_LIB: off + + steps: + - uses: actions/checkout@v2 + - uses: ./.github/actions/ubuntu-prerequisites + - uses: ./.github/actions/build-and-test + + ubuntu20-pg12-gcc10-release: runs-on: ubuntu-20.04 env: diff --git a/CMakeLists.txt b/CMakeLists.txt index 61c946b70..354b2adca 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -52,6 +52,8 @@ option(EXTERNAL_LIBOSMIUM "Do not use the bundled libosmium" OFF) option(EXTERNAL_PROTOZERO "Do not use the bundled protozero" OFF) option(EXTERNAL_FMT "Do not use the bundled fmt" OFF) +set(USE_PROJ_LIB "auto" CACHE STRING "Which version of PROJ API to use: ('4', '6', 'off', or 'auto')") + if (NOT WIN32 AND NOT APPLE) # No need for this path, just a workaround to make cmake work on all systems. # Without this we need the PostgreSQL server libraries installed. @@ -202,22 +204,45 @@ find_package(Threads) set(LIBS ${Boost_LIBRARIES} ${PostgreSQL_LIBRARY} ${OSMIUM_LIBRARIES}) -find_path(PROJ4_INCLUDE_DIR proj_api.h) -find_library(PROJ_LIBRARY NAMES proj) -if (PROJ4_INCLUDE_DIR AND PROJ_LIBRARY) - message(STATUS "Found Proj4 ${PROJ_LIBRARY}") - set(HAVE_GENERIC_PROJ 1) - set(HAVE_PROJ4 1) - list(APPEND LIBS ${PROJ_LIBRARY}) - include_directories(SYSTEM ${PROJ_INCLUDE_DIR}) +if (USE_PROJ_LIB STREQUAL "off") + message(STATUS "Proj library disabled (because USE_PROJ_LIB is set to 'off').") else() - message(STATUS "Proj library not found.") - message(STATUS " Only Mercartor and WGS84 projections will be available.") + find_path(PROJ4_INCLUDE_DIR proj_api.h) + if (PROJ4_INCLUDE_DIR AND NOT USE_PROJ_LIB STREQUAL "6") + message(STATUS "Found proj_api.h") + find_library(PROJ_LIBRARY NAMES proj) + message(STATUS "Found Proj [API 4] ${PROJ_LIBRARY}") + set(HAVE_GENERIC_PROJ 1) + set(HAVE_PROJ4 1) + list(APPEND LIBS ${PROJ_LIBRARY}) + include_directories(SYSTEM ${PROJ4_INCLUDE_DIR}) + elseif (NOT USE_PROJ_LIB STREQUAL "4") + find_path(PROJ6_INCLUDE_DIR proj.h) + find_library(PROJ_LIBRARY NAMES proj) + if (PROJ_LIBRARY) + message(STATUS "Found Proj [API 6] ${PROJ_LIBRARY}") + set(HAVE_GENERIC_PROJ 1) + set(HAVE_PROJ6 1) + list(APPEND LIBS ${PROJ_LIBRARY}) + include_directories(SYSTEM ${PROJ6_INCLUDE_DIR}) + else() + message(STATUS "Proj library not found.") + message(STATUS " Only Mercartor and WGS84 projections will be available.") + endif() + endif() +endif() + +if (USE_PROJ_LIB STREQUAL "4" AND NOT HAVE_PROJ4) + message(FATAL_ERROR "USE_PROJ_LIB was set to '4', but PROJ version 4 API not found") +endif() + +if (USE_PROJ_LIB STREQUAL "6" AND NOT HAVE_PROJ6) + message(FATAL_ERROR "USE_PROJ_LIB was set to '6', but PROJ version 4 API not found") endif() if (LUAJIT_FOUND) list(APPEND LIBS ${LUAJIT_LIBRARIES}) -elseif(LUA_FOUND) +elseif (LUA_FOUND) list(APPEND LIBS ${LUA_LIBRARIES}) endif() diff --git a/README.md b/README.md index b2a9d558a..565bbcef0 100644 --- a/README.md +++ b/README.md @@ -183,6 +183,23 @@ Also available is the new flex output. It is much more flexible than the other outputs. IT IS CURRENTLY EXPERIMENTAL AND SUBJECT TO CHANGE. The flex output is only available if you have compiled osm2pgsql with Lua support. +## Projection Support ## + +Osm2pgsql has builtin support for the Latlong (WGS84, EPSG:4326) and the +WebMercator (EPSG:3857) projection. If you need other projections you have to +compile with the PROJ library. + +Both the older API (PROJ version 4) and the newer API (PROJ version 6.1 and +above) are supported. Usually the CMake configuration will find a suitable +version and use it automatically, but you can set the `USE_PROJ_LIB` CMake +cache variable to choose between the following behaviours: + +* `4`: Look for PROJ library with API version 4. If it is not found, stop with error. +* `6`: Look for PROJ library with API version 6. If it is not found, stop with error. +* `off`: Build without PROJ library. +* `auto`: Choose API 4 if available, otherwise API 6. If both are not available + build without PROJ library. (This is the default.) + ## LuaJIT support ## To speed up Lua tag transformations, [LuaJIT](https://luajit.org/) can be optionally diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index aafedc3e5..3492418b7 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -61,6 +61,8 @@ if (HAVE_PROJ4) set_source_files_properties(reprojection-generic-proj4.cpp PROPERTIES COMPILE_FLAGS -Wno-deprecated-declarations) endif() +elseif(HAVE_PROJ6) + list(APPEND osm2pgsql_lib_SOURCES reprojection-generic-proj6.cpp) else() list(APPEND osm2pgsql_lib_SOURCES reprojection-generic-none.cpp) endif() diff --git a/src/options.cpp b/src/options.cpp index feb3f73bd..e70d0d241 100644 --- a/src/options.cpp +++ b/src/options.cpp @@ -576,6 +576,7 @@ options_t::options_t(int argc, char *argv[]) : options_t() case 'V': fmt::print(stderr, "Compiled using the following library versions:\n"); fmt::print(stderr, "Libosmium {}\n", LIBOSMIUM_VERSION_STRING); + fmt::print(stderr, "Proj {}\n", get_proj_version()); #ifdef HAVE_LUA fmt::print(stderr, "{}", LUA_RELEASE); #ifdef HAVE_LUAJIT diff --git a/src/reprojection-generic-none.cpp b/src/reprojection-generic-none.cpp index 4438be191..ddc01b664 100644 --- a/src/reprojection-generic-none.cpp +++ b/src/reprojection-generic-none.cpp @@ -6,3 +6,6 @@ std::shared_ptr reprojection::make_generic_projection(int) { throw std::runtime_error{"No generic projection library available."}; } + +std::string get_proj_version() { return "[disabled]"; } + diff --git a/src/reprojection-generic-proj4.cpp b/src/reprojection-generic-proj4.cpp index 1a4442651..df8106897 100644 --- a/src/reprojection-generic-proj4.cpp +++ b/src/reprojection-generic-proj4.cpp @@ -1,3 +1,4 @@ +#include "format.hpp" #include "reprojection.hpp" #include @@ -57,3 +58,6 @@ std::shared_ptr reprojection::make_generic_projection(int srs) { return std::make_shared(srs); } + +std::string get_proj_version() { return "[API 4] {}"_format(pj_get_release()); } + diff --git a/src/reprojection-generic-proj6.cpp b/src/reprojection-generic-proj6.cpp new file mode 100644 index 000000000..628fe4d09 --- /dev/null +++ b/src/reprojection-generic-proj6.cpp @@ -0,0 +1,127 @@ +#include "format.hpp" +#include "reprojection.hpp" + +#include +#include + +#include "proj.h" + +namespace { + +/** + * Generic projection using proj library (version 6 and above). + */ +class generic_reprojection_t : public reprojection +{ +public: + explicit generic_reprojection_t(int srs) + : m_target_srs(srs), m_context(proj_context_create()) + { + assert(m_context); + + m_transformation = create_transformation(PROJ_LATLONG, srs); + + m_transformation.reset(proj_normalize_for_visualization( + m_context.get(), m_transformation.get())); + + if (!m_transformation) { + throw std::runtime_error{ + "Invalid projection '{}': {}"_format(srs, errormsg())}; + } + + m_transformation_tile = create_transformation(PROJ_SPHERE_MERC, srs); + } + + osmium::geom::Coordinates reproject(osmium::Location loc) const override + { + return transform(m_transformation.get(), + osmium::geom::Coordinates{loc.lon_without_check(), + loc.lat_without_check()}); + } + + osmium::geom::Coordinates + target_to_tile(osmium::geom::Coordinates coords) const override + { + return transform(m_transformation_tile.get(), coords); + } + + int target_srs() const noexcept override { return m_target_srs; } + + char const *target_desc() const noexcept override { return ""; } + +private: + struct pj_context_deleter_t + { + void operator()(PJ_CONTEXT *ctx) const noexcept + { + proj_context_destroy(ctx); + } + }; + + struct pj_deleter_t + { + void operator()(PJ *p) const noexcept { proj_destroy(p); } + }; + + char const *errormsg() const noexcept + { + return proj_errno_string(proj_context_errno(m_context.get())); + } + + std::unique_ptr create_transformation(int from, + int to) const + { + std::string const source = "epsg:{}"_format(from); + std::string const target = "epsg:{}"_format(to); + + std::unique_ptr trans{proj_create_crs_to_crs( + m_context.get(), source.c_str(), target.c_str(), nullptr)}; + + if (!trans) { + throw std::runtime_error{ + "Invalid projection from {} to {}: {}"_format(from, to, + errormsg())}; + } + return trans; + } + + osmium::geom::Coordinates transform(PJ *transformation, + osmium::geom::Coordinates coords) const + noexcept + { + PJ_COORD c_in; + c_in.lpzt.z = 0.0; + c_in.lpzt.t = HUGE_VAL; + c_in.lpzt.lam = osmium::geom::deg_to_rad(coords.x); + c_in.lpzt.phi = osmium::geom::deg_to_rad(coords.y); + + auto const c_out = proj_trans(transformation, PJ_FWD, c_in); + + return osmium::geom::Coordinates{c_out.xy.x, c_out.xy.y}; + } + + int m_target_srs; + std::unique_ptr m_context; + std::unique_ptr m_transformation; + + /** + * The projection used for tiles. Currently this is fixed to be Spherical + * Mercator. You will usually have tiles in the same projection as used + * for PostGIS, but it is theoretically possible to have your PostGIS data + * in, say, lat/lon but still create tiles in Spherical Mercator. + */ + std::unique_ptr m_transformation_tile; +}; + +} // anonymous namespace + +std::shared_ptr reprojection::make_generic_projection(int srs) +{ + return std::make_shared(srs); +} + +std::string get_proj_version() +{ + return "[API 6] {}"_format(proj_info().version); +} + diff --git a/src/reprojection.hpp b/src/reprojection.hpp index 3ff6fea1c..a176d3b67 100644 --- a/src/reprojection.hpp +++ b/src/reprojection.hpp @@ -9,11 +9,12 @@ * It contains the reprojection class. */ -#include - #include #include +#include +#include + enum Projection { PROJ_LATLONG = 4326, @@ -70,4 +71,6 @@ class reprojection static std::shared_ptr make_generic_projection(int srs); }; +std::string get_proj_version(); + #endif // OSM2PGSQL_REPROJECTION_HPP diff --git a/tests/test-reprojection.cpp b/tests/test-reprojection.cpp index 46c7e867a..76cffe0cc 100644 --- a/tests/test-reprojection.cpp +++ b/tests/test-reprojection.cpp @@ -4,7 +4,7 @@ TEST_CASE("projection 4326", "[NoDB]") { - const osmium::Location loc{10.0, 53.0}; + osmium::Location const loc{10.0, 53.0}; int const srs = 4326; auto const reprojection = reprojection::create_projection(srs); @@ -22,7 +22,7 @@ TEST_CASE("projection 4326", "[NoDB]") TEST_CASE("projection 3857", "[NoDB]") { - const osmium::Location loc{10.0, 53.0}; + osmium::Location const loc{10.0, 53.0}; int const srs = 3857; auto const reprojection = reprojection::create_projection(srs); @@ -41,8 +41,8 @@ TEST_CASE("projection 3857", "[NoDB]") #ifdef HAVE_GENERIC_PROJ TEST_CASE("projection 5520", "[NoDB]") { - const osmium::Location loc{10.0, 53.0}; - int const srs = 5520; + osmium::Location const loc{10.0, 53.0}; + int const srs = 5520; // DHDN / 3-degree Gauss-Kruger zone 1 auto const reprojection = reprojection::create_projection(srs); REQUIRE(reprojection->target_srs() == srs);