From 6874c441d488b08f9723e99b97cbc3b75e2a04ef Mon Sep 17 00:00:00 2001 From: Jochen Topf Date: Tue, 21 Feb 2023 17:29:04 +0100 Subject: [PATCH 1/2] Move functions for expire tiles output into expire_output_t class --- src/expire-output.cpp | 53 +++++++++++++++++++++++++--- src/expire-output.hpp | 19 ++++++++++ src/expire-tiles.cpp | 47 ------------------------- src/expire-tiles.hpp | 80 ------------------------------------------- src/output-flex.cpp | 2 +- src/output-pgsql.cpp | 11 +++--- src/tile.hpp | 51 +++++++++++++++++++++++++++ 7 files changed, 126 insertions(+), 137 deletions(-) diff --git a/src/expire-output.cpp b/src/expire-output.cpp index 6f46e9fbf..5e8fb7dea 100644 --- a/src/expire-output.cpp +++ b/src/expire-output.cpp @@ -9,7 +9,9 @@ #include "expire-output.hpp" -#include "expire-tiles.hpp" +#include "format.hpp" +#include "logging.hpp" +#include "pgsql.hpp" #include "tile.hpp" std::size_t expire_output_t::output(quadkey_list_t const &tile_list, @@ -17,12 +19,53 @@ std::size_t expire_output_t::output(quadkey_list_t const &tile_list, { std::size_t num = 0; if (!m_filename.empty()) { - num = output_tiles_to_file(tile_list, m_minzoom, m_maxzoom, - m_filename.c_str()); + num = output_tiles_to_file(tile_list); } if (!m_table.empty()) { - num = output_tiles_to_table(tile_list, m_minzoom, m_maxzoom, conninfo, - m_schema, m_table); + num = output_tiles_to_table(tile_list, conninfo); } return num; } + +std::size_t expire_output_t::output_tiles_to_file( + quadkey_list_t const &tiles_at_maxzoom) const +{ + FILE *outfile = std::fopen(m_filename.data(), "a"); + if (outfile == nullptr) { + log_warn("Failed to open expired tiles file ({}). Tile expiry " + "list will not be written!", + std::strerror(errno)); + return 0; + } + + auto const count = for_each_tile( + tiles_at_maxzoom, m_minzoom, m_maxzoom, [&](tile_t const &tile) { + fmt::print(outfile, "{}/{}/{}\n", tile.zoom(), tile.x(), tile.y()); + }); + + (void)std::fclose(outfile); + + return count; +} + +std::size_t +expire_output_t::output_tiles_to_table(quadkey_list_t const &tiles_at_maxzoom, + std::string const &conninfo) const +{ + auto const qn = qualified_name(m_schema, m_table); + + pg_conn_t connection{conninfo}; + + connection.exec("PREPARE insert_tiles(int4, int4, int4) AS" + " INSERT INTO {} (zoom, x, y) VALUES ($1, $2, $3)" + " ON CONFLICT DO NOTHING", + qn); + + auto const count = for_each_tile( + tiles_at_maxzoom, m_minzoom, m_maxzoom, [&](tile_t const &tile) { + connection.exec_prepared("insert_tiles", tile.zoom(), tile.x(), + tile.y()); + }); + + return count; +} diff --git a/src/expire-output.hpp b/src/expire-output.hpp index 82579d56f..f3a7fad4f 100644 --- a/src/expire-output.hpp +++ b/src/expire-output.hpp @@ -21,6 +21,8 @@ class expire_output_t { public: + expire_output_t() = default; + explicit expire_output_t(std::string name) : m_name(std::move(name)) {} std::string const &name() const noexcept { return m_name; } @@ -51,6 +53,23 @@ class expire_output_t std::size_t output(quadkey_list_t const &tile_list, std::string const &conninfo) const; + /** + * Write the list of tiles to a file. + * + * \param tiles_at_maxzoom The list of tiles at maximum zoom level + */ + std::size_t + output_tiles_to_file(quadkey_list_t const &tiles_at_maxzoom) const; + + /** + * Write the list of tiles to a database table. + * + * \param tiles_at_maxzoom The list of tiles at maximum zoom level + * \param conninfo database connection info + */ + std::size_t output_tiles_to_table(quadkey_list_t const &tiles_at_maxzoom, + std::string const &conninfo) const; + private: /// The internal (unique) name of the output std::string m_name; diff --git a/src/expire-tiles.cpp b/src/expire-tiles.cpp index de9a3d9a5..38c08f8c7 100644 --- a/src/expire-tiles.cpp +++ b/src/expire-tiles.cpp @@ -27,7 +27,6 @@ #include "expire-tiles.hpp" #include "format.hpp" #include "geom-functions.hpp" -#include "logging.hpp" #include "options.hpp" #include "reprojection.hpp" #include "table.hpp" @@ -292,52 +291,6 @@ void expire_tiles::merge_and_destroy(expire_tiles *other) } } -std::size_t output_tiles_to_file(quadkey_list_t const &tiles_at_maxzoom, - uint32_t minzoom, uint32_t maxzoom, - std::string_view filename) -{ - FILE *outfile = std::fopen(filename.data(), "a"); - if (outfile == nullptr) { - log_warn("Failed to open expired tiles file ({}). Tile expiry " - "list will not be written!", - std::strerror(errno)); - return 0; - } - - auto const count = for_each_tile( - tiles_at_maxzoom, minzoom, maxzoom, [&](tile_t const &tile) { - fmt::print(outfile, "{}/{}/{}\n", tile.zoom(), tile.x(), tile.y()); - }); - - (void)std::fclose(outfile); - - return count; -} - -std::size_t output_tiles_to_table(quadkey_list_t const &tiles_at_maxzoom, - uint32_t minzoom, uint32_t maxzoom, - std::string const &conninfo, - std::string const &schema, - std::string const &table) -{ - auto const qn = qualified_name(schema, table); - - pg_conn_t connection{conninfo}; - - connection.exec("PREPARE insert_tiles(int4, int4, int4) AS" - " INSERT INTO {} (zoom, x, y) VALUES ($1, $2, $3)" - " ON CONFLICT DO NOTHING", - qn); - - auto const count = for_each_tile( - tiles_at_maxzoom, minzoom, maxzoom, [&](tile_t const &tile) { - connection.exec_prepared("insert_tiles", tile.zoom(), tile.x(), - tile.y()); - }); - - return count; -} - int expire_from_result(expire_tiles *expire, pg_result_t const &result, expire_config_t const &expire_config) { diff --git a/src/expire-tiles.hpp b/src/expire-tiles.hpp index a595bb5e5..b7257a30a 100644 --- a/src/expire-tiles.hpp +++ b/src/expire-tiles.hpp @@ -134,84 +134,4 @@ class expire_tiles int expire_from_result(expire_tiles *expire, pg_result_t const &result, expire_config_t const &expire_config); -/** - * Iterate over tiles and call output function for each tile on all requested - * zoom levels. - * - * \tparam OUTPUT Class with operator() taking a tile_t argument - * - * \param tiles_at_maxzoom The list of tiles at maximum zoom level - * \param minzoom Minimum zoom level - * \param maxzoom Maximum zoom level - * \param output Output function - */ -template -std::size_t for_each_tile(quadkey_list_t const &tiles_at_maxzoom, - uint32_t minzoom, uint32_t maxzoom, OUTPUT &&output) -{ - assert(minzoom <= maxzoom); - - if (minzoom == maxzoom) { - for (auto const quadkey : tiles_at_maxzoom) { - std::forward(output)( - tile_t::from_quadkey(quadkey, maxzoom)); - } - return tiles_at_maxzoom.size(); - } - - /** - * Loop over all requested zoom levels (from maximum down to the minimum - * zoom level). - */ - quadkey_t last_quadkey{}; - std::size_t count = 0; - for (auto const quadkey : tiles_at_maxzoom) { - for (uint32_t dz = 0; dz <= maxzoom - minzoom; ++dz) { - auto const qt_current = quadkey.down(dz); - /** - * If dz > 0, there are probably multiple elements whose quadkey - * is equal because they are all sub-tiles of the same tile at the - * current zoom level. We skip all of them after we have written - * the first sibling. - */ - if (qt_current != last_quadkey.down(dz)) { - std::forward(output)( - tile_t::from_quadkey(qt_current, maxzoom - dz)); - ++count; - } - } - last_quadkey = quadkey; - } - return count; -} - -/** - * Write the list of tiles to a file. - * - * \param tiles_at_maxzoom The list of tiles at maximum zoom level - * \param minzoom Minimum zoom level - * \param maxzoom Maximum zoom level - * \param filename Name of the file - */ -std::size_t output_tiles_to_file(quadkey_list_t const &tiles_at_maxzoom, - uint32_t minzoom, uint32_t maxzoom, - std::string_view filename); - -/** - * Write the list of tiles to a database table. The table will be created - * if it doesn't exist already. - * - * \param tiles_at_maxzoom The list of tiles at maximum zoom level - * \param minzoom Minimum zoom level - * \param maxzoom Maximum zoom level - * \param conninfo database connection info - * \param schema The schema the table is in (empty for public schema) - * \param table The table name - */ -std::size_t output_tiles_to_table(quadkey_list_t const &tiles_at_maxzoom, - uint32_t minzoom, uint32_t maxzoom, - std::string const &conninfo, - std::string const &schema, - std::string const &table); - #endif // OSM2PGSQL_EXPIRE_TILES_HPP diff --git a/src/output-flex.cpp b/src/output-flex.cpp index 485ff352f..f694b9714 100644 --- a/src/output-flex.cpp +++ b/src/output-flex.cpp @@ -1339,7 +1339,7 @@ output_flex_t::output_flex_t(std::shared_ptr const &mid, // For backwards compatibility we add a "default" expire output (with // empty name) when the relevant command line options are used. if (options.append && options.expire_tiles_zoom) { - auto &eo = m_expire_outputs->emplace_back(""); + auto &eo = m_expire_outputs->emplace_back(); eo.set_filename(options.expire_tiles_filename); eo.set_minzoom(options.expire_tiles_zoom_min); eo.set_maxzoom(options.expire_tiles_zoom); diff --git a/src/output-pgsql.cpp b/src/output-pgsql.cpp index 9409219f2..2f2526de7 100644 --- a/src/output-pgsql.cpp +++ b/src/output-pgsql.cpp @@ -28,6 +28,7 @@ #include #include +#include "expire-output.hpp" #include "expire-tiles.hpp" #include "geom-from-osm.hpp" #include "geom-functions.hpp" @@ -146,10 +147,12 @@ void output_pgsql_t::stop() } if (get_options()->expire_tiles_zoom_min > 0) { - auto const count = output_tiles_to_file( - m_expire.get_tiles(), get_options()->expire_tiles_zoom_min, - get_options()->expire_tiles_zoom, - get_options()->expire_tiles_filename); + expire_output_t expire_out; + expire_out.set_filename(get_options()->expire_tiles_filename); + expire_out.set_minzoom(get_options()->expire_tiles_zoom_min); + expire_out.set_maxzoom(get_options()->expire_tiles_zoom); + auto const count = + expire_out.output_tiles_to_file(m_expire.get_tiles()); log_info("Wrote {} entries to expired tiles list", count); } } diff --git a/src/tile.hpp b/src/tile.hpp index 29864174e..46dfc1f86 100644 --- a/src/tile.hpp +++ b/src/tile.hpp @@ -215,4 +215,55 @@ class tile_t uint32_t m_zoom = invalid_zoom; }; // class tile_t +/** + * Iterate over tiles and call output function for each tile on all requested + * zoom levels. + * + * \tparam OUTPUT Class with operator() taking a tile_t argument + * + * \param tiles_at_maxzoom The list of tiles at maximum zoom level + * \param minzoom Minimum zoom level + * \param maxzoom Maximum zoom level + * \param output Output function + */ +template +std::size_t for_each_tile(quadkey_list_t const &tiles_at_maxzoom, + uint32_t minzoom, uint32_t maxzoom, OUTPUT &&output) +{ + assert(minzoom <= maxzoom); + + if (minzoom == maxzoom) { + for (auto const quadkey : tiles_at_maxzoom) { + std::forward(output)( + tile_t::from_quadkey(quadkey, maxzoom)); + } + return tiles_at_maxzoom.size(); + } + + /** + * Loop over all requested zoom levels (from maximum down to the minimum + * zoom level). + */ + quadkey_t last_quadkey{}; + std::size_t count = 0; + for (auto const quadkey : tiles_at_maxzoom) { + for (uint32_t dz = 0; dz <= maxzoom - minzoom; ++dz) { + auto const qt_current = quadkey.down(dz); + /** + * If dz > 0, there are probably multiple elements whose quadkey + * is equal because they are all sub-tiles of the same tile at the + * current zoom level. We skip all of them after we have written + * the first sibling. + */ + if (qt_current != last_quadkey.down(dz)) { + std::forward(output)( + tile_t::from_quadkey(qt_current, maxzoom - dz)); + ++count; + } + } + last_quadkey = quadkey; + } + return count; +} + #endif // OSM2PGSQL_TILE_HPP From 5348f0e415d8936ecb09b7ff34b065f42c230a67 Mon Sep 17 00:00:00 2001 From: Jochen Topf Date: Tue, 21 Feb 2023 17:41:54 +0100 Subject: [PATCH 2/2] Move table creation for expire tables where it belongs With the rest of the expire output code. --- src/expire-output.cpp | 11 +++++++++++ src/expire-output.hpp | 7 +++++++ src/output-flex.cpp | 9 +-------- 3 files changed, 19 insertions(+), 8 deletions(-) diff --git a/src/expire-output.cpp b/src/expire-output.cpp index 5e8fb7dea..1d5b241f5 100644 --- a/src/expire-output.cpp +++ b/src/expire-output.cpp @@ -69,3 +69,14 @@ expire_output_t::output_tiles_to_table(quadkey_list_t const &tiles_at_maxzoom, return count; } + +void expire_output_t::create_output_table(pg_conn_t const &connection) const +{ + auto const qn = qualified_name(m_schema, m_table); + connection.exec("CREATE TABLE IF NOT EXISTS {} (" + " zoom int4 NOT NULL," + " x int4 NOT NULL," + " y int4 NOT NULL," + " PRIMARY KEY (zoom, x, y))", + qn); +} diff --git a/src/expire-output.hpp b/src/expire-output.hpp index f3a7fad4f..3b8d83e7d 100644 --- a/src/expire-output.hpp +++ b/src/expire-output.hpp @@ -15,6 +15,8 @@ #include #include +class pg_conn_t; + /** * Output for tile expiry. */ @@ -70,6 +72,11 @@ class expire_output_t std::size_t output_tiles_to_table(quadkey_list_t const &tiles_at_maxzoom, std::string const &conninfo) const; + /** + * Create table for tiles. + */ + void create_output_table(pg_conn_t const &connection) const; + private: /// The internal (unique) name of the output std::string m_name; diff --git a/src/output-flex.cpp b/src/output-flex.cpp index f694b9714..82570b8e8 100644 --- a/src/output-flex.cpp +++ b/src/output-flex.cpp @@ -1274,14 +1274,7 @@ create_expire_tables(std::vector const &expire_outputs, connection.exec("SET client_min_messages = WARNING"); for (auto &expire_output : expire_outputs) { if (!expire_output.table().empty()) { - auto const qn = - qualified_name(expire_output.schema(), expire_output.table()); - connection.exec("CREATE TABLE IF NOT EXISTS {} (" - " zoom int4 NOT NULL," - " x int4 NOT NULL," - " y int4 NOT NULL," - " PRIMARY KEY (zoom, x, y))", - qn); + expire_output.create_output_table(connection); } } }