diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3b2a94d4f..66662ed32 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -39,8 +39,7 @@ jobs: env: PGHOST: /tmp - - ubuntu18-pg95-gcc7-jit: + ubuntu18-pg96-gcc7-jit: runs-on: ubuntu-18.04 env: @@ -48,7 +47,7 @@ jobs: CXX: g++-7 LUA_VERSION: 5.3 LUAJIT_OPTION: ON - POSTGRESQL_VERSION: 9.5 + POSTGRESQL_VERSION: 9.6 POSTGIS_VERSION: 2.4 BUILD_TYPE: Release diff --git a/CMakeLists.txt b/CMakeLists.txt index 021b792b3..9b15d29f0 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -66,10 +66,10 @@ if (NOT WIN32 AND NOT APPLE) set(PostgreSQL_TYPE_INCLUDE_DIR /usr/include) endif() -set(MINIMUM_POSTGRESQL_SERVER_VERSION "9.5") -set(MINIMUM_POSTGRESQL_SERVER_VERSION_NUM "90500") +set(MINIMUM_POSTGRESQL_SERVER_VERSION "9.6") +set(MINIMUM_POSTGRESQL_SERVER_VERSION_NUM "90600") -set(PostgreSQL_ADDITIONAL_VERSIONS "14" "13" "12" "11" "10" "9.6" "9.5") +set(PostgreSQL_ADDITIONAL_VERSIONS "15" "14" "13" "12" "11" "10" "9.6") ############################################################# # Version diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index fb52fa4b6..14b81a8b1 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -2,7 +2,6 @@ add_library(osm2pgsql_lib STATIC) target_sources(osm2pgsql_lib PRIVATE - db-check.cpp db-copy.cpp dependency-manager.cpp expire-tiles.cpp diff --git a/src/db-check.cpp b/src/db-check.cpp deleted file mode 100644 index 36e78c061..000000000 --- a/src/db-check.cpp +++ /dev/null @@ -1,79 +0,0 @@ -/** - * SPDX-License-Identifier: GPL-2.0-or-later - * - * This file is part of osm2pgsql (https://osm2pgsql.org/). - * - * Copyright (C) 2006-2022 by the osm2pgsql developer community. - * For a full list of authors see the git log. - */ - -#include "db-check.hpp" -#include "format.hpp" -#include "logging.hpp" -#include "options.hpp" -#include "pgsql.hpp" -#include "version.hpp" - -#include -#include -#include - -/** - * Check whether the table with the specified name exists in the specified - * schema in the database. Leave schema empty to check in the 'public' schema. - */ -static bool has_table(pg_conn_t const &db_connection, std::string const &schema, - std::string const &table) -{ - auto const sql = "SELECT count(*) FROM pg_tables" - " WHERE schemaname='{}' AND tablename='{}'"_format( - schema.empty() ? "public" : schema, table); - auto const res = db_connection.query(PGRES_TUPLES_OK, sql); - char const *const num = res.get_value(0, 0); - - return num[0] == '1' && num[1] == '\0'; -} - -void check_db(options_t const &options) -{ - pg_conn_t db_connection{options.conninfo}; - - auto const settings = get_postgresql_settings(db_connection); - - try { - log_info("Database version: {}", settings.at("server_version")); - - auto const version_str = settings.at("server_version_num"); - auto const version = std::strtoul(version_str.c_str(), nullptr, 10); - if (version < get_minimum_postgresql_server_version_num()) { - throw std::runtime_error{ - "Your database version is too old (need at least {})."_format( - get_minimum_postgresql_server_version())}; - } - - if (settings.at("server_encoding") != "UTF8") { - throw std::runtime_error{"Database is not using UTF8 encoding."}; - } - - auto const postgis_version = get_postgis_version(db_connection); - log_info("PostGIS version: {}.{}", postgis_version.major, - postgis_version.minor); - - // If we are in append mode and the middle nodes table isn't there, - // it probably means we used a flat node store when we created this - // database. Check for that and stop if it looks like we are missing - // the node location store option. - if (options.append && options.flat_node_file.empty()) { - if (!has_table(db_connection, options.middle_dbschema, - options.prefix + "_nodes")) { - throw std::runtime_error{ - "You seem to not have a nodes table. Did " - "you forget the --flat-nodes option?"}; - } - } - - } catch (std::out_of_range const &) { - // Thrown by the settings.at() if the named setting isn't found - throw std::runtime_error{"Can't access database setting."}; - } -} diff --git a/src/db-check.hpp b/src/db-check.hpp deleted file mode 100644 index 00dd3bf86..000000000 --- a/src/db-check.hpp +++ /dev/null @@ -1,21 +0,0 @@ -#ifndef OSM2PGSQL_DB_CHECK_HPP -#define OSM2PGSQL_DB_CHECK_HPP - -/** - * SPDX-License-Identifier: GPL-2.0-or-later - * - * This file is part of osm2pgsql (https://osm2pgsql.org/). - * - * Copyright (C) 2006-2022 by the osm2pgsql developer community. - * For a full list of authors see the git log. - */ - -class options_t; - -/** - * Get settings from the database and check that minimum requirements for - * osm2pgsql are met. This also prints the database version. - */ -void check_db(options_t const &options); - -#endif // OSM2PGSQL_DB_CHECK_HPP diff --git a/src/flex-table.cpp b/src/flex-table.cpp index 562bbe744..c720f8aa9 100644 --- a/src/flex-table.cpp +++ b/src/flex-table.cpp @@ -258,7 +258,7 @@ void table_connection_t::start(bool append) { assert(m_db_connection); - if (!has_schema(*m_db_connection, table().schema())) { + if (!has_schema(table().schema())) { throw std::runtime_error{ "Schema '{0}' not available. " "Use 'CREATE SCHEMA \"{0}\";' to create it."_format( @@ -267,7 +267,7 @@ void table_connection_t::start(bool append) for (auto const &ts : {table().data_tablespace(), table().index_tablespace()}) { - if (!has_tablespace(*m_db_connection, ts)) { + if (!has_tablespace(ts)) { throw std::runtime_error{ "Tablespace '{0}' not available. " "Use 'CREATE TABLESPACE \"{0}\" ...;' to create it."_format( @@ -276,7 +276,7 @@ void table_connection_t::start(bool append) } if (table().has_hstore_column()) { - if (!has_extension(*m_db_connection, "hstore")) { + if (!has_extension("hstore")) { throw std::runtime_error{"Extension 'hstore' not available. Use " "'CREATE EXTENSION hstore;' to load it."}; } @@ -336,7 +336,7 @@ void table_connection_t::stop(bool updateable, bool append) std::string sql = "INSERT INTO {} ({}) SELECT {} FROM {}"_format( table().full_tmp_name(), columns, columns, table().full_name()); - auto const postgis_version = get_postgis_version(*m_db_connection); + auto const postgis_version = get_postgis_version(); sql += " ORDER BY "; if (postgis_version.major == 2 && postgis_version.minor < 4) { diff --git a/src/osm2pgsql.cpp b/src/osm2pgsql.cpp index 83a9230b9..2494dbace 100644 --- a/src/osm2pgsql.cpp +++ b/src/osm2pgsql.cpp @@ -7,7 +7,6 @@ * For a full list of authors see the git log. */ -#include "db-check.hpp" #include "dependency-manager.hpp" #include "input.hpp" #include "logging.hpp" @@ -15,6 +14,9 @@ #include "options.hpp" #include "osmdata.hpp" #include "output.hpp" +#include "pgsql.hpp" +#include "pgsql-capabilities.hpp" +#include "pgsql-helper.hpp" #include "util.hpp" #include "version.hpp" @@ -76,6 +78,26 @@ static void run(options_t const &options) osmdata.stop(); } +void check_db(options_t const &options) +{ + pg_conn_t db_connection{options.conninfo}; + + init_database_capabilities(db_connection); + + // If we are in append mode and the middle nodes table isn't there, + // it probably means we used a flat node store when we created this + // database. Check for that and stop if it looks like we are missing + // the node location store option. + if (options.append && options.flat_node_file.empty()) { + if (!has_table(db_connection, options.middle_dbschema, + options.prefix + "_nodes")) { + throw std::runtime_error{ + "You seem to not have a nodes table. Did " + "you forget the --flat-nodes option?"}; + } + } +} + int main(int argc, char *argv[]) { try { diff --git a/src/pgsql-capabilities.cpp b/src/pgsql-capabilities.cpp index fc09eccb5..400cc25c8 100644 --- a/src/pgsql-capabilities.cpp +++ b/src/pgsql-capabilities.cpp @@ -8,59 +8,169 @@ */ #include "format.hpp" +#include "logging.hpp" +#include "pgsql-capabilities.hpp" #include "pgsql.hpp" -#include "pgsql-helper.hpp" +#include "version.hpp" +#include #include +#include #include -static std::set init_set_from_table(pg_conn_t const &db_connection, - char const *table, - char const *column, - char const *condition) +struct database_capabilities_t { - std::set values; + std::map settings; + std::set extensions; + std::set schemas; + std::set tablespaces; + std::set index_methods; + + std::string database_name; + + uint32_t database_version = 0; + postgis_version postgis{}; +}; + +static database_capabilities_t &capabilities() noexcept +{ + static database_capabilities_t c; + return c; +} + +static void init_set_from_query(std::set *set, + pg_conn_t const &db_connection, + char const *table, char const *column, + char const *condition = "true") +{ auto const res = db_connection.query( PGRES_TUPLES_OK, "SELECT {} FROM {} WHERE {}"_format(column, table, condition)); for (int i = 0; i < res.num_tuples(); ++i) { - values.insert(res.get_value_as_string(i, 0)); + set->insert(res.get_value_as_string(i, 0)); } +} + +/// Get all config settings from the database. +static void init_settings(pg_conn_t const &db_connection) +{ + auto const res = db_connection.query( + PGRES_TUPLES_OK, "SELECT name, setting FROM pg_settings"); - return values; + for (int i = 0; i < res.num_tuples(); ++i) { + capabilities().settings.emplace(res.get_value_as_string(i, 0), + res.get_value_as_string(i, 1)); + } } -bool has_extension(pg_conn_t const &db_connection, std::string const &value) +static void init_database_name(pg_conn_t const &db_connection) { - static const std::set values = init_set_from_table( - db_connection, "pg_catalog.pg_extension", "extname", "true"); + auto const res = + db_connection.query(PGRES_TUPLES_OK, "SELECT current_catalog"); + + if (res.num_tuples() != 1) { + throw std::runtime_error{ + "Database error: Can not access database name."}; + } - return values.count(value); + capabilities().database_name = res.get_value_as_string(0, 0); } -bool has_schema(pg_conn_t const &db_connection, std::string const &value) +static void init_postgis_version(pg_conn_t const &db_connection) { - static const std::set values = init_set_from_table( - db_connection, "pg_catalog.pg_namespace", "nspname", - "nspname !~ '^pg_' AND nspname <> 'information_schema'"); + auto const res = db_connection.query( + PGRES_TUPLES_OK, "SELECT regexp_split_to_table(extversion, '\\.') FROM" + " pg_extension WHERE extname='postgis'"); - if (value.empty()) { - return true; + if (res.num_tuples() == 0) { + throw std::runtime_error{ + "The postgis extension is not enabled on the database '{}'." + " Are you using the correct database?" + " Enable with 'CREATE EXTENSION postgis;'"_format( + capabilities().database_name)}; } - return values.count(value); + capabilities().postgis = {std::stoi(res.get_value_as_string(0, 0)), + std::stoi(res.get_value_as_string(1, 0))}; } -bool has_tablespace(pg_conn_t const &db_connection, std::string const &value) +void init_database_capabilities(pg_conn_t const &db_connection) { - static const std::set values = - init_set_from_table(db_connection, "pg_catalog.pg_tablespace", - "spcname", "spcname != 'pg_global'"); + init_settings(db_connection); + init_database_name(db_connection); + init_postgis_version(db_connection); + + try { + log_info("Database version: {}", + capabilities().settings.at("server_version")); + log_info("PostGIS version: {}.{}", capabilities().postgis.major, + capabilities().postgis.minor); + + auto const version_str = + capabilities().settings.at("server_version_num"); + capabilities().database_version = + std::strtoul(version_str.c_str(), nullptr, 10); + if (capabilities().database_version < + get_minimum_postgresql_server_version_num()) { + throw std::runtime_error{ + "Your database version is too old (need at least {})."_format( + get_minimum_postgresql_server_version())}; + } + if (capabilities().settings.at("server_encoding") != "UTF8") { + throw std::runtime_error{"Database is not using UTF8 encoding."}; + } + + } catch (std::out_of_range const &) { + // Thrown by the settings.at() if the named setting isn't found + throw std::runtime_error{"Can't access database setting."}; + } + + init_set_from_query(&capabilities().extensions, db_connection, + "pg_catalog.pg_extension", "extname"); + init_set_from_query( + &capabilities().schemas, db_connection, "pg_catalog.pg_namespace", + "nspname", "nspname !~ '^pg_' AND nspname <> 'information_schema'"); + init_set_from_query(&capabilities().tablespaces, db_connection, + "pg_catalog.pg_tablespace", "spcname", + "spcname != 'pg_global'"); + init_set_from_query(&capabilities().index_methods, db_connection, + "pg_catalog.pg_am", "amname", "amtype = 'i'"); +} + +bool has_extension(std::string const &value) +{ + return capabilities().extensions.count(value); +} + +bool has_schema(std::string const &value) +{ if (value.empty()) { return true; } + return capabilities().schemas.count(value); +} - return values.count(value); +bool has_tablespace(std::string const &value) +{ + if (value.empty()) { + return true; + } + return capabilities().tablespaces.count(value); +} + +bool has_index_method(std::string const &value) +{ + return capabilities().index_methods.count(value); +} + +uint32_t get_database_version() noexcept +{ + return capabilities().database_version; +} + +postgis_version get_postgis_version() noexcept +{ + return capabilities().postgis; } diff --git a/src/pgsql-capabilities.hpp b/src/pgsql-capabilities.hpp index 0d8bb0700..83e7b0e7f 100644 --- a/src/pgsql-capabilities.hpp +++ b/src/pgsql-capabilities.hpp @@ -10,12 +10,27 @@ * For a full list of authors see the git log. */ -#include "pgsql.hpp" - #include -bool has_extension(pg_conn_t const &db_connection, std::string const &value); -bool has_schema(pg_conn_t const &db_connection, std::string const &value); -bool has_tablespace(pg_conn_t const &db_connection, std::string const &value); +class pg_conn_t; + +void init_database_capabilities(pg_conn_t const &db_connection); + +bool has_extension(std::string const &value); +bool has_schema(std::string const &value); +bool has_tablespace(std::string const &value); +bool has_index_method(std::string const &value); + +/// Get PostgreSQL version in the format (major * 10000 + minor). +uint32_t get_database_version() noexcept; + +struct postgis_version +{ + int major; + int minor; +}; + +/// Get PostGIS major and minor version. +postgis_version get_postgis_version() noexcept; #endif // OSM2PGSQL_PGSQL_CAPABILITIES_HPP diff --git a/src/pgsql-helper.cpp b/src/pgsql-helper.cpp index 9b9f26573..d46e33638 100644 --- a/src/pgsql-helper.cpp +++ b/src/pgsql-helper.cpp @@ -77,3 +77,15 @@ void analyze_table(pg_conn_t const &db_connection, std::string const &schema, auto const qual_name = qualified_name(schema, name); db_connection.exec("ANALYZE {}"_format(qual_name)); } + +bool has_table(pg_conn_t const &db_connection, std::string const &schema, + std::string const &table) +{ + auto const sql = "SELECT count(*) FROM pg_tables" + " WHERE schemaname='{}' AND tablename='{}'"_format( + schema.empty() ? "public" : schema, table); + auto const res = db_connection.query(PGRES_TUPLES_OK, sql); + char const *const num = res.get_value(0, 0); + + return num[0] == '1' && num[1] == '\0'; +} diff --git a/src/pgsql-helper.hpp b/src/pgsql-helper.hpp index 8077fedcc..088f14bf1 100644 --- a/src/pgsql-helper.hpp +++ b/src/pgsql-helper.hpp @@ -42,4 +42,11 @@ void drop_geom_check_trigger(pg_conn_t *db_connection, void analyze_table(pg_conn_t const &db_connection, std::string const &schema, std::string const &name); +/** + * Check whether the table with the specified name exists in the specified + * schema in the database. Leave schema empty to check in the 'public' schema. + */ +bool has_table(pg_conn_t const &db_connection, std::string const &schema, + std::string const &table); + #endif // OSM2PGSQL_PGSQL_HELPER_HPP diff --git a/src/pgsql.cpp b/src/pgsql.cpp index d2ee6876d..e99aaf9d0 100644 --- a/src/pgsql.cpp +++ b/src/pgsql.cpp @@ -209,48 +209,3 @@ void check_identifier(std::string const &name, char const *in) throw std::runtime_error{ "Special characters are not allowed in {}: '{}'."_format(in, name)}; } - -std::map -get_postgresql_settings(pg_conn_t const &db_connection) -{ - auto const res = db_connection.query( - PGRES_TUPLES_OK, "SELECT name, setting FROM pg_settings"); - - std::map settings; - for (int i = 0; i < res.num_tuples(); ++i) { - settings[res.get_value_as_string(i, 0)] = res.get_value_as_string(i, 1); - } - - return settings; -} - -static std::string get_database_name(pg_conn_t const &db_connection) -{ - auto const res = - db_connection.query(PGRES_TUPLES_OK, "SELECT current_catalog"); - - if (res.num_tuples() != 1) { - throw std::runtime_error{ - "Database error: Can not access database name."}; - } - - return res.get_value_as_string(0, 0); -} - -postgis_version get_postgis_version(pg_conn_t const &db_connection) -{ - auto const res = db_connection.query( - PGRES_TUPLES_OK, "SELECT regexp_split_to_table(extversion, '\\.') FROM" - " pg_extension WHERE extname='postgis'"); - - if (res.num_tuples() == 0) { - throw std::runtime_error{ - "The postgis extension is not enabled on the database '{}'." - " Are you using the correct database?" - " Enable with 'CREATE EXTENSION postgis;'"_format( - get_database_name(db_connection))}; - } - - return {std::stoi(res.get_value_as_string(0, 0)), - std::stoi(res.get_value_as_string(1, 0))}; -} diff --git a/src/pgsql.hpp b/src/pgsql.hpp index 169be3325..dcbf10c2e 100644 --- a/src/pgsql.hpp +++ b/src/pgsql.hpp @@ -285,17 +285,4 @@ std::string qualified_name(std::string const &schema, std::string const &name); */ void check_identifier(std::string const &name, char const *in); -struct postgis_version -{ - int major; - int minor; -}; - -/// Get all config settings from the database. -std::map -get_postgresql_settings(pg_conn_t const &db_connection); - -/// Get PostGIS major and minor version. -postgis_version get_postgis_version(pg_conn_t const &db_connection); - #endif // OSM2PGSQL_PGSQL_HPP diff --git a/src/table.cpp b/src/table.cpp index 940af176d..eedcfd3c9 100644 --- a/src/table.cpp +++ b/src/table.cpp @@ -18,6 +18,7 @@ #include "format.hpp" #include "logging.hpp" #include "options.hpp" +#include "pgsql-capabilities.hpp" #include "pgsql-helper.hpp" #include "table.hpp" #include "taginfo.hpp" @@ -205,7 +206,7 @@ void table_t::stop(bool updateable, bool enable_hstore_index, "CREATE TABLE {} {} AS SELECT * FROM {}"_format( qual_tmp_name, m_table_space, qual_name); - auto const postgis_version = get_postgis_version(*m_sql_conn); + auto const postgis_version = get_postgis_version(); sql += " ORDER BY "; if (postgis_version.major == 2 && postgis_version.minor < 4) { diff --git a/tests/common-pg.hpp b/tests/common-pg.hpp index 59d23b4c4..b69ebbaf0 100644 --- a/tests/common-pg.hpp +++ b/tests/common-pg.hpp @@ -18,6 +18,7 @@ #include "format.hpp" #include "options.hpp" #include "pgsql.hpp" +#include "pgsql-capabilities.hpp" #include #ifdef _MSC_VER @@ -107,6 +108,7 @@ class tempdb_t conn_t local = connect(); local.exec("CREATE EXTENSION postgis"); local.exec("CREATE EXTENSION hstore"); + init_database_capabilities(local); } catch (std::runtime_error const &e) { fmt::print(stderr, "Test database cannot be created: {}\n" diff --git a/tests/test-output-flex-schema.cpp b/tests/test-output-flex-schema.cpp index baf6541ed..5dbdd6685 100644 --- a/tests/test-output-flex-schema.cpp +++ b/tests/test-output-flex-schema.cpp @@ -23,6 +23,7 @@ TEST_CASE("config with schema should work") auto conn = db.db().connect(); conn.exec("CREATE SCHEMA IF NOT EXISTS myschema;"); + init_database_capabilities(conn); REQUIRE_NOTHROW(db.run_file(options, data_file)); diff --git a/tests/test-pgsql-capabilities.cpp b/tests/test-pgsql-capabilities.cpp index c6086cce9..c6275319e 100644 --- a/tests/test-pgsql-capabilities.cpp +++ b/tests/test-pgsql-capabilities.cpp @@ -17,23 +17,44 @@ static testing::db::import_t const db; TEST_CASE("has_extension() should work") { - auto const conn = db.db().connect(); - REQUIRE(has_extension(conn, "postgis")); - REQUIRE_FALSE(has_schema(conn, "xzxzxzxz")); + init_database_capabilities(db.db().connect()); + REQUIRE(has_extension("postgis")); + REQUIRE_FALSE(has_schema("xzxzxzxz")); } TEST_CASE("has_schema() should work") { - auto const conn = db.db().connect(); - REQUIRE(has_schema(conn, "public")); - REQUIRE_FALSE(has_schema(conn, "xzxzxzxz")); - REQUIRE_FALSE(has_schema(conn, "pg_toast")); + init_database_capabilities(db.db().connect()); + REQUIRE(has_schema("public")); + REQUIRE_FALSE(has_schema("xzxzxzxz")); + REQUIRE_FALSE(has_schema("pg_toast")); } TEST_CASE("has_tablespace() should work") { - auto const conn = db.db().connect(); - REQUIRE(has_tablespace(conn, "pg_default")); - REQUIRE_FALSE(has_tablespace(conn, "xzxzxzxz")); - REQUIRE_FALSE(has_tablespace(conn, "pg_global")); + init_database_capabilities(db.db().connect()); + REQUIRE(has_tablespace("pg_default")); + REQUIRE_FALSE(has_tablespace("xzxzxzxz")); + REQUIRE_FALSE(has_tablespace("pg_global")); +} + +TEST_CASE("has_index_method() should work") +{ + init_database_capabilities(db.db().connect()); + REQUIRE(has_index_method("btree")); + REQUIRE_FALSE(has_index_method("xzxzxzxz")); +} + +TEST_CASE("PostgreSQL version") +{ + init_database_capabilities(db.db().connect()); + auto const version = get_database_version(); + REQUIRE(version >= 9); +} + +TEST_CASE("PostGIS version") +{ + init_database_capabilities(db.db().connect()); + auto const postgis_version = get_postgis_version(); + REQUIRE(postgis_version.major >= 2); } diff --git a/tests/test-pgsql.cpp b/tests/test-pgsql.cpp index 611825f23..202c5cc8d 100644 --- a/tests/test-pgsql.cpp +++ b/tests/test-pgsql.cpp @@ -34,13 +34,6 @@ TEST_CASE("Table name with schema") REQUIRE(qualified_name("osm", "foo") == R"("osm"."foo")"); } -TEST_CASE("PostGIS version") -{ - auto conn = db.db().connect(); - auto const postgis_version = get_postgis_version(conn); - REQUIRE(postgis_version.major >= 2); -} - TEST_CASE("query with SELECT should work") { auto conn = db.db().connect();