diff --git a/CMakeLists.txt b/CMakeLists.txt index 8e17bdda..e99034dc 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -68,9 +68,17 @@ option(PAIMON_ENABLE_LUMINA "Whether to enable lumina vector index" OFF) option(PAIMON_ENABLE_LUCENE "Whether to enable lucene index" OFF) option(PAIMON_ENABLE_TANTIVY "Whether to enable tantivy-fulltext global index (Rust FFI, experimental)" OFF) +option(PAIMON_ENABLE_REST "Whether to enable the rest catalog (requires libcurl)" OFF) if(PAIMON_ENABLE_ORC) add_definitions(-DPAIMON_ENABLE_ORC) endif() +if(PAIMON_ENABLE_REST) + add_definitions(-DPAIMON_ENABLE_REST) +endif() +# libcurl backs the HTTP client shared by the S3 file system and the rest catalog. +if(PAIMON_ENABLE_S3 OR PAIMON_ENABLE_REST) + find_package(CURL REQUIRED) +endif() if(PAIMON_ENABLE_AVRO) add_definitions(-DPAIMON_ENABLE_AVRO) endif() @@ -498,9 +506,6 @@ install(FILES "${CMAKE_CURRENT_BINARY_DIR}/PaimonConfig.cmake" config_summary_message() -if(PAIMON_ENABLE_S3) - find_package(CURL REQUIRED) -endif() add_subdirectory(src/paimon) add_subdirectory(src/paimon/fs/local) if(PAIMON_ENABLE_JINDO) diff --git a/ci/scripts/build_paimon.sh b/ci/scripts/build_paimon.sh index 3df51435..c55a8a6f 100755 --- a/ci/scripts/build_paimon.sh +++ b/ci/scripts/build_paimon.sh @@ -146,6 +146,7 @@ CMAKE_ARGS=( "-DPAIMON_ENABLE_LUMINA=${ENABLE_LUMINA}" "-DPAIMON_ENABLE_LUCENE=ON" "-DPAIMON_ENABLE_TANTIVY=${ENABLE_TANTIVY}" + "-DPAIMON_ENABLE_REST=ON" "-DPAIMON_LINT_GIT_TARGET_COMMIT=${lint_git_target_commit}" ) diff --git a/docs/source/building.rst b/docs/source/building.rst index 2877ed9a..0c5ece36 100644 --- a/docs/source/building.rst +++ b/docs/source/building.rst @@ -126,6 +126,7 @@ boolean flags to ``cmake``. * ``-DPAIMON_ENABLE_LUMINA=ON``: Support for the Lumina vector index. * ``-DPAIMON_ENABLE_LUCENE=ON``: Support for Lucene full-text search indexes * ``-DPAIMON_ENABLE_TANTIVY=ON``: Enable the experimental Tantivy full-text index Rust FFI. +* ``-DPAIMON_ENABLE_REST=ON``: Support for the REST catalog (``metastore=rest``), requires the libcurl development package. Third-party dependency source ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/docs/source/user_guide/catalog.rst b/docs/source/user_guide/catalog.rst index 5ff90536..c57c50c1 100644 --- a/docs/source/user_guide/catalog.rst +++ b/docs/source/user_guide/catalog.rst @@ -23,15 +23,63 @@ Paimon C++ provides a :ref:`Catalog abstraction ` to manage the abstraction provides a series of ways to help you better integrate with computing engines. We always recommend that you use Catalog to access the Paimon table. +Paimon C++ supports two metastores, selected with the catalog option +``metastore``: the filesystem metastore (default) and the REST metastore. + Filesystem Catalog ~~~~~~~~~~~~~~~~~~ -Paimon C++ catalog currently support one types of metastores filesystem metastore (default), -which stores both metadata and table files in filesystems. +The filesystem metastore (``metastore=filesystem``, the default) stores both +metadata and table files in filesystems. The ``root_path`` argument of +``Catalog::Create`` is the warehouse directory holding the databases and tables. + +REST Catalog +~~~~~~~~~~~~ +With the REST metastore (``metastore=rest``), catalog metadata is managed by a +remote catalog server exposed through a REST API; table data itself is still read +and written through the table paths returned by the server. See `Java Paimon REST +Catalog `_ for the concept +and the server-side protocol. + +REST catalog support is an optional build component: configure the build with +``-DPAIMON_ENABLE_REST=ON`` (see :ref:`cpp_build_optional_components`). + +When ``metastore=rest``, the ``root_path`` argument of ``Catalog::Create`` is not +a filesystem path but the warehouse (instance) name under which the tables are +registered on the REST server. The catalog is configured through the +``CatalogOptions`` keys: + +* ``metastore``: must be ``rest`` to select the REST catalog. +* ``uri``: server url of the REST catalog server. +* ``token.provider``: authentication provider of the REST catalog; currently only + ``bear`` is supported (the protocol's historical spelling of "bearer"). +* ``token``: token of the ``bear`` token provider. +* ``table-default.``: table option defaults applied when a created table + left ```` unset. +* ``header.``: sent as the ```` http header on every request to the + server. The server may configure headers of its own through the ``/v1/config`` + endpoint, which are merged with these as any other option is. + +.. code-block:: cpp + + std::map options = { + {"metastore", "rest"}, + {"uri", "http://127.0.0.1:8080"}, + {"token.provider", "bear"}, + {"token", ""}, + }; + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr catalog, + paimon::Catalog::Create(/*root_path=*/"my_instance", options)); + +On creation the catalog queries the server's ``/v1/config`` endpoint and merges +its response with the options above: the server's overrides win over the client +options, which in turn win over the server's defaults. -.. note:: +Databases and tables are then created, listed, loaded, renamed and dropped +through the regular ``Catalog`` API, and table snapshots can be listed through +``Catalog::ListSnapshots``. - Current Paimon C++ only supports filesystem catalog. In the future, we will - support REST catalog. - By using the Paimon REST catalog, changes to the catalog will be directly stored - in a remote catalog server which exposed through REST API. See `Java Paimon REST - Catalog `_. +The C++ REST catalog covers the database, table and snapshot operations of the +``Catalog`` API. The parts of the Java REST catalog that have no C++ counterpart +yet — altering a database or a table, views, functions, partitions, tags, branch +management and consumers — are not supported, and neither is the ``dlf`` token +provider. diff --git a/include/paimon/catalog/catalog.h b/include/paimon/catalog/catalog.h index b5d6bddd..9213bde9 100644 --- a/include/paimon/catalog/catalog.h +++ b/include/paimon/catalog/catalog.h @@ -58,7 +58,9 @@ class PAIMON_EXPORT Catalog { /// %Factory method for creating a `Catalog` instance. /// - /// @param root_path Path to the root directory where the catalog is located. + /// @param root_path Path to the root directory where the catalog is located. For the + /// REST catalog (`CatalogOptions::METASTORE` set to "rest") this is + /// instead the warehouse (instance) name registered on the server. /// @param options Configuration options for catalog initialization. /// @param file_system Specifies the file system for file operations. /// If not set, use default file system (configured in diff --git a/include/paimon/catalog_options.h b/include/paimon/catalog_options.h new file mode 100644 index 00000000..f58a876c --- /dev/null +++ b/include/paimon/catalog_options.h @@ -0,0 +1,45 @@ +/* + * Copyright 2026-present Alibaba Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "paimon/visibility.h" + +namespace paimon { + +/// Catalog-level configuration option keys; table-level keys live in `Options`. +struct PAIMON_EXPORT CatalogOptions { + /// "metastore" - Metastore of the paimon catalog. + /// Supported values are "filesystem" (default) and "rest". + static const char METASTORE[]; + + /// "uri" - Server url of the REST catalog. Only used when METASTORE is "rest". + static const char URI[]; + + /// "token" - Token of the "bear" token provider of the REST catalog. + static const char TOKEN[]; + + /// "token.provider" - Authentication provider of the REST catalog. Only "bear" is + /// supported ("bear" is the protocol's historical spelling of "bearer", do not "fix" it). + static const char TOKEN_PROVIDER[]; + + /// "table-default." - Prefix of the catalog options that provide table option + /// defaults: "table-default.=" applies "=" to a created + /// table when the caller left "" unset. + static const char TABLE_DEFAULT_OPTION_PREFIX[]; +}; + +} // namespace paimon diff --git a/include/paimon/utils/special_field_ids.h b/include/paimon/utils/special_field_ids.h index c4e03f00..829f2988 100644 --- a/include/paimon/utils/special_field_ids.h +++ b/include/paimon/utils/special_field_ids.h @@ -42,6 +42,10 @@ class SpecialFieldIds { /// Special field ID reserved for index score. Value: CPP_FIELD_ID_END - 1 inline static constexpr int32_t INDEX_SCORE = CPP_FIELD_ID_END - 1; + + /// Lowest field ID reserved for system fields; IDs at or above it are excluded from the + /// highest field ID of a schema. Value: INT32_MAX / 2 + inline static constexpr int32_t SYSTEM_FIELD_ID_START = std::numeric_limits::max() / 2; }; } // namespace paimon diff --git a/src/paimon/CMakeLists.txt b/src/paimon/CMakeLists.txt index ce2f5ddf..552f772d 100644 --- a/src/paimon/CMakeLists.txt +++ b/src/paimon/CMakeLists.txt @@ -15,6 +15,7 @@ # limitations under the License. set(PAIMON_COMMON_SRCS + common/catalog_options.cpp common/compression/block_compression_factory.cpp common/compression/block_compressor.cpp common/compression/block_decompressor.cpp @@ -179,13 +180,20 @@ set(PAIMON_COMMON_SRCS common/utils/roaring_bitmap32.cpp common/utils/roaring_bitmap64.cpp common/utils/row_range_index.cpp + common/utils/sensitive_config_utils.cpp common/utils/status.cpp - common/utils/string_utils.cpp) + common/utils/string_utils.cpp + common/utils/url_utils.cpp) +# The shared HTTP client is used by both the object store file systems and the +# rest catalog. +set(PAIMON_CURL_LINK_LIBS) +if(PAIMON_ENABLE_S3 OR PAIMON_ENABLE_REST) + list(APPEND PAIMON_COMMON_SRCS common/utils/http_client.cpp) + set(PAIMON_CURL_LINK_LIBS CURL::libcurl) +endif() if(PAIMON_ENABLE_S3) - list(APPEND PAIMON_COMMON_SRCS common/fs/http_client.cpp - common/fs/object_store_file_system.cpp) - set(PAIMON_OBJECT_STORE_LINK_LIBS CURL::libcurl) + list(APPEND PAIMON_COMMON_SRCS common/fs/object_store_file_system.cpp) endif() set(PAIMON_CORE_SRCS @@ -228,6 +236,7 @@ set(PAIMON_CORE_SRCS core/casting/timestamp_to_timestamp_cast_executor.cpp core/casting/casting_utils.cpp core/catalog/catalog.cpp + core/catalog/catalog_utils.cpp core/catalog/file_system_catalog.cpp core/catalog/identifier.cpp core/core_options.cpp @@ -405,6 +414,18 @@ set(PAIMON_CORE_SRCS core/utils/snapshot_manager.cpp core/utils/tag_manager.cpp) +if(PAIMON_ENABLE_REST) + list(APPEND + PAIMON_CORE_SRCS + rest/rest_http_client.cpp + rest/resource_paths.cpp + rest/rest_api.cpp + rest/rest_auth.cpp + rest/rest_catalog.cpp + rest/rest_messages.cpp + rest/rest_util.cpp) +endif() + add_paimon_lib(paimon SOURCES ${PAIMON_COMMON_SRCS} @@ -418,7 +439,7 @@ add_paimon_lib(paimon xxhash Threads::Threads RapidJSON - ${PAIMON_OBJECT_STORE_LINK_LIBS} + ${PAIMON_CURL_LINK_LIBS} STATIC_LINK_LIBS arrow tbb @@ -428,7 +449,7 @@ add_paimon_lib(paimon xxhash Threads::Threads RapidJSON - ${PAIMON_OBJECT_STORE_LINK_LIBS} + ${PAIMON_CURL_LINK_LIBS} SHARED_LINK_FLAGS ${PAIMON_VERSION_SCRIPT_FLAGS}) @@ -436,6 +457,13 @@ add_subdirectory(common/file_index) add_subdirectory(common/global_index) if(PAIMON_BUILD_TESTS) + # The shared HTTP client is compiled only for the components that need libcurl, + # so its test follows the same gate. + set(PAIMON_COMMON_HTTP_CLIENT_TEST_SRCS) + if(PAIMON_ENABLE_S3 OR PAIMON_ENABLE_REST) + set(PAIMON_COMMON_HTTP_CLIENT_TEST_SRCS common/utils/http_client_test.cpp) + endif() + add_paimon_test(memory_test SOURCES common/memory/memory_pool_test.cpp @@ -593,10 +621,13 @@ if(PAIMON_BUILD_TESTS) common/io/cache/lru_cache_test.cpp common/utils/byte_range_combiner_test.cpp common/utils/scope_guard_test.cpp + common/utils/sensitive_config_utils_test.cpp common/utils/serialization_utils_test.cpp common/utils/status_test.cpp common/utils/stream_utils_test.cpp common/utils/string_utils_test.cpp + common/utils/url_utils_test.cpp + ${PAIMON_COMMON_HTTP_CLIENT_TEST_SRCS} common/utils/range_test.cpp common/utils/uuid_test.cpp common/utils/decimal_utils_test.cpp @@ -874,4 +905,20 @@ if(PAIMON_BUILD_TESTS) EXTRA_INCLUDES ${JINDOSDK_INCLUDE_DIR}) + if(PAIMON_ENABLE_REST) + add_paimon_test(rest_test + SOURCES + rest/rest_http_client_test.cpp + rest/mock_rest_server.cpp + rest/resource_paths_test.cpp + rest/rest_catalog_test.cpp + rest/rest_messages_test.cpp + rest/rest_util_test.cpp + STATIC_LINK_LIBS + paimon_shared + test_utils_static + ${TEST_STATIC_LINK_LIBS} + ${GTEST_LINK_TOOLCHAIN}) + endif() + endif() diff --git a/src/paimon/common/catalog_options.cpp b/src/paimon/common/catalog_options.cpp new file mode 100644 index 00000000..6e89e80a --- /dev/null +++ b/src/paimon/common/catalog_options.cpp @@ -0,0 +1,27 @@ +/* + * Copyright 2026-present Alibaba Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "paimon/catalog_options.h" + +namespace paimon { + +const char CatalogOptions::METASTORE[] = "metastore"; +const char CatalogOptions::URI[] = "uri"; +const char CatalogOptions::TOKEN[] = "token"; +const char CatalogOptions::TOKEN_PROVIDER[] = "token.provider"; +const char CatalogOptions::TABLE_DEFAULT_OPTION_PREFIX[] = "table-default."; + +} // namespace paimon diff --git a/src/paimon/common/fs/http_client.cpp b/src/paimon/common/utils/http_client.cpp similarity index 91% rename from src/paimon/common/fs/http_client.cpp rename to src/paimon/common/utils/http_client.cpp index 3ff589b4..e61c7faa 100644 --- a/src/paimon/common/fs/http_client.cpp +++ b/src/paimon/common/utils/http_client.cpp @@ -17,7 +17,7 @@ * under the License. */ -#include "paimon/common/fs/http_client.h" +#include "paimon/common/utils/http_client.h" #include @@ -47,11 +47,6 @@ class CurlGlobalGuard { } }; -std::shared_ptr GetCurlGlobalGuard() { - static auto guard = std::make_shared(); - return guard; -} - void TrimHttpWhitespace(std::string* value) { constexpr char kHttpWhitespace[] = " \t\r\n"; size_t begin = value->find_first_not_of(kHttpWhitespace); @@ -95,16 +90,7 @@ size_t WriteCallback(char* data, size_t size, size_t count, void* user_data) { size_t HeaderCallback(char* data, size_t size, size_t count, void* user_data) { auto* context = static_cast(user_data); size_t bytes = size * count; - std::string line(data, bytes); - size_t colon = line.find(':'); - if (colon != std::string::npos) { - std::string name = line.substr(0, colon); - TrimHttpWhitespace(&name); - name = StringUtils::ToLowerCase(name); - std::string value = line.substr(colon + 1); - TrimHttpWhitespace(&value); - context->response.headers[name] = std::move(value); - } + ParseHttpHeaderLine(data, bytes, &context->response.headers); return bytes; } @@ -119,9 +105,31 @@ bool IsRetryable(CURLcode code, int64_t status_code) { } // namespace +std::shared_ptr EnsureCurlGlobalInit() { + static auto guard = std::make_shared(); + return guard; +} + +void ParseHttpHeaderLine(const char* data, size_t size, HttpHeaders* headers) { + std::string line(data, size); + size_t colon = line.find(':'); + if (colon == std::string::npos) { + return; + } + std::string name = line.substr(0, colon); + TrimHttpWhitespace(&name); + if (name.empty()) { + return; + } + name = StringUtils::ToLowerCase(name); + std::string value = line.substr(colon + 1); + TrimHttpWhitespace(&value); + (*headers)[name] = std::move(value); +} + class CurlHttpClient::Impl { public: - Impl() : guard_(GetCurlGlobalGuard()) {} + Impl() : guard_(EnsureCurlGlobalInit()) {} ~Impl() { for (CURL* handle : handles_) { @@ -146,7 +154,7 @@ class CurlHttpClient::Impl { } private: - std::shared_ptr guard_; + std::shared_ptr guard_; mutable std::mutex mutex_; mutable std::vector handles_; }; diff --git a/src/paimon/common/fs/http_client.h b/src/paimon/common/utils/http_client.h similarity index 81% rename from src/paimon/common/fs/http_client.h rename to src/paimon/common/utils/http_client.h index 5dd8d029..0dd3e742 100644 --- a/src/paimon/common/fs/http_client.h +++ b/src/paimon/common/utils/http_client.h @@ -34,6 +34,14 @@ enum class HttpMethod { HEAD, GET }; using HttpHeaders = std::map; using HttpBodyConsumer = std::function; +/// Ensures libcurl's global state is initialized; the returned guard keeps it alive. +PAIMON_EXPORT std::shared_ptr EnsureCurlGlobalInit(); + +/// Parses one raw HTTP header line into `headers`, lower-casing the name and trimming +/// HTTP whitespace around the name and value; lines without a ':' or with an empty +/// name are ignored. +PAIMON_EXPORT void ParseHttpHeaderLine(const char* data, size_t size, HttpHeaders* headers); + struct HttpRequest { HttpMethod method = HttpMethod::GET; std::string url; diff --git a/src/paimon/common/utils/http_client_test.cpp b/src/paimon/common/utils/http_client_test.cpp new file mode 100644 index 00000000..5bf3ce23 --- /dev/null +++ b/src/paimon/common/utils/http_client_test.cpp @@ -0,0 +1,47 @@ +/* + * Copyright 2026-present Alibaba Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "paimon/common/utils/http_client.h" + +#include + +#include "gtest/gtest.h" + +namespace paimon::test { + +TEST(HttpClientUtilTest, ParseHttpHeaderLine) { + HttpHeaders headers; + const std::string line = "Content-Type: application/json\r\n"; + ParseHttpHeaderLine(line.data(), line.size(), &headers); + ASSERT_EQ((HttpHeaders{{"content-type", "application/json"}}), headers); + + const std::string overwrite = "content-type: text/PLAIN \r\n"; + ParseHttpHeaderLine(overwrite.data(), overwrite.size(), &headers); + ASSERT_EQ((HttpHeaders{{"content-type", "text/PLAIN"}}), headers); + + // Lines without a colon (like the status line) and empty names are ignored. + const std::string status_line = "HTTP/1.1 200 OK\r\n"; + ParseHttpHeaderLine(status_line.data(), status_line.size(), &headers); + const std::string empty_name = ": value\r\n"; + ParseHttpHeaderLine(empty_name.data(), empty_name.size(), &headers); + ASSERT_EQ(1, headers.size()); + + const std::string empty_value = "x-empty:\r\n"; + ParseHttpHeaderLine(empty_value.data(), empty_value.size(), &headers); + ASSERT_EQ("", headers.at("x-empty")); +} + +} // namespace paimon::test diff --git a/src/paimon/common/utils/sensitive_config_utils.cpp b/src/paimon/common/utils/sensitive_config_utils.cpp new file mode 100644 index 00000000..28740412 --- /dev/null +++ b/src/paimon/common/utils/sensitive_config_utils.cpp @@ -0,0 +1,130 @@ +/* + * Copyright 2026-present Alibaba Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "paimon/common/utils/sensitive_config_utils.h" + +#include "paimon/common/utils/string_utils.h" + +namespace paimon { + +namespace { + +// Substring markers of a credential, matched against the normalized form so that the +// separators of a key or a message do not matter. +constexpr const char* kSensitiveMarkers[] = { + "password", "secret", "token", "credential", "accesskey", "accountkey", + "encryptionkey", "authorization", "privatekey", "apikey", "sas"}; + +// Keys whose value is masked as a whole rather than keeping a trailing hint. Every true +// secret is here; only an identifier-like key keeps a tail. "accessKeySecret" normalizes +// to contain "secret" and is masked as a whole, while "accessKeyId" hits only the +// "accesskey" marker of `kSensitiveMarkers` and keeps its tail. This mirrors AWS/Azure, +// where the access key id is loggable but the secret, the token and the SAS are not. +constexpr const char* kFullMaskMarkers[] = {"password", "token", "authorization", "secret", + "credential", "privatekey", "encryptionkey", "apikey", + "accountkey", "sas"}; + +// A value shorter than this reveals too much of itself through a four character tail, so +// it is masked as a whole. +constexpr size_t kMinLengthForTail = 12; +constexpr size_t kTailLength = 4; + +// Free-form text is scanned for one more marker: in a message "signature" names a +// credential, while in an option key it names a request signature algorithm. +constexpr const char kTextOnlyMarker[] = "signature"; + +// The Azure SAS "sig" would be ambiguous once separators are removed ("sig" is a +// substring of many words), so it is matched literally instead. +constexpr const char* kLiteralTextMarkers[] = {"sig=", "\"sig\"", "'sig'"}; + +// Drops every character of an already lower-cased string that is not a letter or a +// digit, so "access-key", "access.key" and "accessKey" all normalize to "accesskey". +std::string StripSeparators(const std::string& lowered) { + std::string normalized; + normalized.reserve(lowered.size()); + for (char c : lowered) { + if ((c >= 'a' && c <= 'z') || (c >= '0' && c <= '9')) { + normalized.push_back(c); + } + } + return normalized; +} + +template +bool ContainsMarker(const std::string& normalized, const char* const (&markers)[N]) { + for (const char* marker : markers) { + if (normalized.find(marker) != std::string::npos) { + return true; + } + } + return false; +} + +std::string NormalizeKey(const std::string& key) { + return StripSeparators(StringUtils::ToLowerCase(key)); +} + +} // namespace + +bool SensitiveConfigUtils::IsSensitiveKey(const std::string& key) { + if (key.empty()) { + return false; + } + return ContainsMarker(NormalizeKey(key), kSensitiveMarkers); +} + +std::string SensitiveConfigUtils::RedactValue(const std::string& key, const std::string& value) { + if (key.empty()) { + return value; + } + std::string normalized = NormalizeKey(key); + if (!ContainsMarker(normalized, kSensitiveMarkers)) { + return value; + } + if (!ContainsMarker(normalized, kFullMaskMarkers) && value.size() >= kMinLengthForTail) { + return "****" + value.substr(value.size() - kTailLength); + } + return kRedacted; +} + +std::map SensitiveConfigUtils::RedactMap( + const std::map& options) { + std::map redacted; + for (const auto& [key, value] : options) { + redacted.emplace(key, RedactValue(key, value)); + } + return redacted; +} + +std::string SensitiveConfigUtils::RedactText(const std::string& text) { + if (text.empty()) { + return text; + } + std::string lowered = StringUtils::ToLowerCase(text); + for (const char* marker : kLiteralTextMarkers) { + if (lowered.find(marker) != std::string::npos) { + return kRedacted; + } + } + std::string normalized = StripSeparators(lowered); + if (ContainsMarker(normalized, kSensitiveMarkers) || + normalized.find(kTextOnlyMarker) != std::string::npos) { + return kRedacted; + } + return text; +} + +} // namespace paimon diff --git a/src/paimon/common/utils/sensitive_config_utils.h b/src/paimon/common/utils/sensitive_config_utils.h new file mode 100644 index 00000000..71715609 --- /dev/null +++ b/src/paimon/common/utils/sensitive_config_utils.h @@ -0,0 +1,60 @@ +/* + * Copyright 2026-present Alibaba Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include + +#include "paimon/visibility.h" + +namespace paimon { + +/// Redacts credentials (passwords, secrets, tokens, access keys) before they reach a log +/// line, an error message or a user visible table. +class PAIMON_EXPORT SensitiveConfigUtils { + public: + SensitiveConfigUtils() = delete; + ~SensitiveConfigUtils() = delete; + + /// Replaces a redacted option value, and the whole text of a redacted message. + static constexpr const char* kRedacted = "******"; + + /// Returns whether `key` names a credential. The key is matched lower-cased and + /// stripped of separators, so "dlf.access-key-secret", "fs.s3a.access.key", + /// "fs.azure.account-key.store" and "accessKeySecret" all hit a marker. + static bool IsSensitiveKey(const std::string& key); + + /// Returns `value` masked when `key` names a credential, and `value` unchanged + /// otherwise. A key naming a true secret (a token, a password, an account key, ...) + /// is masked as a whole, while an identifier-like key ("dlf.access-key-id" hits only + /// the "accesskey" marker) keeps the last four characters of a long enough value: + /// enough to tell two credentials apart in a support case, not enough to use either. + static std::string RedactValue(const std::string& key, const std::string& value); + + /// Returns a copy of `options` with the value of every sensitive key redacted by + /// `RedactValue`. Every surface exposing catalog or table options to users (e.g. the + /// `sys.catalog_options` table) must pass them through this filter. + static std::map RedactMap( + const std::map& options); + + /// Redacts free-form text such as a server error message: arbitrary text cannot be + /// masked per-secret reliably, so the whole text becomes `kRedacted` as soon as any + /// marker of a sensitive value ("password", "token", "sig=", ...) appears. + static std::string RedactText(const std::string& text); +}; + +} // namespace paimon diff --git a/src/paimon/common/utils/sensitive_config_utils_test.cpp b/src/paimon/common/utils/sensitive_config_utils_test.cpp new file mode 100644 index 00000000..5e8b5350 --- /dev/null +++ b/src/paimon/common/utils/sensitive_config_utils_test.cpp @@ -0,0 +1,107 @@ +/* + * Copyright 2026-present Alibaba Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "paimon/common/utils/sensitive_config_utils.h" + +#include +#include + +#include "gtest/gtest.h" +#include "paimon/testing/utils/testharness.h" + +namespace paimon::test { + +TEST(SensitiveConfigUtilsTest, IsSensitiveKey) { + // the key is matched lower-cased and stripped of separators, so the same marker hits + // whichever separator style a key uses + ASSERT_TRUE(SensitiveConfigUtils::IsSensitiveKey("token")); + ASSERT_TRUE(SensitiveConfigUtils::IsSensitiveKey("dlf.access-key-secret")); + ASSERT_TRUE(SensitiveConfigUtils::IsSensitiveKey("fs.s3a.access.key")); + ASSERT_TRUE(SensitiveConfigUtils::IsSensitiveKey("fs.azure.account-key.store1")); + ASSERT_TRUE(SensitiveConfigUtils::IsSensitiveKey("accessKeySecret")); + ASSERT_TRUE(SensitiveConfigUtils::IsSensitiveKey("client.credential")); + ASSERT_TRUE(SensitiveConfigUtils::IsSensitiveKey("fs.azure.sas.container")); + ASSERT_TRUE(SensitiveConfigUtils::IsSensitiveKey("HTTP.Authorization")); + + // the match is a substring one, so a key merely naming a credential is masked too + ASSERT_TRUE(SensitiveConfigUtils::IsSensitiveKey("token.provider")); + + ASSERT_FALSE(SensitiveConfigUtils::IsSensitiveKey("")); + ASSERT_FALSE(SensitiveConfigUtils::IsSensitiveKey("uri")); + ASSERT_FALSE(SensitiveConfigUtils::IsSensitiveKey("file.format")); + // "signature" marks free-form text, not an option key: in a key it names the signing + // algorithm rather than a stored credential + ASSERT_FALSE(SensitiveConfigUtils::IsSensitiveKey("dlf.signing-algorithm")); +} + +TEST(SensitiveConfigUtilsTest, RedactValue) { + // a key naming no credential keeps its value + ASSERT_EQ("orc", SensitiveConfigUtils::RedactValue("file.format", "orc")); + ASSERT_EQ("v", SensitiveConfigUtils::RedactValue("", "v")); + + // a key naming a true secret is masked as a whole, however long the value is + ASSERT_EQ(SensitiveConfigUtils::kRedacted, + SensitiveConfigUtils::RedactValue("token", "bearer-credential-1")); + ASSERT_EQ(SensitiveConfigUtils::kRedacted, + SensitiveConfigUtils::RedactValue("dlf.access-key-secret", "secret-value-1")); + + // an identifier-like key keeps the last four characters of a long enough value, so + // two credentials can be told apart without disclosing either + ASSERT_EQ("****k-id", SensitiveConfigUtils::RedactValue("dlf.access-key-id", "an-access-k-id")); + // a short value would reveal too much of itself through the tail + ASSERT_EQ(SensitiveConfigUtils::kRedacted, + SensitiveConfigUtils::RedactValue("dlf.access-key-id", "short-id")); +} + +TEST(SensitiveConfigUtilsTest, RedactMap) { + const std::map options = { + {"uri", "http://127.0.0.1:8080"}, + {"token", "bearer-credential"}, + {"dlf.access-key-secret", "ak-secret"}, + {"file.format", "orc"}, + }; + std::map redacted = SensitiveConfigUtils::RedactMap(options); + // every key stays listed, only the credential-carrying values are replaced + ASSERT_EQ(options.size(), redacted.size()); + ASSERT_EQ("http://127.0.0.1:8080", redacted.at("uri")); + ASSERT_EQ("orc", redacted.at("file.format")); + ASSERT_EQ(SensitiveConfigUtils::kRedacted, redacted.at("token")); + ASSERT_EQ(SensitiveConfigUtils::kRedacted, redacted.at("dlf.access-key-secret")); + + ASSERT_TRUE(SensitiveConfigUtils::RedactMap({}).empty()); +} + +TEST(SensitiveConfigUtilsTest, RedactText) { + // a marker anywhere in the text redacts all of it: arbitrary text cannot be masked + // per-secret reliably + ASSERT_EQ(SensitiveConfigUtils::kRedacted, + SensitiveConfigUtils::RedactText("invalid password=abc123")); + ASSERT_EQ(SensitiveConfigUtils::kRedacted, + SensitiveConfigUtils::RedactText("bad ACCESS-KEY provided")); + ASSERT_EQ(SensitiveConfigUtils::kRedacted, + SensitiveConfigUtils::RedactText("url?X-Amz-Signature=deadbeef")); + // the Azure SAS "sig" is matched literally, since it is ambiguous once the separators + // are stripped + ASSERT_EQ(SensitiveConfigUtils::kRedacted, + SensitiveConfigUtils::RedactText("url?sig=deadbeef")); + + ASSERT_EQ("table t1 not found", SensitiveConfigUtils::RedactText("table t1 not found")); + // a word merely containing "sig" is not a marker + ASSERT_EQ("design is invalid", SensitiveConfigUtils::RedactText("design is invalid")); + ASSERT_EQ("", SensitiveConfigUtils::RedactText("")); +} + +} // namespace paimon::test diff --git a/src/paimon/common/utils/url_utils.cpp b/src/paimon/common/utils/url_utils.cpp new file mode 100644 index 00000000..883a4b77 --- /dev/null +++ b/src/paimon/common/utils/url_utils.cpp @@ -0,0 +1,132 @@ +/* + * Copyright 2026-present Alibaba Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "paimon/common/utils/url_utils.h" + +#include + +namespace paimon { + +namespace { + +bool IsAlphaNumeric(char c) { + return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9'); +} + +bool IsFormUnreservedChar(char c) { + return IsAlphaNumeric(c) || c == '.' || c == '-' || c == '*' || c == '_'; +} + +bool IsRfc3986UnreservedChar(char c) { + return IsAlphaNumeric(c) || c == '-' || c == '.' || c == '_' || c == '~'; +} + +void AppendPercentEncoded(char c, std::string* out) { + char buf[4]; + std::snprintf(buf, sizeof(buf), "%%%02X", static_cast(c)); + out->append(buf); +} + +int32_t HexValue(char c) { + if (c >= '0' && c <= '9') { + return c - '0'; + } + if (c >= 'a' && c <= 'f') { + return c - 'a' + 10; + } + if (c >= 'A' && c <= 'F') { + return c - 'A' + 10; + } + return -1; +} + +} // namespace + +std::string UrlUtils::EncodeString(const std::string& input) { + std::string encoded; + encoded.reserve(input.size()); + for (char c : input) { + if (IsFormUnreservedChar(c)) { + encoded.push_back(c); + } else if (c == ' ') { + encoded.push_back('+'); + } else { + AppendPercentEncoded(c, &encoded); + } + } + return encoded; +} + +std::string UrlUtils::DecodeString(const std::string& input) { + std::string decoded; + decoded.reserve(input.size()); + for (size_t i = 0; i < input.size(); i++) { + char c = input[i]; + if (c == '+') { + decoded.push_back(' '); + } else if (c == '%' && i + 2 < input.size()) { + int32_t high = HexValue(input[i + 1]); + int32_t low = HexValue(input[i + 2]); + if (high >= 0 && low >= 0) { + decoded.push_back(static_cast((high << 4) | low)); + i += 2; + } else { + decoded.push_back(c); + } + } else { + decoded.push_back(c); + } + } + return decoded; +} + +std::string UrlUtils::PercentEncode(std::string_view value, bool preserve_slash) { + std::string encoded; + encoded.reserve(value.size()); + for (char c : value) { + if (IsRfc3986UnreservedChar(c) || (preserve_slash && c == '/')) { + encoded.push_back(c); + } else { + AppendPercentEncoded(c, &encoded); + } + } + return encoded; +} + +Result UrlUtils::PercentDecode(std::string_view value) { + std::string decoded; + decoded.reserve(value.size()); + for (size_t i = 0; i < value.size(); i++) { + char c = value[i]; + if (c == '%') { + if (i + 2 >= value.size()) { + return Status::IOError("invalid percent encoding in URL component"); + } + int32_t high = HexValue(value[i + 1]); + int32_t low = HexValue(value[i + 2]); + if (high < 0 || low < 0) { + return Status::IOError("invalid percent encoding in URL component"); + } + decoded.push_back(static_cast((high << 4) | low)); + i += 2; + } else { + decoded.push_back(c); + } + } + return decoded; +} + +} // namespace paimon diff --git a/src/paimon/common/utils/url_utils.h b/src/paimon/common/utils/url_utils.h new file mode 100644 index 00000000..a045dc06 --- /dev/null +++ b/src/paimon/common/utils/url_utils.h @@ -0,0 +1,54 @@ +/* + * Copyright 2026-present Alibaba Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include + +#include "paimon/result.h" +#include "paimon/visibility.h" + +namespace paimon { + +/// URL encoding and decoding shared by the REST catalog and the object store file +/// systems. +class PAIMON_EXPORT UrlUtils { + public: + UrlUtils() = delete; + ~UrlUtils() = delete; + + /// URL-encode a string in the `application/x-www-form-urlencoded` flavor (UTF-8): + /// alphanumeric characters and ".", "-", "*", "_" are kept, a space becomes "+", all + /// other bytes become percent-encoded "%XX". This is what the REST catalog server + /// expects; unlike RFC 3986 it also escapes "~". + static std::string EncodeString(const std::string& input); + + /// Decodes `EncodeString` output ('+' back to space, "%XX" to the byte); malformed + /// escape sequences are kept as-is instead of failing. + static std::string DecodeString(const std::string& input); + + /// Percent-encode a URL component with RFC 3986 rules: alphanumeric characters and + /// "-", ".", "_", "~" are kept, every other byte (including a space) becomes "%XX". + /// With `preserve_slash`, '/' is kept as-is so an object key keeps its path shape. + static std::string PercentEncode(std::string_view value, bool preserve_slash = false); + + /// Strict inverse of `PercentEncode`: "%XX" becomes the byte, '+' is kept as-is, and + /// a '%' not followed by two hex digits is an error. + static Result PercentDecode(std::string_view value); +}; + +} // namespace paimon diff --git a/src/paimon/common/utils/url_utils_test.cpp b/src/paimon/common/utils/url_utils_test.cpp new file mode 100644 index 00000000..ebdc2bdc --- /dev/null +++ b/src/paimon/common/utils/url_utils_test.cpp @@ -0,0 +1,84 @@ +/* + * Copyright 2026-present Alibaba Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "paimon/common/utils/url_utils.h" + +#include "gtest/gtest.h" +#include "paimon/status.h" +#include "paimon/testing/utils/testharness.h" + +namespace paimon::test { + +TEST(UrlUtilsTest, EncodeString) { + ASSERT_EQ("abcDEF012.-*_", UrlUtils::EncodeString("abcDEF012.-*_")); + ASSERT_EQ("a+b", UrlUtils::EncodeString("a b")); + ASSERT_EQ("a%2Fb", UrlUtils::EncodeString("a/b")); + ASSERT_EQ("a%3Db%26c", UrlUtils::EncodeString("a=b&c")); + // '~' is the character distinguishing the two encoding flavors: EncodeString + // (form-urlencoded) escapes it while PercentEncode (RFC 3986) keeps it. + ASSERT_EQ("%7E", UrlUtils::EncodeString("~")); + ASSERT_EQ("a%2Bb", UrlUtils::EncodeString("a+b")); + ASSERT_EQ("%E4%B8%AD", UrlUtils::EncodeString("中")); +} + +TEST(UrlUtilsTest, DecodeString) { + ASSERT_EQ("a/b", UrlUtils::DecodeString("a%2Fb")); + ASSERT_EQ("a b", UrlUtils::DecodeString("a+b")); + ASSERT_EQ("中", UrlUtils::DecodeString("%E4%B8%AD")); + ASSERT_EQ("a/", UrlUtils::DecodeString("a%2F")); + // Malformed escapes are kept as-is. + ASSERT_EQ("a%", UrlUtils::DecodeString("a%")); + ASSERT_EQ("%2", UrlUtils::DecodeString("%2")); + ASSERT_EQ("%ZZ", UrlUtils::DecodeString("%ZZ")); + ASSERT_EQ("100%", UrlUtils::DecodeString("100%")); +} + +TEST(UrlUtilsTest, PercentEncode) { + ASSERT_EQ("abcDEF012-._~", UrlUtils::PercentEncode("abcDEF012-._~")); + ASSERT_EQ("a%20b", UrlUtils::PercentEncode("a b")); + ASSERT_EQ("a%2Ab", UrlUtils::PercentEncode("a*b")); + ASSERT_EQ("a%2Fb", UrlUtils::PercentEncode("a/b")); + ASSERT_EQ("a/b%3Dc", UrlUtils::PercentEncode("a/b=c", /*preserve_slash=*/true)); + // a literal '%' is encoded whether or not '/' is kept, so an input already reading + // "%2F" cannot come out as a slash + ASSERT_EQ("a%252Fb", UrlUtils::PercentEncode("a%2Fb", /*preserve_slash=*/true)); + ASSERT_EQ("%E4%B8%AD", UrlUtils::PercentEncode("中")); +} + +TEST(UrlUtilsTest, PercentDecode) { + ASSERT_OK_AND_ASSIGN(std::string decoded, UrlUtils::PercentDecode("a%2Fb%20c")); + ASSERT_EQ("a/b c", decoded); + // '+' is form encoding only; percent decoding keeps it. + ASSERT_OK_AND_ASSIGN(decoded, UrlUtils::PercentDecode("a+b")); + ASSERT_EQ("a+b", decoded); + ASSERT_OK_AND_ASSIGN(decoded, UrlUtils::PercentDecode("%e4%b8%ad")); + ASSERT_EQ("中", decoded); + // Malformed escapes are an error. + ASSERT_NOK(UrlUtils::PercentDecode("a%").status()); + ASSERT_NOK(UrlUtils::PercentDecode("%2").status()); + ASSERT_NOK(UrlUtils::PercentDecode("%ZZ").status()); + ASSERT_NOK(UrlUtils::PercentDecode("100%").status()); +} + +TEST(UrlUtilsTest, RoundTrip) { + const std::string input = "db 1/table$branch_b1=中%"; + ASSERT_EQ(input, UrlUtils::DecodeString(UrlUtils::EncodeString(input))); + ASSERT_OK_AND_ASSIGN(std::string decoded, + UrlUtils::PercentDecode(UrlUtils::PercentEncode(input))); + ASSERT_EQ(input, decoded); +} + +} // namespace paimon::test diff --git a/src/paimon/core/catalog/catalog.cpp b/src/paimon/core/catalog/catalog.cpp index ab26ce0c..97f13afa 100644 --- a/src/paimon/core/catalog/catalog.cpp +++ b/src/paimon/core/catalog/catalog.cpp @@ -20,8 +20,13 @@ #include +#include "paimon/catalog_options.h" +#include "paimon/common/utils/string_utils.h" #include "paimon/core/catalog/file_system_catalog.h" #include "paimon/core/core_options.h" +#ifdef PAIMON_ENABLE_REST +#include "paimon/rest/rest_catalog.h" +#endif namespace paimon { @@ -33,6 +38,24 @@ const char Catalog::DB_LOCATION_PROP[] = "location"; Result> Catalog::Create(const std::string& root_path, const std::map& options, const std::shared_ptr& file_system) { + std::string metastore = "filesystem"; + auto metastore_iter = options.find(CatalogOptions::METASTORE); + if (metastore_iter != options.end()) { + // Matched leniently in lower case; the Java catalog factory looks the metastore + // up by its exact identifier, so only the exact spelling is portable. + metastore = StringUtils::ToLowerCase(metastore_iter->second); + } + if (metastore == "rest") { +#ifdef PAIMON_ENABLE_REST + return RestCatalog::Create(root_path, options, file_system); +#else + return Status::NotImplemented( + "the rest catalog requires building paimon with PAIMON_ENABLE_REST=ON"); +#endif + } + if (metastore != "filesystem") { + return Status::Invalid("unsupported metastore: ", metastore); + } PAIMON_ASSIGN_OR_RAISE(CoreOptions core_options, CoreOptions::FromMap(options, file_system)); return std::make_unique(core_options.GetFileSystem(), root_path, options); } diff --git a/src/paimon/core/catalog/catalog_utils.cpp b/src/paimon/core/catalog/catalog_utils.cpp new file mode 100644 index 00000000..7a6768ab --- /dev/null +++ b/src/paimon/core/catalog/catalog_utils.cpp @@ -0,0 +1,71 @@ +/* + * Copyright 2026-present Alibaba Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "paimon/core/catalog/catalog_utils.h" + +#include + +#include "fmt/format.h" +#include "paimon/catalog/catalog.h" +#include "paimon/result.h" + +namespace paimon { + +namespace { + +Status SystemTableError(const Identifier& identifier, const std::string& action) { + return Status::Invalid(fmt::format("Cannot '{}' for system table '{}', please use data table.", + action, identifier.ToString())); +} + +} // namespace + +bool CatalogUtils::IsSystemDatabase(const std::string& db_name) { + return db_name == Catalog::SYSTEM_DATABASE_NAME; +} + +Status CatalogUtils::CheckNotSystemDatabase(const std::string& db_name, const std::string& action) { + if (IsSystemDatabase(db_name)) { + return Status::Invalid( + fmt::format("Cannot '{}' for system database '{}'.", action, db_name)); + } + return Status::OK(); +} + +Status CatalogUtils::CheckNotSystemTable(const Identifier& identifier, const std::string& action) { + // The system database is checked first so that an identifier of "sys" is rejected + // without being parsed as a table name. + if (IsSystemDatabase(identifier.GetDatabaseName())) { + return SystemTableError(identifier, action); + } + PAIMON_ASSIGN_OR_RAISE(bool is_system_table, identifier.IsSystemTable()); + if (is_system_table) { + return SystemTableError(identifier, action); + } + return Status::OK(); +} + +Status CatalogUtils::CheckNotBranch(const Identifier& identifier, const std::string& action) { + PAIMON_ASSIGN_OR_RAISE(std::optional branch, identifier.GetBranchName()); + if (branch) { + return Status::Invalid(fmt::format( + "Cannot '{}' for branch table '{}', please modify the table with the default branch.", + action, identifier.ToString())); + } + return Status::OK(); +} + +} // namespace paimon diff --git a/src/paimon/core/catalog/catalog_utils.h b/src/paimon/core/catalog/catalog_utils.h new file mode 100644 index 00000000..b9a54ac1 --- /dev/null +++ b/src/paimon/core/catalog/catalog_utils.h @@ -0,0 +1,46 @@ +/* + * Copyright 2026-present Alibaba Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include + +#include "paimon/catalog/identifier.h" +#include "paimon/status.h" + +namespace paimon { + +/// Checks shared by the catalog implementations. Every check takes an `action` naming +/// the rejected operation in the error message, e.g. "dropTable". +class CatalogUtils { + public: + CatalogUtils() = delete; + ~CatalogUtils() = delete; + + /// Returns whether `db_name` is the reserved system database "sys". + static bool IsSystemDatabase(const std::string& db_name); + + /// Fails when `db_name` is the system database. + static Status CheckNotSystemDatabase(const std::string& db_name, const std::string& action); + + /// Fails when `identifier` denotes a system table or any table of the system database. + static Status CheckNotSystemTable(const Identifier& identifier, const std::string& action); + + /// Fails when `identifier` carries a "$branch_" suffix. + static Status CheckNotBranch(const Identifier& identifier, const std::string& action); +}; + +} // namespace paimon diff --git a/src/paimon/core/catalog/file_system_catalog.cpp b/src/paimon/core/catalog/file_system_catalog.cpp index 292af2e8..a036c080 100644 --- a/src/paimon/core/catalog/file_system_catalog.cpp +++ b/src/paimon/core/catalog/file_system_catalog.cpp @@ -31,6 +31,7 @@ #include "paimon/common/utils/arrow/status_utils.h" #include "paimon/common/utils/path_util.h" #include "paimon/common/utils/string_utils.h" +#include "paimon/core/catalog/catalog_utils.h" #include "paimon/core/core_options.h" #include "paimon/core/snapshot.h" #include "paimon/core/table/system/global_system_tables.h" @@ -60,10 +61,7 @@ FileSystemCatalog::FileSystemCatalog(const std::shared_ptr& fs, Status FileSystemCatalog::CreateDatabase(const std::string& db_name, const std::map& options, bool ignore_if_exists) { - if (IsSystemDatabase(db_name)) { - return Status::Invalid( - fmt::format("Cannot create database for system database {}.", db_name)); - } + PAIMON_RETURN_NOT_OK(CatalogUtils::CheckNotSystemDatabase(db_name, "createDatabase")); PAIMON_ASSIGN_OR_RAISE(bool exist, DatabaseExists(db_name)); if (exist) { if (ignore_if_exists) { @@ -94,7 +92,7 @@ Status FileSystemCatalog::CreateDatabaseImpl(const std::string& db_name, } Result FileSystemCatalog::DatabaseExists(const std::string& db_name) const { - if (IsSystemDatabase(db_name)) { + if (CatalogUtils::IsSystemDatabase(db_name)) { return true; } return fs_->Exists(NewDatabasePath(warehouse_, db_name)); @@ -102,7 +100,7 @@ Result FileSystemCatalog::DatabaseExists(const std::string& db_name) const Result FileSystemCatalog::TableExists(const Identifier& identifier) const { // Handle sys database global tables - if (IsSystemDatabase(identifier.GetDatabaseName())) { + if (CatalogUtils::IsSystemDatabase(identifier.GetDatabaseName())) { return GlobalSystemTableLoader::IsSupported(identifier.GetTableName(), catalog_options_); } PAIMON_ASSIGN_OR_RAISE(bool is_system_table, identifier.IsSystemTable()); @@ -136,12 +134,8 @@ Status FileSystemCatalog::CreateTable(const Identifier& identifier, ArrowSchema* const std::vector& primary_keys, const std::map& options, bool ignore_if_exists) { - PAIMON_ASSIGN_OR_RAISE(bool is_system_table, IsSystemTable(identifier)); - if (is_system_table) { - return Status::Invalid( - fmt::format("Cannot create table for system table {}, please use data table.", - identifier.ToString())); - } + PAIMON_RETURN_NOT_OK(CatalogUtils::CheckNotBranch(identifier, "createTable")); + PAIMON_RETURN_NOT_OK(CatalogUtils::CheckNotSystemTable(identifier, "createTable")); PAIMON_ASSIGN_OR_RAISE(bool db_exist, DatabaseExists(identifier.GetDatabaseName())); if (!db_exist) { return Status::Invalid( @@ -198,16 +192,12 @@ const std::map& FileSystemCatalog::GetOptions() const return catalog_options_; } -bool FileSystemCatalog::IsSystemDatabase(const std::string& db_name) { - return db_name == SYSTEM_DATABASE_NAME; -} - Result FileSystemCatalog::IsSpecifiedSystemTable(const Identifier& identifier) { return identifier.IsSystemTable(); } Result FileSystemCatalog::IsSystemTable(const Identifier& identifier) { - if (IsSystemDatabase(identifier.GetDatabaseName())) { + if (CatalogUtils::IsSystemDatabase(identifier.GetDatabaseName())) { return true; } return IsSpecifiedSystemTable(identifier); @@ -241,7 +231,7 @@ Result> FileSystemCatalog::ListDatabases() const { } Result> FileSystemCatalog::ListTables(const std::string& db_name) const { - if (IsSystemDatabase(db_name)) { + if (CatalogUtils::IsSystemDatabase(db_name)) { return GlobalSystemTableLoader::GetSupportedTableNames(catalog_options_); } std::string database_path = NewDatabasePath(warehouse_, db_name); @@ -276,7 +266,7 @@ Result FileSystemCatalog::TableExistsInFileSystem(const std::string& table Result> FileSystemCatalog::LoadTableSchema( const Identifier& identifier) const { // Handle sys database global tables - if (IsSystemDatabase(identifier.GetDatabaseName())) { + if (CatalogUtils::IsSystemDatabase(identifier.GetDatabaseName())) { PAIMON_ASSIGN_OR_RAISE(bool supported, GlobalSystemTableLoader::IsSupported( identifier.GetTableName(), catalog_options_)); if (!supported) { @@ -341,10 +331,7 @@ Result> FileSystemCatalog::GetTable(const Identifier& ide Status FileSystemCatalog::DropDatabase(const std::string& name, bool ignore_if_not_exists, bool cascade) { - if (IsSystemDatabase(name)) { - return Status::Invalid(fmt::format("Cannot drop system database {}.", name)); - } - + PAIMON_RETURN_NOT_OK(CatalogUtils::CheckNotSystemDatabase(name, "dropDatabase")); PAIMON_ASSIGN_OR_RAISE(bool exist, DatabaseExists(name)); if (!exist) { if (ignore_if_not_exists) { @@ -427,10 +414,10 @@ Status FileSystemCatalog::DropTableImpl(const Identifier& identifier, } Status FileSystemCatalog::DropTable(const Identifier& identifier, bool ignore_if_not_exists) { - PAIMON_ASSIGN_OR_RAISE(bool is_system_table, IsSystemTable(identifier)); - if (is_system_table) { - return Status::Invalid(fmt::format("Cannot drop system table {}.", identifier.ToString())); - } + // A branch identifier resolves to the main table directory, so without this check + // dropping "t$branch_b" would delete the whole table "t". + PAIMON_RETURN_NOT_OK(CatalogUtils::CheckNotBranch(identifier, "dropTable")); + PAIMON_RETURN_NOT_OK(CatalogUtils::CheckNotSystemTable(identifier, "dropTable")); PAIMON_ASSIGN_OR_RAISE(std::string table_path, GetTableLocation(identifier)); PAIMON_ASSIGN_OR_RAISE(bool exist, fs_->Exists(table_path)); if (!exist) { @@ -490,12 +477,10 @@ Status FileSystemCatalog::DropTable(const Identifier& identifier, bool ignore_if Status FileSystemCatalog::RenameTable(const Identifier& from_table, const Identifier& to_table, bool ignore_if_not_exists) { - PAIMON_ASSIGN_OR_RAISE(bool is_from_system_table, IsSystemTable(from_table)); - PAIMON_ASSIGN_OR_RAISE(bool is_to_system_table, IsSystemTable(to_table)); - if (is_from_system_table || is_to_system_table) { - return Status::Invalid(fmt::format("Cannot rename system table {} or {}.", - from_table.ToString(), to_table.ToString())); - } + PAIMON_RETURN_NOT_OK(CatalogUtils::CheckNotBranch(from_table, "renameTable")); + PAIMON_RETURN_NOT_OK(CatalogUtils::CheckNotBranch(to_table, "renameTable")); + PAIMON_RETURN_NOT_OK(CatalogUtils::CheckNotSystemTable(from_table, "renameTable")); + PAIMON_RETURN_NOT_OK(CatalogUtils::CheckNotSystemTable(to_table, "renameTable")); if (from_table.GetDatabaseName() != to_table.GetDatabaseName()) { return Status::Invalid( @@ -523,24 +508,6 @@ Status FileSystemCatalog::RenameTable(const Identifier& from_table, const Identi return Status::OK(); } -namespace { -SnapshotInfo::CommitKind ConvertCommitKind(Snapshot::CommitKind internal) { - if (internal == Snapshot::CommitKind::Append()) { - return SnapshotInfo::CommitKind::APPEND; - } - if (internal == Snapshot::CommitKind::Compact()) { - return SnapshotInfo::CommitKind::COMPACT; - } - if (internal == Snapshot::CommitKind::Overwrite()) { - return SnapshotInfo::CommitKind::OVERWRITE; - } - if (internal == Snapshot::CommitKind::Analyze()) { - return SnapshotInfo::CommitKind::ANALYZE; - } - return SnapshotInfo::CommitKind::UNKNOWN; -} -} // namespace - Result> FileSystemCatalog::ListSnapshots( const Identifier& identifier, const std::string& branch) const { PAIMON_ASSIGN_OR_RAISE(bool exists, TableExists(identifier)); @@ -555,20 +522,9 @@ Result> FileSystemCatalog::ListSnapshots( std::vector result; result.reserve(snapshots.size()); - for (const auto& snap : snapshots) { - SnapshotInfo info; - info.snapshot_id = snap.Id(); - info.schema_id = snap.SchemaId(); - info.commit_user = snap.CommitUser(); - info.commit_kind = ConvertCommitKind(snap.GetCommitKind()); - info.time_millis = snap.TimeMillis(); - info.total_record_count = snap.TotalRecordCount(); - info.delta_record_count = snap.DeltaRecordCount(); - info.watermark = snap.Watermark(); - result.push_back(std::move(info)); + result.push_back(snap.ToSnapshotInfo()); } - return result; } diff --git a/src/paimon/core/catalog/file_system_catalog.h b/src/paimon/core/catalog/file_system_catalog.h index 3a464ae8..3925aff8 100644 --- a/src/paimon/core/catalog/file_system_catalog.h +++ b/src/paimon/core/catalog/file_system_catalog.h @@ -73,7 +73,6 @@ class FileSystemCatalog : public Catalog { static std::string NewDatabasePath(const std::string& warehouse, const std::string& db_name); static Result NewDataTablePath(const std::string& warehouse, const Identifier& identifier); - static bool IsSystemDatabase(const std::string& db_name); static Result IsSpecifiedSystemTable(const Identifier& identifier); static Result IsSystemTable(const Identifier& identifier); Result>> TableSchemaExists( diff --git a/src/paimon/core/catalog/file_system_catalog_test.cpp b/src/paimon/core/catalog/file_system_catalog_test.cpp index 2a07b262..9f6a568f 100644 --- a/src/paimon/core/catalog/file_system_catalog_test.cpp +++ b/src/paimon/core/catalog/file_system_catalog_test.cpp @@ -93,7 +93,7 @@ TEST(FileSystemCatalogTest, TestCreateSystemDatabaseAndTable) { FileSystemCatalog catalog(core_options.GetFileSystem(), dir->Str(), options); ASSERT_NOK_WITH_MSG(catalog.CreateDatabase(Catalog::SYSTEM_DATABASE_NAME, options, /*ignore_if_exists=*/true), - "Cannot create database for system database"); + "Cannot 'createDatabase' for system database 'sys'."); } /// Do not support create system table. { @@ -116,7 +116,7 @@ TEST(FileSystemCatalogTest, TestCreateSystemDatabaseAndTable) { ASSERT_TRUE(arrow::ExportSchema(typed_schema, &schema).ok()); ASSERT_NOK_WITH_MSG( catalog.CreateTable(Identifier("db1", "ta$ble"), &schema, {"f1"}, {}, options, false), - "Cannot create table for system table"); + "Cannot 'createTable' for system table"); ArrowSchemaRelease(&schema); } } @@ -208,11 +208,51 @@ TEST(FileSystemCatalogTest, TestOptionsSystemTableCatalog) { ASSERT_TRUE(arrow::ExportSchema(*typed_schema, &system_create_schema).ok()); ASSERT_NOK_WITH_MSG( catalog.CreateTable(options_identifier, &system_create_schema, {}, {}, options, false), - "Cannot create table for system table"); + "Cannot 'createTable' for system table"); ArrowSchemaRelease(&system_create_schema); - ASSERT_NOK_WITH_MSG(catalog.DropTable(options_identifier, false), "Cannot drop system table"); + ASSERT_NOK_WITH_MSG(catalog.DropTable(options_identifier, false), + "Cannot 'dropTable' for system table"); ASSERT_NOK_WITH_MSG(catalog.RenameTable(options_identifier, Identifier("db1", "tbl2"), false), - "Cannot rename system table"); + "Cannot 'renameTable' for system table"); +} + +TEST(FileSystemCatalogTest, TestBranchIdentifierRejectedForTableOperations) { + std::map options; + options[Options::FILE_SYSTEM] = "local"; + options[Options::FILE_FORMAT] = "orc"; + ASSERT_OK_AND_ASSIGN(auto core_options, CoreOptions::FromMap(options)); + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + FileSystemCatalog catalog(core_options.GetFileSystem(), dir->Str(), options); + ASSERT_OK(catalog.CreateDatabase("db1", options, /*ignore_if_exists=*/true)); + + auto typed_schema = + arrow::schema({arrow::field("f0", arrow::int32()), arrow::field("f1", arrow::utf8())}); + ::ArrowSchema schema; + ASSERT_TRUE(arrow::ExportSchema(*typed_schema, &schema).ok()); + ASSERT_OK(catalog.CreateTable(Identifier("db1", "tbl1"), &schema, + /*partition_keys=*/{}, /*primary_keys=*/{}, options, + /*ignore_if_exists=*/false)); + ArrowSchemaRelease(&schema); + + // A branch identifier resolves to the main table directory, so create, drop and rename + // must reject it: dropping "tbl1$branch_b1" would otherwise delete the whole "tbl1". + Identifier branch_identifier("db1", "tbl1$branch_b1"); + ::ArrowSchema branch_create_schema; + ASSERT_TRUE(arrow::ExportSchema(*typed_schema, &branch_create_schema).ok()); + ASSERT_NOK_WITH_MSG( + catalog.CreateTable(branch_identifier, &branch_create_schema, {}, {}, options, false), + "Cannot 'createTable' for branch table"); + ArrowSchemaRelease(&branch_create_schema); + ASSERT_NOK_WITH_MSG(catalog.DropTable(branch_identifier, false), + "Cannot 'dropTable' for branch table"); + ASSERT_NOK_WITH_MSG(catalog.RenameTable(branch_identifier, Identifier("db1", "tbl2"), false), + "Cannot 'renameTable' for branch table"); + ASSERT_NOK_WITH_MSG(catalog.RenameTable(Identifier("db1", "tbl1"), branch_identifier, false), + "Cannot 'renameTable' for branch table"); + // the table itself is untouched by the rejected operations + ASSERT_OK_AND_ASSIGN(bool exists, catalog.TableExists(Identifier("db1", "tbl1"))); + ASSERT_TRUE(exists); } TEST(FileSystemCatalogTest, TestAuditLogAndBinlogSystemTableCatalog) { @@ -279,11 +319,12 @@ TEST(FileSystemCatalogTest, TestAuditLogAndBinlogSystemTableCatalog) { ASSERT_TRUE(arrow::ExportSchema(*typed_schema, &system_create_schema).ok()); ASSERT_NOK_WITH_MSG( catalog.CreateTable(audit_log_identifier, &system_create_schema, {}, {}, options, false), - "Cannot create table for system table"); + "Cannot 'createTable' for system table"); ArrowSchemaRelease(&system_create_schema); - ASSERT_NOK_WITH_MSG(catalog.DropTable(binlog_identifier, false), "Cannot drop system table"); + ASSERT_NOK_WITH_MSG(catalog.DropTable(binlog_identifier, false), + "Cannot 'dropTable' for system table"); ASSERT_NOK_WITH_MSG(catalog.RenameTable(audit_log_identifier, Identifier("db1", "tbl2"), false), - "Cannot rename system table"); + "Cannot 'renameTable' for system table"); } TEST(FileSystemCatalogTest, TestMetadataSystemTableCatalog) { @@ -422,11 +463,12 @@ TEST(FileSystemCatalogTest, TestMetadataSystemTableCatalog) { ASSERT_TRUE(arrow::ExportSchema(*typed_schema, &system_create_schema).ok()); ASSERT_NOK_WITH_MSG( catalog.CreateTable(snapshots_identifier, &system_create_schema, {}, {}, options, false), - "Cannot create table for system table"); + "Cannot 'createTable' for system table"); ArrowSchemaRelease(&system_create_schema); - ASSERT_NOK_WITH_MSG(catalog.DropTable(snapshots_identifier, false), "Cannot drop system table"); + ASSERT_NOK_WITH_MSG(catalog.DropTable(snapshots_identifier, false), + "Cannot 'dropTable' for system table"); ASSERT_NOK_WITH_MSG(catalog.RenameTable(snapshots_identifier, Identifier("db1", "tbl2"), false), - "Cannot rename system table"); + "Cannot 'renameTable' for system table"); } TEST(FileSystemCatalogTest, TestCreateTableWithBlob) { @@ -748,7 +790,7 @@ TEST(FileSystemCatalogTest, TestDropDatabase) { ASSERT_NOK_WITH_MSG(catalog.DropDatabase(Catalog::SYSTEM_DATABASE_NAME, /*ignore_if_not_exists=*/false, /*cascade=*/false), - "Cannot drop system database sys."); + "Cannot 'dropDatabase' for system database 'sys'."); ArrowSchemaRelease(&schema); } @@ -787,10 +829,10 @@ TEST(FileSystemCatalogTest, TestDropTable) { ASSERT_FALSE(exist); /// Test 4: Drop system table. - ASSERT_NOK_WITH_MSG( - catalog.DropTable(Identifier("test_db", "tbl$system"), - /*ignore_if_not_exists=*/false), - "Cannot drop system table Identifier{database='test_db', table='tbl$system'}."); + ASSERT_NOK_WITH_MSG(catalog.DropTable(Identifier("test_db", "tbl$system"), + /*ignore_if_not_exists=*/false), + "Cannot 'dropTable' for system table 'Identifier{database='test_db', " + "table='tbl$system'}', please use data table."); ArrowSchemaRelease(&schema); } @@ -856,7 +898,7 @@ TEST(FileSystemCatalogTest, TestRenameTable) { ASSERT_NOK_WITH_MSG(catalog.RenameTable(Identifier("test_db", "tbl$system"), Identifier("test_db", "new_system_tbl"), /*ignore_if_not_exists=*/false), - "Cannot rename system table"); + "Cannot 'renameTable' for system table"); ArrowSchemaRelease(&schema1); ArrowSchemaRelease(&schema2); diff --git a/src/paimon/core/schema/table_schema.cpp b/src/paimon/core/schema/table_schema.cpp index 2beff595..e5139478 100644 --- a/src/paimon/core/schema/table_schema.cpp +++ b/src/paimon/core/schema/table_schema.cpp @@ -39,6 +39,7 @@ #include "paimon/core/schema/arrow_schema_validator.h" #include "paimon/defs.h" #include "paimon/status.h" +#include "paimon/utils/special_field_ids.h" #include "rapidjson/allocators.h" #include "rapidjson/document.h" #include "rapidjson/rapidjson.h" @@ -240,6 +241,67 @@ Result> TableSchema::CreateFromJson(const std::stri table_schema.options_, table_schema.comment_, table_schema.time_millis_); } +namespace { + +Status CollectFieldIds(const rapidjson::Value& fields, std::set* seen_ids, + int32_t* max_id); + +Status CollectTypeFieldIds(const rapidjson::Value& type, std::set* seen_ids, + int32_t* max_id) { + if (!type.IsObject()) { + return Status::OK(); + } + // ROW nests its fields under "fields", ARRAY/MULTISET under "element", MAP under "key"/"value". + if (type.HasMember("fields") && type["fields"].IsArray()) { + PAIMON_RETURN_NOT_OK(CollectFieldIds(type["fields"], seen_ids, max_id)); + } + if (type.HasMember("element")) { + PAIMON_RETURN_NOT_OK(CollectTypeFieldIds(type["element"], seen_ids, max_id)); + } + if (type.HasMember("key")) { + PAIMON_RETURN_NOT_OK(CollectTypeFieldIds(type["key"], seen_ids, max_id)); + } + if (type.HasMember("value")) { + PAIMON_RETURN_NOT_OK(CollectTypeFieldIds(type["value"], seen_ids, max_id)); + } + return Status::OK(); +} + +Status CollectFieldIds(const rapidjson::Value& fields, std::set* seen_ids, + int32_t* max_id) { + for (const auto& field : fields.GetArray()) { + if (!field.IsObject()) { + return Status::Invalid("Broken schema, a field must be an object."); + } + if (!field.HasMember("id") || !field["id"].IsInt()) { + return Status::Invalid("Broken schema, a field misses an integer id."); + } + int32_t id = field["id"].GetInt(); + if (!seen_ids->insert(id).second) { + return Status::Invalid(fmt::format("Broken schema, field id {} is duplicated.", id)); + } + if (id < SpecialFieldIds::SYSTEM_FIELD_ID_START) { + *max_id = std::max(*max_id, id); + } + if (field.HasMember("type")) { + PAIMON_RETURN_NOT_OK(CollectTypeFieldIds(field["type"], seen_ids, max_id)); + } + } + return Status::OK(); +} + +} // namespace + +Result TableSchema::ComputeHighestFieldId(const rapidjson::Value& fields) { + if (!fields.IsArray()) { + return Status::Invalid("Broken schema, 'fields' must be an array."); + } + int32_t max_id = -1; + std::set seen_ids; + PAIMON_RETURN_NOT_OK(CollectFieldIds(fields, &seen_ids, &max_id)); + return max_id; +} + Result> TableSchema::InitSchema( int64_t schema_id, const std::vector& fields, int32_t highest_field_id, const std::vector& partition_keys, const std::vector& primary_keys, diff --git a/src/paimon/core/schema/table_schema.h b/src/paimon/core/schema/table_schema.h index bc0c94e4..eab109cf 100644 --- a/src/paimon/core/schema/table_schema.h +++ b/src/paimon/core/schema/table_schema.h @@ -54,6 +54,13 @@ class TableSchema : public DataSchema, public Jsonizable { static Result> CreateFromJson(const std::string& json_str); + /// Computes the highest non-system field id across all nesting levels of a "fields" JSON + /// array, or -1 when there is none; ids at or above `SpecialFieldIds::SYSTEM_FIELD_ID_START` + /// are excluded. A non-object field or a missing, non-integer or duplicated field id fails + /// the computation. Used when the schema source (e.g. a rest catalog server) does not + /// report the "highestFieldId" itself. + static Result ComputeHighestFieldId(const rapidjson::Value& fields); + rapidjson::Value ToJson(rapidjson::Document::AllocatorType* allocator) const noexcept(false) override; diff --git a/src/paimon/core/schema/table_schema_test.cpp b/src/paimon/core/schema/table_schema_test.cpp index 988f17f9..fd0ac422 100644 --- a/src/paimon/core/schema/table_schema_test.cpp +++ b/src/paimon/core/schema/table_schema_test.cpp @@ -30,6 +30,8 @@ #include "paimon/fs/local/local_file_system.h" #include "paimon/status.h" #include "paimon/testing/utils/testharness.h" +#include "paimon/utils/special_field_ids.h" +#include "rapidjson/document.h" namespace paimon::test { @@ -151,6 +153,43 @@ TEST_F(TableSchemaTest, TestGetFieldTypeForVariant) { ASSERT_EQ(variant_type, FieldType::VARIANT); } +TEST_F(TableSchemaTest, TestComputeHighestFieldId) { + auto compute = [](const std::string& fields_json) -> Result { + rapidjson::Document doc; + doc.Parse(fields_json.c_str()); + EXPECT_FALSE(doc.HasParseError()) << fields_json; + return TableSchema::ComputeHighestFieldId(doc); + }; + // ids inside nested rows are included + ASSERT_OK_AND_ASSIGN(int32_t highest, compute(R"([ + {"id": 0, "name": "f0", "type": "INT"}, + {"id": 1, "name": "s", "type": {"type": "ROW", + "fields": [{"id": 5, "name": "inner", "type": "INT"}]}} + ])")); + ASSERT_EQ(5, highest); + // system field ids are reserved and excluded from the highest field id + ASSERT_OK_AND_ASSIGN(highest, compute(R"([{"id": 3, "name": "f0", "type": "INT"}, + {"id": )" + + std::to_string(SpecialFieldIds::SEQUENCE_NUMBER) + + R"(, "name": "_SEQUENCE_NUMBER", "type": "BIGINT"}])")); + ASSERT_EQ(3, highest); + ASSERT_OK_AND_ASSIGN(highest, compute("[]")); + ASSERT_EQ(-1, highest); + // broken schemas fail instead of being silently skipped + ASSERT_NOK_WITH_MSG(compute(R"([{"id": 0, "name": "a", "type": "INT"}, + {"id": 0, "name": "b", "type": "INT"}])") + .status(), + "duplicated"); + ASSERT_NOK_WITH_MSG(compute(R"([{"name": "a", "type": "INT"}])").status(), "integer id"); + ASSERT_NOK_WITH_MSG(compute(R"([{"id": "0", "name": "a", "type": "INT"}])").status(), + "integer id"); + ASSERT_NOK_WITH_MSG(compute("[1]").status(), "must be an object"); + rapidjson::Document not_an_array; + not_an_array.Parse("{}"); + ASSERT_NOK_WITH_MSG(TableSchema::ComputeHighestFieldId(not_an_array).status(), + "must be an array"); +} + TEST_F(TableSchemaTest, TestInvalidCreate) { // partial fields have field id arrow::FieldVector fields = MakeArrowField(3); diff --git a/src/paimon/core/snapshot.cpp b/src/paimon/core/snapshot.cpp index d3081c91..cd5f933f 100644 --- a/src/paimon/core/snapshot.cpp +++ b/src/paimon/core/snapshot.cpp @@ -315,4 +315,27 @@ Result Snapshot::FromPath(const std::shared_ptr& fs, return snapshot; } +SnapshotInfo Snapshot::ToSnapshotInfo() const { + SnapshotInfo info; + info.snapshot_id = Id(); + info.schema_id = SchemaId(); + info.commit_user = CommitUser(); + if (commit_kind_ == CommitKind::Append()) { + info.commit_kind = SnapshotInfo::CommitKind::APPEND; + } else if (commit_kind_ == CommitKind::Compact()) { + info.commit_kind = SnapshotInfo::CommitKind::COMPACT; + } else if (commit_kind_ == CommitKind::Overwrite()) { + info.commit_kind = SnapshotInfo::CommitKind::OVERWRITE; + } else if (commit_kind_ == CommitKind::Analyze()) { + info.commit_kind = SnapshotInfo::CommitKind::ANALYZE; + } else { + info.commit_kind = SnapshotInfo::CommitKind::UNKNOWN; + } + info.time_millis = TimeMillis(); + info.total_record_count = TotalRecordCount(); + info.delta_record_count = DeltaRecordCount(); + info.watermark = Watermark(); + return info; +} + } // namespace paimon diff --git a/src/paimon/core/snapshot.h b/src/paimon/core/snapshot.h index 1b7f6593..2d2cd061 100644 --- a/src/paimon/core/snapshot.h +++ b/src/paimon/core/snapshot.h @@ -27,6 +27,7 @@ #include "paimon/common/utils/jsonizable.h" #include "paimon/result.h" +#include "paimon/snapshot/snapshot_info.h" #include "paimon/type_fwd.h" #include "rapidjson/allocators.h" #include "rapidjson/document.h" @@ -131,6 +132,8 @@ class Snapshot : public Jsonizable { bool operator==(const Snapshot& other) const; bool TEST_Equal(const Snapshot& other) const; + SnapshotInfo ToSnapshotInfo() const; + public: static constexpr int64_t FIRST_SNAPSHOT_ID = 1; static constexpr int32_t TABLE_STORE_02_VERSION = 1; diff --git a/src/paimon/core/snapshot_test.cpp b/src/paimon/core/snapshot_test.cpp index 606e3200..7562f38d 100644 --- a/src/paimon/core/snapshot_test.cpp +++ b/src/paimon/core/snapshot_test.cpp @@ -329,6 +329,38 @@ TEST_F(SnapshotTest, TestSnapshotInfoCommitKindToString) { ASSERT_EQ("UNKNOWN", SnapshotInfo::CommitKindToString(SnapshotInfo::CommitKind::UNKNOWN)); } +TEST_F(SnapshotTest, TestToSnapshotInfo) { + auto make_snapshot = [](const Snapshot::CommitKind& kind) { + return Snapshot( + /*id=*/10, /*schema_id=*/15, /*base_manifest_list=*/"bml", + /*base_manifest_list_size=*/10, + /*delta_manifest_list=*/"dml", /*delta_manifest_list_size=*/20, + /*changelog_manifest_list=*/std::nullopt, /*changelog_manifest_list_size=*/std::nullopt, + /*index_manifest=*/std::nullopt, /*commit_user=*/"user1", /*commit_identifier=*/20, + kind, /*time_millis=*/1234, /*total_record_count=*/35, /*delta_record_count=*/40, + /*changelog_record_count=*/std::nullopt, /*watermark=*/50, + /*statistics=*/std::nullopt, /*properties=*/std::nullopt, + /*next_row_id=*/std::nullopt); + }; + SnapshotInfo info = make_snapshot(Snapshot::CommitKind::Append()).ToSnapshotInfo(); + ASSERT_EQ(10, info.snapshot_id); + ASSERT_EQ(15, info.schema_id); + ASSERT_EQ("user1", info.commit_user); + ASSERT_EQ(SnapshotInfo::CommitKind::APPEND, info.commit_kind); + ASSERT_EQ(1234, info.time_millis); + ASSERT_EQ(35, info.total_record_count.value()); + ASSERT_EQ(40, info.delta_record_count.value()); + ASSERT_EQ(50, info.watermark.value()); + ASSERT_EQ(SnapshotInfo::CommitKind::COMPACT, + make_snapshot(Snapshot::CommitKind::Compact()).ToSnapshotInfo().commit_kind); + ASSERT_EQ(SnapshotInfo::CommitKind::OVERWRITE, + make_snapshot(Snapshot::CommitKind::Overwrite()).ToSnapshotInfo().commit_kind); + ASSERT_EQ(SnapshotInfo::CommitKind::ANALYZE, + make_snapshot(Snapshot::CommitKind::Analyze()).ToSnapshotInfo().commit_kind); + ASSERT_EQ(SnapshotInfo::CommitKind::UNKNOWN, + make_snapshot(Snapshot::CommitKind::Unknown()).ToSnapshotInfo().commit_kind); +} + TEST_F(SnapshotTest, TestChangelogManifestListSerialization) { // Test with changelog_manifest_list set to a non-null value { diff --git a/src/paimon/core/table/system/global_system_tables.cpp b/src/paimon/core/table/system/global_system_tables.cpp index bdfab0f0..dd1b4e60 100644 --- a/src/paimon/core/table/system/global_system_tables.cpp +++ b/src/paimon/core/table/system/global_system_tables.cpp @@ -20,6 +20,7 @@ #include "paimon/core/table/system/global_system_tables.h" #include +#include #include #include #include @@ -33,6 +34,7 @@ #include "paimon/common/data/generic_row.h" #include "paimon/common/utils/options_utils.h" #include "paimon/common/utils/path_util.h" +#include "paimon/common/utils/sensitive_config_utils.h" #include "paimon/common/utils/string_utils.h" #include "paimon/core/core_options.h" #include "paimon/core/io/data_file_meta.h" @@ -336,9 +338,14 @@ Result> CatalogOptionsSystemTable::ArrowSchema() Result> CatalogOptionsSystemTable::BuildRows() const { PAIMON_ASSIGN_OR_RAISE(std::shared_ptr schema, ArrowSchema()); + // Catalog options may carry credentials (e.g. the REST token), which must never reach + // anyone able to query this table; a secret is masked as a whole, an identifier-like + // value keeps at most a short tail. + const std::map masked_options = + SensitiveConfigUtils::RedactMap(context_.catalog_options); std::vector rows; - rows.reserve(context_.catalog_options.size()); - for (const auto& [key, value] : context_.catalog_options) { + rows.reserve(masked_options.size()); + for (const auto& [key, value] : masked_options) { GenericRow row(schema->num_fields()); row.SetField(0, StringValue(key)); row.SetField(1, StringValue(value)); diff --git a/src/paimon/fs/s3/s3_file_system.cpp b/src/paimon/fs/s3/s3_file_system.cpp index 4d896680..49668b70 100644 --- a/src/paimon/fs/s3/s3_file_system.cpp +++ b/src/paimon/fs/s3/s3_file_system.cpp @@ -45,52 +45,22 @@ #include #include "fmt/format.h" -#include "paimon/common/fs/http_client.h" +#include "paimon/common/utils/http_client.h" #include "paimon/common/utils/options_utils.h" #include "paimon/common/utils/scope_guard.h" #include "paimon/common/utils/string_utils.h" +#include "paimon/common/utils/url_utils.h" #include "paimon/executor.h" namespace paimon::s3 { namespace { -Result PercentEncode(std::string_view value, bool preserve_slash) { - if (value.size() > std::numeric_limits::max()) { - return Status::IOError("S3 URL component is too large to encode"); - } - char* encoded = curl_easy_escape(nullptr, value.data(), static_cast(value.size())); - if (encoded == nullptr) { - return Status::IOError("failed to URL encode S3 component"); - } - ScopeGuard free_encoded([encoded] { curl_free(encoded); }); - std::string result(encoded); - if (preserve_slash) { - result = StringUtils::Replace(result, "%2F", "/"); - } - return result; -} - Result PercentDecode(std::string_view value, const std::string& field) { - for (size_t position = 0; position < value.size(); ++position) { - if (value[position] == '%' && - (position + 2 >= value.size() || - !std::isxdigit(static_cast(value[position + 1])) || - !std::isxdigit(static_cast(value[position + 2])))) { - return Status::IOError(fmt::format("invalid URL encoding in S3 {}", field)); - } - } - if (value.size() > std::numeric_limits::max()) { - return Status::IOError(fmt::format("S3 {} is too large to URL decode", field)); + Result decoded = UrlUtils::PercentDecode(value); + if (!decoded.ok()) { + return Status::IOError(fmt::format("invalid URL encoding in S3 {}", field)); } - int decoded_size = 0; - char* decoded = - curl_easy_unescape(nullptr, value.data(), static_cast(value.size()), &decoded_size); - if (decoded == nullptr) { - return Status::IOError(fmt::format("failed to URL decode S3 {}", field)); - } - ScopeGuard free_decoded([decoded] { curl_free(decoded); }); - std::string result(decoded, decoded_size); - return result; + return decoded; } Result ParseNonNegativeInt64(const std::string& value, const std::string& field) { @@ -707,13 +677,10 @@ class S3ObjectStoreClient : public ObjectStoreClient, int32_t max_keys) const override { std::string query = "list-type=2&delimiter=%2F&encoding-type=url"; if (!path.key.empty()) { - PAIMON_ASSIGN_OR_RAISE(std::string encoded_prefix, PercentEncode(path.key, false)); - query += "&prefix=" + encoded_prefix; + query += "&prefix=" + UrlUtils::PercentEncode(path.key); } if (!continuation_token.empty()) { - PAIMON_ASSIGN_OR_RAISE(std::string encoded_token, - PercentEncode(continuation_token, false)); - query += "&continuation-token=" + encoded_token; + query += "&continuation-token=" + UrlUtils::PercentEncode(continuation_token); } if (max_keys > 0) { query += "&max-keys=" + std::to_string(max_keys); @@ -850,13 +817,11 @@ class S3ObjectStoreClient : public ObjectStoreClient, : endpoint_.scheme != "http" || IsIpAddressAuthority(endpoint_.authority) || !IsVirtualHostableS3Bucket(object.bucket, true)); if (use_path_style) { - PAIMON_ASSIGN_OR_RAISE(std::string encoded_bucket, PercentEncode(object.bucket, false)); - request_path += encoded_bucket + "/"; + request_path += UrlUtils::PercentEncode(object.bucket) + "/"; } else { authority = object.bucket + "." + authority; } - PAIMON_ASSIGN_OR_RAISE(std::string encoded_key, PercentEncode(object.key, true)); - request_path += encoded_key; + request_path += UrlUtils::PercentEncode(object.key, /*preserve_slash=*/true); if (!query.empty()) { request_path += "?" + query; } diff --git a/src/paimon/fs/s3/s3_file_system.h b/src/paimon/fs/s3/s3_file_system.h index 12caf8b7..8c4eebbc 100644 --- a/src/paimon/fs/s3/s3_file_system.h +++ b/src/paimon/fs/s3/s3_file_system.h @@ -23,8 +23,8 @@ #include #include -#include "paimon/common/fs/http_client.h" #include "paimon/common/fs/object_store_file_system.h" +#include "paimon/common/utils/http_client.h" namespace paimon::s3 { diff --git a/src/paimon/rest/mock_rest_server.cpp b/src/paimon/rest/mock_rest_server.cpp new file mode 100644 index 00000000..0bf4ec01 --- /dev/null +++ b/src/paimon/rest/mock_rest_server.cpp @@ -0,0 +1,218 @@ +/* + * Copyright 2026-present Alibaba Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "paimon/rest/mock_rest_server.h" + +#include +#include +#include +#include + +#include +#include +#include +#include + +#include "fmt/format.h" +#include "paimon/common/utils/string_utils.h" +#include "paimon/common/utils/url_utils.h" + +namespace paimon { + +namespace { + +bool ReceiveAll(int32_t fd, std::string* buffer, size_t min_size) { + char chunk[4096]; + while (buffer->size() < min_size) { + ssize_t received = ::recv(fd, chunk, sizeof(chunk), 0); + if (received <= 0) { + return false; + } + buffer->append(chunk, static_cast(received)); + } + return true; +} + +std::map ParseQuery(const std::string& query) { + std::map params; + for (const std::string& pair : StringUtils::Split(query, "&", /*ignore_empty=*/true)) { + size_t eq = pair.find('='); + if (eq == std::string::npos) { + params[UrlUtils::DecodeString(pair)] = ""; + } else { + params[UrlUtils::DecodeString(pair.substr(0, eq))] = + UrlUtils::DecodeString(pair.substr(eq + 1)); + } + } + return params; +} + +} // namespace + +MockRestServer::MockRestServer(Handler handler, int32_t listen_fd, int32_t port) + : handler_(std::move(handler)), listen_fd_(listen_fd), port_(port) { + accept_thread_ = std::thread([this] { AcceptLoop(); }); +} + +Result> MockRestServer::Start(Handler handler) { + int32_t listen_fd = ::socket(AF_INET, SOCK_STREAM, 0); + if (listen_fd < 0) { + return Status::IOError("mock rest server: failed to create socket: ", std::strerror(errno)); + } + int32_t reuse = 1; + ::setsockopt(listen_fd, SOL_SOCKET, SO_REUSEADDR, &reuse, sizeof(reuse)); + struct sockaddr_in address; + std::memset(&address, 0, sizeof(address)); + address.sin_family = AF_INET; + address.sin_addr.s_addr = htonl(INADDR_LOOPBACK); + address.sin_port = 0; + if (::bind(listen_fd, reinterpret_cast(&address), sizeof(address)) < 0) { + ::close(listen_fd); + return Status::IOError("mock rest server: failed to bind: ", std::strerror(errno)); + } + if (::listen(listen_fd, 16) < 0) { + ::close(listen_fd); + return Status::IOError("mock rest server: failed to listen: ", std::strerror(errno)); + } + socklen_t address_len = sizeof(address); + if (::getsockname(listen_fd, reinterpret_cast(&address), &address_len) < 0) { + ::close(listen_fd); + return Status::IOError("mock rest server: failed to get port: ", std::strerror(errno)); + } + int32_t port = ntohs(address.sin_port); + return std::unique_ptr(new MockRestServer(std::move(handler), listen_fd, port)); +} + +MockRestServer::~MockRestServer() { + Stop(); +} + +void MockRestServer::Stop() { + if (stopped_.exchange(true)) { + return; + } + // `shutdown` wakes the blocked `accept`; the fd is closed only after the accept + // thread has joined, so it cannot be recycled by another thread while the accept + // thread might still use it. + ::shutdown(listen_fd_, SHUT_RDWR); + if (accept_thread_.joinable()) { + accept_thread_.join(); + } + ::close(listen_fd_); +} + +std::string MockRestServer::GetBaseUri() const { + return fmt::format("http://127.0.0.1:{}", port_); +} + +void MockRestServer::AcceptLoop() { + while (!stopped_.load()) { + int32_t connection_fd = ::accept(listen_fd_, nullptr, nullptr); + if (connection_fd < 0) { + if (stopped_.load()) { + return; + } + continue; + } + // The connection is handled on the accept thread; the socket timeouts keep a + // wedged peer from blocking `Stop()` indefinitely. + struct timeval timeout; + timeout.tv_sec = 30; + timeout.tv_usec = 0; + ::setsockopt(connection_fd, SOL_SOCKET, SO_RCVTIMEO, &timeout, sizeof(timeout)); + ::setsockopt(connection_fd, SOL_SOCKET, SO_SNDTIMEO, &timeout, sizeof(timeout)); + HandleConnection(connection_fd); + ::close(connection_fd); + } +} + +void MockRestServer::HandleConnection(int32_t connection_fd) { + std::string buffer; + size_t header_end; + while ((header_end = buffer.find("\r\n\r\n")) == std::string::npos) { + if (!ReceiveAll(connection_fd, &buffer, buffer.size() + 1)) { + return; + } + } + + Request request; + std::string header_part = buffer.substr(0, header_end); + std::vector lines = StringUtils::Split(header_part, "\r\n", + /*ignore_empty=*/true); + if (lines.empty()) { + return; + } + std::vector request_line = StringUtils::Split(lines[0], " ", + /*ignore_empty=*/true); + if (request_line.size() < 2) { + return; + } + request.method = request_line[0]; + std::string target = request_line[1]; + size_t question = target.find('?'); + if (question == std::string::npos) { + request.path = UrlUtils::DecodeString(target); + } else { + request.path = UrlUtils::DecodeString(target.substr(0, question)); + request.query_params = ParseQuery(target.substr(question + 1)); + } + size_t content_length = 0; + for (size_t i = 1; i < lines.size(); i++) { + size_t colon = lines[i].find(':'); + if (colon == std::string::npos) { + continue; + } + std::string name = lines[i].substr(0, colon); + std::string value = lines[i].substr(colon + 1); + StringUtils::Trim(&name); + StringUtils::Trim(&value); + request.headers[StringUtils::ToLowerCase(name)] = value; + } + auto length_iter = request.headers.find("content-length"); + if (length_iter != request.headers.end()) { + content_length = + static_cast(std::strtoul(length_iter->second.c_str(), nullptr, 10)); + } + size_t body_begin = header_end + 4; + if (!ReceiveAll(connection_fd, &buffer, body_begin + content_length)) { + return; + } + request.body = buffer.substr(body_begin, content_length); + + Response response = handler_(request); + if (response.close_without_response) { + return; + } + std::string extra_headers; + for (const auto& [name, value] : response.headers) { + extra_headers += fmt::format("{}: {}\r\n", name, value); + } + std::string payload = fmt::format( + "HTTP/1.1 {} MOCK\r\nContent-Type: {}\r\nContent-Length: {}\r\n{}Connection: " + "close\r\n\r\n{}", + response.code, response.content_type, response.body.size() + response.missing_body_bytes, + extra_headers, response.body); + size_t sent = 0; + while (sent < payload.size()) { + ssize_t written = ::send(connection_fd, payload.data() + sent, payload.size() - sent, 0); + if (written <= 0) { + return; + } + sent += static_cast(written); + } +} + +} // namespace paimon diff --git a/src/paimon/rest/mock_rest_server.h b/src/paimon/rest/mock_rest_server.h new file mode 100644 index 00000000..240f661e --- /dev/null +++ b/src/paimon/rest/mock_rest_server.h @@ -0,0 +1,91 @@ +/* + * Copyright 2026-present Alibaba Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include +#include +#include +#include +#include + +#include "paimon/result.h" +#include "paimon/status.h" + +namespace paimon { + +/// A minimal blocking HTTP/1.1 server for REST catalog unit tests. It listens on a +/// random port of 127.0.0.1, parses one request per connection and answers it with the +/// response returned by the registered handler. Test only, not production code. +class MockRestServer { + public: + struct Request { + std::string method; + /// Url-decoded path, e.g. "/v1/config". + std::string path; + /// Url-decoded query parameters. + std::map query_params; + /// Headers with lower-cased names. + std::map headers; + std::string body; + }; + + struct Response { + int32_t code = 200; + std::string body; + std::string content_type = "application/json"; + std::map headers; + /// Close without answering; the client observes CURLE_GOT_NOTHING, a transport + /// error that is never retried. + bool close_without_response = false; + /// Advertise this many bytes beyond the actual body in `Content-Length`, then + /// close: the client sees the response cut off mid-body (CURLE_PARTIAL_FILE), a + /// retriable transport error. + size_t missing_body_bytes = 0; + }; + + using Handler = std::function; + + /// Starts the server; `handler` runs on the accept thread, so any state it touches + /// must be thread-safe. + static Result> Start(Handler handler); + + ~MockRestServer(); + + void Stop(); + + int32_t GetPort() const { + return port_; + } + + /// "http://127.0.0.1:" + std::string GetBaseUri() const; + + private: + MockRestServer(Handler handler, int32_t listen_fd, int32_t port); + + void AcceptLoop(); + void HandleConnection(int32_t connection_fd); + + Handler handler_; + int32_t listen_fd_ = -1; + int32_t port_ = 0; + std::atomic stopped_{false}; + std::thread accept_thread_; +}; + +} // namespace paimon diff --git a/src/paimon/rest/resource_paths.cpp b/src/paimon/rest/resource_paths.cpp new file mode 100644 index 00000000..48448237 --- /dev/null +++ b/src/paimon/rest/resource_paths.cpp @@ -0,0 +1,64 @@ +/* + * Copyright 2026-present Alibaba Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "paimon/rest/resource_paths.h" + +#include "paimon/common/utils/url_utils.h" + +namespace paimon { + +namespace { +constexpr const char kV1[] = "/v1"; +} // namespace + +ResourcePaths::ResourcePaths(const std::string& prefix) { + base_ = kV1; + if (!prefix.empty()) { + base_ += "/" + UrlUtils::EncodeString(prefix); + } +} + +std::string ResourcePaths::Config() { + return std::string(kV1) + "/config"; +} + +std::string ResourcePaths::Databases() const { + return base_ + "/databases"; +} + +std::string ResourcePaths::Database(const std::string& database_name) const { + return Databases() + "/" + UrlUtils::EncodeString(database_name); +} + +std::string ResourcePaths::Tables(const std::string& database_name) const { + return Database(database_name) + "/tables"; +} + +std::string ResourcePaths::Table(const std::string& database_name, + const std::string& table_name) const { + return Tables(database_name) + "/" + UrlUtils::EncodeString(table_name); +} + +std::string ResourcePaths::RenameTable() const { + return base_ + "/tables/rename"; +} + +std::string ResourcePaths::Snapshots(const std::string& database_name, + const std::string& table_name) const { + return Table(database_name, table_name) + "/snapshots"; +} + +} // namespace paimon diff --git a/src/paimon/rest/resource_paths.h b/src/paimon/rest/resource_paths.h new file mode 100644 index 00000000..ea36bf48 --- /dev/null +++ b/src/paimon/rest/resource_paths.h @@ -0,0 +1,45 @@ +/* + * Copyright 2026-present Alibaba Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include + +namespace paimon { + +/// Builds resource paths of the REST catalog server. All path segments are +/// url-encoded; `prefix` (usually pushed down by the server through `/v1/config`) may +/// be empty, in which case it is skipped. +class ResourcePaths { + public: + explicit ResourcePaths(const std::string& prefix); + + /// "/v1/config", the only path that does not carry the prefix. + static std::string Config(); + + std::string Databases() const; + std::string Database(const std::string& database_name) const; + std::string Tables(const std::string& database_name) const; + std::string Table(const std::string& database_name, const std::string& table_name) const; + std::string RenameTable() const; + std::string Snapshots(const std::string& database_name, const std::string& table_name) const; + + private: + /// "/v1" or "/v1/{encoded prefix}". + std::string base_; +}; + +} // namespace paimon diff --git a/src/paimon/rest/resource_paths_test.cpp b/src/paimon/rest/resource_paths_test.cpp new file mode 100644 index 00000000..782f3f69 --- /dev/null +++ b/src/paimon/rest/resource_paths_test.cpp @@ -0,0 +1,40 @@ +/* + * Copyright 2026-present Alibaba Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "paimon/rest/resource_paths.h" + +#include "gtest/gtest.h" + +namespace paimon::test { + +TEST(ResourcePathsTest, WithPrefix) { + ResourcePaths paths("my prefix"); + ASSERT_EQ("/v1/config", ResourcePaths::Config()); + ASSERT_EQ("/v1/my+prefix/databases", paths.Databases()); + ASSERT_EQ("/v1/my+prefix/databases/db%231", paths.Database("db#1")); + ASSERT_EQ("/v1/my+prefix/databases/db/tables", paths.Tables("db")); + ASSERT_EQ("/v1/my+prefix/databases/db/tables/t1", paths.Table("db", "t1")); + ASSERT_EQ("/v1/my+prefix/tables/rename", paths.RenameTable()); + ASSERT_EQ("/v1/my+prefix/databases/db/tables/t1/snapshots", paths.Snapshots("db", "t1")); +} + +TEST(ResourcePathsTest, WithoutPrefix) { + ResourcePaths paths(""); + ASSERT_EQ("/v1/databases", paths.Databases()); + ASSERT_EQ("/v1/tables/rename", paths.RenameTable()); +} + +} // namespace paimon::test diff --git a/src/paimon/rest/rest_api.cpp b/src/paimon/rest/rest_api.cpp new file mode 100644 index 00000000..393a0938 --- /dev/null +++ b/src/paimon/rest/rest_api.cpp @@ -0,0 +1,280 @@ +/* + * Copyright 2026-present Alibaba Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "paimon/rest/rest_api.h" + +#include +#include +#include + +#include "fmt/format.h" +#include "paimon/catalog_options.h" +#include "paimon/common/utils/rapidjson_util.h" +#include "paimon/common/utils/sensitive_config_utils.h" +#include "paimon/logging.h" +#include "paimon/rest/rest_util.h" + +namespace paimon { + +namespace { +// The `PAIMON_ASSIGN_OR_RAISE` macro cannot take a declaration type containing a comma. +using StringMap = std::map; + +// A successful response body may carry credentials (e.g. a token response), so neither it +// nor the parse failure quoting it is echoed into the error message; only the request path +// is reported. +template +Status ParseResponseBody(const std::string& body, const std::string& path, ResponseT* entity) { + if (!RapidJsonUtil::FromJsonString(body, entity).ok()) { + return Status::Invalid( + fmt::format("failed to deserialize the response of {} from the rest server", path)); + } + return Status::OK(); +} +} // namespace + +RestApi::RestApi(std::unique_ptr client, + std::unique_ptr auth_provider, + const std::map& base_headers, + const std::map& options, const ResourcePaths& paths) + : client_(std::move(client)), + auth_provider_(std::move(auth_provider)), + base_headers_(base_headers), + options_(options), + resource_paths_(paths) {} + +Result> RestApi::Create(const std::map& options, + const std::string& warehouse, bool config_required, + const RestHttpClient::Config& http_config) { + auto uri_iter = options.find(CatalogOptions::URI); + if (uri_iter == options.end() || uri_iter->second.empty()) { + return Status::Invalid(fmt::format("option '{}' must be configured for the rest catalog", + CatalogOptions::URI)); + } + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr client, + RestHttpClient::Create(uri_iter->second, http_config)); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr auth_provider, + AuthProvider::Create(options)); + + std::map merged_options = options; + std::map base_headers = + RestUtil::ExtractPrefixMap(options, kHeaderOptionPrefix); + if (config_required) { + std::map query_params; + if (!warehouse.empty()) { + query_params[kQueryParamWarehouse] = warehouse; + } + RestAuthParameter auth_parameter = + RestAuthParameter::Create("GET", ResourcePaths::Config(), query_params, ""); + PAIMON_ASSIGN_OR_RAISE(StringMap headers, + auth_provider->MergeAuthHeader(base_headers, auth_parameter)); + PAIMON_ASSIGN_OR_RAISE( + RestHttpClient::Response response, + client->Execute("GET", ResourcePaths::Config(), query_params, headers, "")); + if (!response.IsSuccessful()) { + return ErrorToStatus(response); + } + ConfigResponse config; + PAIMON_RETURN_NOT_OK(ParseResponseBody(response.body, ResourcePaths::Config(), &config)); + merged_options = config.Merge(options); + for (const auto& [key, value] : + RestUtil::ExtractPrefixMap(merged_options, kHeaderOptionPrefix)) { + base_headers[key] = value; + } + std::shared_ptr logger = Logger::GetLogger("RestApi"); + PAIMON_LOG_DEBUG(logger, + "merged %zu client options with the /v1/config response into %zu options", + options.size(), merged_options.size()); + } + std::string prefix; + auto prefix_iter = merged_options.find(kOptionUrlPrefix); + if (prefix_iter != merged_options.end()) { + prefix = prefix_iter->second; + } + return std::unique_ptr(new RestApi(std::move(client), std::move(auth_provider), + base_headers, merged_options, + ResourcePaths(prefix))); +} + +Status RestApi::ErrorToStatus(const RestHttpClient::Response& response) { + // The code of the parsed error body takes precedence over the http status, which + // a gateway may have rewritten. + int64_t code = response.code; + std::string message; + std::string resource_info; + bool body_parsed = false; + if (!response.body.empty()) { + ErrorResponse error; + if (RapidJsonUtil::FromJsonString(response.body, &error).ok()) { + body_parsed = true; + // The message may embed secrets (e.g. "password=..."), so it is redacted. + message = SensitiveConfigUtils::RedactText(error.GetMessage()); + if (!error.GetResourceType().empty()) { + resource_info = fmt::format(" (resource type: {}, resource name: {})", + error.GetResourceType(), error.GetResourceName()); + } + if (error.GetCode() != 0) { + code = error.GetCode(); + } + } + } + if (message.empty()) { + // The body is never echoed (it may carry credentials), so a server answering with + // something other than an error object is told apart by the message alone. + message = body_parsed ? fmt::format("empty error message (http status {})", response.code) + : fmt::format("unparsable error response body (http status {})", + response.code); + } + message += resource_info; + std::string request_id = RestUtil::ExtractRequestId(response.headers); + if (request_id != RestUtil::kUnknownRequestId) { + message += fmt::format(" requestId:{}", request_id); + } + Status status; + switch (code) { + case HttpStatus::kBadRequest: + status = Status::Invalid(message); + break; + case HttpStatus::kUnauthorized: + status = Status::IOError("not authorized: ", message); + break; + case HttpStatus::kForbidden: + status = Status::IOError("forbidden: ", message); + break; + case HttpStatus::kNotFound: + status = Status::NotExist(message); + break; + case HttpStatus::kConflict: + status = Status::Exist(message); + break; + case HttpStatus::kInternalServerError: + status = Status::IOError("server error: ", message); + break; + case HttpStatus::kNotImplemented: + status = Status::NotImplemented(message); + break; + case HttpStatus::kServiceUnavailable: + status = Status::IOError("service unavailable: ", message); + break; + default: + status = + Status::IOError(fmt::format("rest request failed with code {}: {}", code, message)); + break; + } + return status.WithDetail(std::make_shared(code)); +} + +Result RestApi::Execute( + const std::string& method, const std::string& path, + const std::map& query_params, const std::string& body) const { + RestAuthParameter auth_parameter = RestAuthParameter::Create(method, path, query_params, body); + StringMap request_headers = base_headers_; + if (!body.empty()) { + // Set before merging the auth headers so a provider that signs headers covers it. + request_headers["Content-Type"] = "application/json"; + } + PAIMON_ASSIGN_OR_RAISE(StringMap headers, + auth_provider_->MergeAuthHeader(request_headers, auth_parameter)); + PAIMON_ASSIGN_OR_RAISE(RestHttpClient::Response response, + client_->Execute(method, path, query_params, headers, body)); + if (!response.IsSuccessful()) { + return ErrorToStatus(response); + } + return response; +} + +template +Result RestApi::GetEntity(const std::string& path, + const std::map& query_params) const { + PAIMON_ASSIGN_OR_RAISE(RestHttpClient::Response response, + Execute("GET", path, query_params, "")); + ResponseT entity; + PAIMON_RETURN_NOT_OK(ParseResponseBody(response.body, path, &entity)); + return entity; +} + +template +Result> RestApi::ListAllPages( + const std::string& path) const { + std::vector items; + std::map query_params; + while (true) { + PAIMON_ASSIGN_OR_RAISE(ResponseT response, GetEntity(path, query_params)); + const auto& data = response.Data(); + items.insert(items.end(), data.begin(), data.end()); + const std::optional& next_page_token = response.NextPageToken(); + if (!next_page_token || next_page_token.value().empty() || data.empty()) { + return items; + } + query_params[kQueryParamPageToken] = next_page_token.value(); + } +} + +Result> RestApi::ListDatabases() const { + return ListAllPages(resource_paths_.Databases()); +} + +Status RestApi::CreateDatabase(const std::string& name, + const std::map& options) const { + CreateDatabaseRequest request(name, options); + PAIMON_ASSIGN_OR_RAISE(std::string body, request.ToJsonString()); + return Execute("POST", resource_paths_.Databases(), {}, body).status(); +} + +Result RestApi::GetDatabase(const std::string& name) const { + return GetEntity(resource_paths_.Database(name), {}); +} + +Status RestApi::DropDatabase(const std::string& name) const { + return Execute("DELETE", resource_paths_.Database(name), {}, "").status(); +} + +Result> RestApi::ListTables(const std::string& database_name) const { + return ListAllPages(resource_paths_.Tables(database_name)); +} + +Result RestApi::GetTable(const Identifier& identifier) const { + return GetEntity( + resource_paths_.Table(identifier.GetDatabaseName(), identifier.GetTableName()), {}); +} + +Status RestApi::CreateTable(const Identifier& identifier, const std::string& schema_json) const { + CreateTableRequest request(identifier.GetDatabaseName(), identifier.GetTableName(), + schema_json); + PAIMON_ASSIGN_OR_RAISE(std::string body, request.ToJsonString()); + return Execute("POST", resource_paths_.Tables(identifier.GetDatabaseName()), {}, body).status(); +} + +Status RestApi::DropTable(const Identifier& identifier) const { + return Execute("DELETE", + resource_paths_.Table(identifier.GetDatabaseName(), identifier.GetTableName()), + {}, "") + .status(); +} + +Status RestApi::RenameTable(const Identifier& from_table, const Identifier& to_table) const { + RenameTableRequest request(from_table.GetDatabaseName(), from_table.GetTableName(), + to_table.GetDatabaseName(), to_table.GetTableName()); + PAIMON_ASSIGN_OR_RAISE(std::string body, request.ToJsonString()); + return Execute("POST", resource_paths_.RenameTable(), {}, body).status(); +} + +Result> RestApi::ListSnapshots(const Identifier& identifier) const { + return ListAllPages( + resource_paths_.Snapshots(identifier.GetDatabaseName(), identifier.GetTableName())); +} + +} // namespace paimon diff --git a/src/paimon/rest/rest_api.h b/src/paimon/rest/rest_api.h new file mode 100644 index 00000000..c2394ced --- /dev/null +++ b/src/paimon/rest/rest_api.h @@ -0,0 +1,138 @@ +/* + * Copyright 2026-present Alibaba Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include +#include +#include + +#include "paimon/catalog/identifier.h" +#include "paimon/core/snapshot.h" +#include "paimon/rest/resource_paths.h" +#include "paimon/rest/rest_auth.h" +#include "paimon/rest/rest_http_client.h" +#include "paimon/rest/rest_messages.h" +#include "paimon/result.h" +#include "paimon/status.h" + +namespace paimon { + +/// Carries the code of a failed rest request (the parsed error body's code, falling back +/// to the http status) so callers can tell e.g. an authentication failure (401/403) from +/// another IO error. +class RestErrorDetail : public StatusDetail { + public: + static constexpr const char* kTypeId = "rest-error-detail"; + + explicit RestErrorDetail(int64_t code) : code_(code) {} + + const char* type_id() const override { + return kTypeId; + } + + std::string ToString() const override { + return "rest error code " + std::to_string(code_); + } + + int64_t GetCode() const { + return code_; + } + + private: + int64_t code_; +}; + +/// The client of the REST catalog server. This layer only talks HTTP + JSON and never +/// touches the file system. +class RestApi { + public: + static constexpr const char* kQueryParamPageToken = "pageToken"; + static constexpr const char* kQueryParamWarehouse = "warehouse"; + /// Option key of the url path prefix inserted after "/v1", usually pushed down by + /// "/v1/config". + static constexpr const char* kOptionUrlPrefix = "prefix"; + /// Options with this prefix are sent as http headers (with the prefix stripped). + static constexpr const char* kHeaderOptionPrefix = "header."; + + /// Creates the api client. + /// + /// @param options Client side options; `CatalogOptions::URI` and + /// `CatalogOptions::TOKEN_PROVIDER` are required. + /// @param warehouse Warehouse sent as query parameter of "/v1/config"; may be empty. + /// @param config_required When true, fetch "/v1/config" and merge the server + /// defaults/overrides into `options` with the precedence + /// overrides > client options > defaults. + /// @param http_config Transport level settings, mainly overridable for tests. + static Result> Create( + const std::map& options, const std::string& warehouse, + bool config_required, const RestHttpClient::Config& http_config = RestHttpClient::Config()); + + /// Options merged with the server side config. + const std::map& GetMergedOptions() const { + return options_; + } + + Result> ListDatabases() const; + Status CreateDatabase(const std::string& name, + const std::map& options) const; + Result GetDatabase(const std::string& name) const; + Status DropDatabase(const std::string& name) const; + + Result> ListTables(const std::string& database_name) const; + Result GetTable(const Identifier& identifier) const; + /// `schema_json` uses the schema JSON layout of the protocol + /// (fields/partitionKeys/primaryKeys/options/comment). + Status CreateTable(const Identifier& identifier, const std::string& schema_json) const; + Status DropTable(const Identifier& identifier) const; + Status RenameTable(const Identifier& from_table, const Identifier& to_table) const; + + Result> ListSnapshots(const Identifier& identifier) const; + + /// Maps a non-successful http response to a status: 404 becomes `NotExist`, 409 + /// becomes `Exist`, 400 becomes `Invalid`, 501 becomes `NotImplemented` and the + /// other codes become `IOError`. The status carries a `RestErrorDetail` with the + /// mapped code. + static Status ErrorToStatus(const RestHttpClient::Response& response); + + private: + RestApi(std::unique_ptr client, std::unique_ptr auth_provider, + const std::map& base_headers, + const std::map& options, const ResourcePaths& paths); + + /// Executes one request with the authentication headers merged in, mapping a + /// non-successful response to an error status. + Result Execute(const std::string& method, const std::string& path, + const std::map& query_params, + const std::string& body) const; + + template + Result GetEntity(const std::string& path, + const std::map& query_params) const; + + /// Fetches all pages, stopping at an empty page or a missing/empty next page token. + template + Result> ListAllPages(const std::string& path) const; + + std::unique_ptr client_; + std::unique_ptr auth_provider_; + std::map base_headers_; + std::map options_; + ResourcePaths resource_paths_; +}; + +} // namespace paimon diff --git a/src/paimon/rest/rest_auth.cpp b/src/paimon/rest/rest_auth.cpp new file mode 100644 index 00000000..65d6d755 --- /dev/null +++ b/src/paimon/rest/rest_auth.cpp @@ -0,0 +1,71 @@ +/* + * Copyright 2026-present Alibaba Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "paimon/rest/rest_auth.h" + +#include "fmt/format.h" +#include "paimon/catalog_options.h" +#include "paimon/common/utils/string_utils.h" +#include "paimon/common/utils/url_utils.h" + +namespace paimon { + +RestAuthParameter RestAuthParameter::Create(const std::string& method, + const std::string& resource_path, + const std::map& query_params, + const std::string& data) { + RestAuthParameter parameter; + parameter.method = method; + parameter.resource_path = resource_path; + for (const auto& [key, value] : query_params) { + parameter.parameters[key] = UrlUtils::EncodeString(value); + } + parameter.data = data; + return parameter; +} + +Result> BearTokenAuthProvider::MergeAuthHeader( + const std::map& base_header, + const RestAuthParameter& parameter) const { + std::map headers = base_header; + headers["Authorization"] = "Bearer " + token_; + return headers; +} + +Result> AuthProvider::Create( + const std::map& options) { + auto provider_iter = options.find(CatalogOptions::TOKEN_PROVIDER); + if (provider_iter == options.end() || provider_iter->second.empty()) { + return Status::Invalid(fmt::format("option '{}' must be configured for the rest catalog", + CatalogOptions::TOKEN_PROVIDER)); + } + // Matched leniently in lower case; other clients may match the provider name + // case-sensitively, so only the exact "bear" spelling is portable. + std::string provider = StringUtils::ToLowerCase(provider_iter->second); + if (provider == "bear") { + auto token_iter = options.find(CatalogOptions::TOKEN); + if (token_iter == options.end() || token_iter->second.empty()) { + return Status::Invalid( + fmt::format("option '{}' must be configured for the bear token provider", + CatalogOptions::TOKEN)); + } + return std::make_unique(token_iter->second); + } + return Status::NotImplemented( + fmt::format("unsupported token provider: {}, only 'bear' is supported for now", provider)); +} + +} // namespace paimon diff --git a/src/paimon/rest/rest_auth.h b/src/paimon/rest/rest_auth.h new file mode 100644 index 00000000..22da08d9 --- /dev/null +++ b/src/paimon/rest/rest_auth.h @@ -0,0 +1,73 @@ +/* + * Copyright 2026-present Alibaba Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include +#include + +#include "paimon/result.h" +#include "paimon/status.h" + +namespace paimon { + +/// Input of one request signature. +struct RestAuthParameter { + std::string method; + std::string resource_path; + /// Query parameters with url-encoded values. + std::map parameters; + /// Request body, empty when the request carries none. + std::string data; + + /// Builds the parameter of one request. `query_params` are stored url-encoded, since + /// a signing provider signs the query string as it is sent; building the parameter + /// only through here keeps a request from being signed over unencoded values. + static RestAuthParameter Create(const std::string& method, const std::string& resource_path, + const std::map& query_params, + const std::string& data); +}; + +/// Generates authentication headers for REST catalog requests. +class AuthProvider { + public: + virtual ~AuthProvider() = default; + + /// Returns `base_header` merged with the authentication headers of this provider. + virtual Result> MergeAuthHeader( + const std::map& base_header, + const RestAuthParameter& parameter) const = 0; + + /// Creates the provider configured by `CatalogOptions::TOKEN_PROVIDER`. + static Result> Create( + const std::map& options); +}; + +/// Adds `Authorization: Bearer `. +class BearTokenAuthProvider : public AuthProvider { + public: + explicit BearTokenAuthProvider(const std::string& token) : token_(token) {} + + Result> MergeAuthHeader( + const std::map& base_header, + const RestAuthParameter& parameter) const override; + + private: + std::string token_; +}; + +} // namespace paimon diff --git a/src/paimon/rest/rest_catalog.cpp b/src/paimon/rest/rest_catalog.cpp new file mode 100644 index 00000000..6706bb50 --- /dev/null +++ b/src/paimon/rest/rest_catalog.cpp @@ -0,0 +1,442 @@ +/* + * Copyright 2026-present Alibaba Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "paimon/rest/rest_catalog.h" + +#include +#include + +#include "arrow/c/bridge.h" +#include "fmt/format.h" +#include "paimon/catalog/table.h" +#include "paimon/catalog_options.h" +#include "paimon/common/utils/arrow/status_utils.h" +#include "paimon/common/utils/rapidjson_util.h" +#include "paimon/common/utils/string_utils.h" +#include "paimon/core/catalog/catalog_utils.h" +#include "paimon/core/core_options.h" +#include "paimon/core/schema/table_schema.h" +#include "paimon/core/table/system/global_system_tables.h" +#include "paimon/core/table/system/system_table.h" +#include "paimon/core/table/system/system_table_schema.h" +#include "paimon/defs.h" +#include "paimon/fs/file_system.h" +#include "paimon/rest/rest_util.h" +#include "rapidjson/document.h" + +namespace paimon { + +namespace { + +constexpr const char kPathOption[] = "path"; + +// Maps the default branch "main" to "no branch": it is addressed as the bare table. The +// comparison ignores case, as the identifier the Java client sends is built the same way, +// so "MAIN" also resolves to the bare table and cannot address a branch of that name. +// `BranchManager::IsMainBranch`, which names the branch directory of a table, stays +// case-sensitive: this normalization only decides how a table is addressed on the server. +std::optional NormalizeBranch(std::optional branch) { + if (branch && StringUtils::ToLowerCase(branch.value()) == Identifier::kDefaultMainBranch) { + return std::nullopt; + } + return branch; +} + +// Builds the "$branch_" object name addressing `branch` of `table_name` +// on the rest server. +std::string BranchObjectName(std::string table_name, const std::string& branch) { + table_name.append(Identifier::kSystemTableSplitter); + table_name.append(Identifier::kSystemBranchPrefix); + table_name.append(branch); + return table_name; +} + +// Builds the identifier sent to the rest server: the system table suffix is stripped +// while the branch stays in the object name, so the server resolves the branch itself and +// returns the branch's own schema. The path the server reports is the data table root in +// either case; the branch subdirectory is derived downstream from the branch option and +// must not be applied twice (see `ToTableSchema`). +Result ToLoadIdentifier(const Identifier& identifier) { + PAIMON_ASSIGN_OR_RAISE(std::string data_table_name, identifier.GetDataTableName()); + PAIMON_ASSIGN_OR_RAISE(std::optional branch, identifier.GetBranchName()); + branch = NormalizeBranch(std::move(branch)); + std::string object_name = branch ? BranchObjectName(std::move(data_table_name), branch.value()) + : std::move(data_table_name); + return Identifier(identifier.GetDatabaseName(), object_name); +} + +} // namespace + +RestCatalog::RestCatalog(std::unique_ptr api, const std::shared_ptr& fs, + const std::string& warehouse) + : api_(std::move(api)), + fs_(fs), + warehouse_(warehouse), + table_default_options_(RestUtil::ExtractPrefixMap( + api_->GetMergedOptions(), CatalogOptions::TABLE_DEFAULT_OPTION_PREFIX)), + logger_(Logger::GetLogger("RestCatalog")) {} + +Result> RestCatalog::Create( + const std::string& warehouse, const std::map& options, + const std::shared_ptr& file_system, const RestHttpClient::Config& http_config) { + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr api, + RestApi::Create(options, warehouse, /*config_required=*/true, http_config)); + PAIMON_ASSIGN_OR_RAISE(CoreOptions core_options, + CoreOptions::FromMap(api->GetMergedOptions(), file_system)); + return std::unique_ptr( + new RestCatalog(std::move(api), core_options.GetFileSystem(), warehouse)); +} + +const std::map& RestCatalog::GetOptions() const { + return api_->GetMergedOptions(); +} + +Status RestCatalog::CreateDatabase(const std::string& name, + const std::map& options, + bool ignore_if_exists) { + PAIMON_RETURN_NOT_OK(CatalogUtils::CheckNotSystemDatabase(name, "createDatabase")); + Status status = api_->CreateDatabase(name, options); + if (status.IsExist() && ignore_if_exists) { + return Status::OK(); + } + return status; +} + +Result> RestCatalog::ListDatabases() const { + return api_->ListDatabases(); +} + +Result RestCatalog::DatabaseExists(const std::string& db_name) const { + if (CatalogUtils::IsSystemDatabase(db_name)) { + return true; + } + Result response = api_->GetDatabase(db_name); + if (response.ok()) { + return true; + } + if (response.status().IsNotExist()) { + return false; + } + return response.status(); +} + +Status RestCatalog::DropDatabase(const std::string& name, bool ignore_if_not_exists, bool cascade) { + PAIMON_RETURN_NOT_OK(CatalogUtils::CheckNotSystemDatabase(name, "dropDatabase")); + if (!cascade) { + Result> tables = ListTables(name); + if (!tables.ok()) { + if (tables.status().IsNotExist() && ignore_if_not_exists) { + return Status::OK(); + } + return tables.status(); + } + if (!tables.value().empty()) { + return Status::Invalid( + fmt::format("Cannot drop non-empty database {}. Use cascade=true to force.", name)); + } + } + Status status = api_->DropDatabase(name); + if (status.IsNotExist() && ignore_if_not_exists) { + return Status::OK(); + } + return status; +} + +std::string RestCatalog::GetDatabaseLocation(const std::string& db_name) const { + // The virtual "sys" database has no location and is unknown to the server. + if (CatalogUtils::IsSystemDatabase(db_name)) { + return ""; + } + Result response = api_->GetDatabase(db_name); + if (!response.ok()) { + PAIMON_LOG_WARN(logger_, "failed to get location of database %s: %s", db_name.c_str(), + response.status().ToString().c_str()); + return ""; + } + return response.value().GetLocation(); +} + +Result> RestCatalog::ListTables(const std::string& db_name) const { + if (CatalogUtils::IsSystemDatabase(db_name)) { + return GlobalSystemTableLoader::GetSupportedTableNames(api_->GetMergedOptions()); + } + return api_->ListTables(db_name); +} + +Status RestCatalog::CreateTable(const Identifier& identifier, ArrowSchema* c_schema, + const std::vector& partition_keys, + const std::vector& primary_keys, + const std::map& options, + bool ignore_if_exists) { + PAIMON_RETURN_NOT_OK(CatalogUtils::CheckNotBranch(identifier, "createTable")); + PAIMON_RETURN_NOT_OK(CatalogUtils::CheckNotSystemTable(identifier, "createTable")); + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr schema, + arrow::ImportSchema(c_schema)); + std::map effective_options = options; + for (const auto& [key, value] : table_default_options_) { + effective_options.emplace(key, value); + } + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr table_schema, + TableSchema::Create(TableSchema::FIRST_SCHEMA_ID, schema, partition_keys, + primary_keys, effective_options)); + std::string schema_json; + try { + rapidjson::Document doc; + doc.SetObject(); + rapidjson::Document::AllocatorType& allocator = doc.GetAllocator(); + doc.AddMember(rapidjson::StringRef("fields"), + RapidJsonUtil::SerializeValue(table_schema->Fields(), &allocator).Move(), + allocator); + doc.AddMember(rapidjson::StringRef("partitionKeys"), + RapidJsonUtil::SerializeValue(partition_keys, &allocator).Move(), allocator); + doc.AddMember(rapidjson::StringRef("primaryKeys"), + RapidJsonUtil::SerializeValue(primary_keys, &allocator).Move(), allocator); + doc.AddMember(rapidjson::StringRef("options"), + RapidJsonUtil::SerializeValue(effective_options, &allocator).Move(), + allocator); + schema_json = RestUtil::JsonToString(doc); + } catch (const std::exception& e) { + return Status::SerializationError("failed to serialize create table schema: ", e.what()); + } + Status status = api_->CreateTable(identifier, schema_json); + if (status.IsExist() && ignore_if_exists) { + return Status::OK(); + } + return status; +} + +Result RestCatalog::TableExists(const Identifier& identifier) const { + if (CatalogUtils::IsSystemDatabase(identifier.GetDatabaseName())) { + return GlobalSystemTableLoader::IsSupported(identifier.GetTableName(), + api_->GetMergedOptions()); + } + PAIMON_ASSIGN_OR_RAISE(bool is_system_table, identifier.IsSystemTable()); + if (is_system_table) { + PAIMON_ASSIGN_OR_RAISE(std::optional system_table_name, + identifier.GetSystemTableName()); + if (!system_table_name || !SystemTableLoader::IsSupported(system_table_name.value())) { + return false; + } + } + PAIMON_ASSIGN_OR_RAISE(Identifier load_identifier, ToLoadIdentifier(identifier)); + Result response = api_->GetTable(load_identifier); + if (response.ok()) { + return true; + } + if (response.status().IsNotExist()) { + return false; + } + return response.status(); +} + +Result RestCatalog::GetTableLocation(const Identifier& identifier) const { + PAIMON_ASSIGN_OR_RAISE(Identifier load_identifier, ToLoadIdentifier(identifier)); + PAIMON_ASSIGN_OR_RAISE(GetTableResponse response, api_->GetTable(load_identifier)); + return response.GetPath(); +} + +Status RestCatalog::DropTable(const Identifier& identifier, bool ignore_if_not_exists) { + PAIMON_RETURN_NOT_OK(CatalogUtils::CheckNotBranch(identifier, "dropTable")); + PAIMON_RETURN_NOT_OK(CatalogUtils::CheckNotSystemTable(identifier, "dropTable")); + Status status = api_->DropTable(identifier); + if (status.IsNotExist() && ignore_if_not_exists) { + return Status::OK(); + } + return status; +} + +Status RestCatalog::RenameTable(const Identifier& from_table, const Identifier& to_table, + bool ignore_if_not_exists) { + PAIMON_RETURN_NOT_OK(CatalogUtils::CheckNotBranch(from_table, "renameTable")); + PAIMON_RETURN_NOT_OK(CatalogUtils::CheckNotBranch(to_table, "renameTable")); + PAIMON_RETURN_NOT_OK(CatalogUtils::CheckNotSystemTable(from_table, "renameTable")); + PAIMON_RETURN_NOT_OK(CatalogUtils::CheckNotSystemTable(to_table, "renameTable")); + Status status = api_->RenameTable(from_table, to_table); + if (status.IsNotExist() && ignore_if_not_exists) { + return Status::OK(); + } + return status; +} + +Result> RestCatalog::ToTableSchema( + const GetTableResponse& response, const std::optional& branch) { + std::string table_schema_json; + try { + rapidjson::Document src; + src.Parse(response.GetSchemaJson().c_str()); + if (src.HasParseError() || !src.IsObject() || !src.HasMember("fields") || + !src["fields"].IsArray()) { + return Status::Invalid("invalid table schema json from the rest server"); + } + rapidjson::Document out; + out.SetObject(); + rapidjson::Document::AllocatorType& allocator = out.GetAllocator(); + out.AddMember(rapidjson::StringRef("version"), TableSchema::CURRENT_VERSION, allocator); + out.AddMember(rapidjson::StringRef("id"), response.GetSchemaId(), allocator); + PAIMON_ASSIGN_OR_RAISE(int32_t highest_field_id, + TableSchema::ComputeHighestFieldId(src["fields"])); + rapidjson::Value fields(rapidjson::kArrayType); + fields.CopyFrom(src["fields"], allocator); + out.AddMember(rapidjson::StringRef("highestFieldId"), highest_field_id, allocator); + out.AddMember(rapidjson::StringRef("fields"), fields.Move(), allocator); + // A missing or wrong-typed member fails instead of defaulting to an empty value, + // which would silently change table semantics (an omitted "partitionKeys" would + // load a partitioned table as unpartitioned); Java rejects absent members too. + for (const char* key : {"partitionKeys", "primaryKeys"}) { + if (!src.HasMember(key)) { + return Status::Invalid(fmt::format( + "invalid table schema json from the rest server: missing '{}'", key)); + } + if (!src[key].IsArray()) { + return Status::Invalid(fmt::format( + "invalid table schema json from the rest server: '{}' is not an array", key)); + } + rapidjson::Value keys(rapidjson::kArrayType); + keys.CopyFrom(src[key], allocator); + out.AddMember(rapidjson::StringRef(key), keys.Move(), allocator); + } + if (!src.HasMember("options")) { + return Status::Invalid( + "invalid table schema json from the rest server: missing 'options'"); + } + if (!src["options"].IsObject()) { + return Status::Invalid( + "invalid table schema json from the rest server: 'options' is not an object"); + } + auto options_map = + RapidJsonUtil::DeserializeValue>(src["options"]); + options_map[kPathOption] = response.GetPath(); + response.GetAuditFields().PutAuditOptionsTo(&options_map); + if (branch) { + options_map[Options::BRANCH] = branch.value(); + } + out.AddMember(rapidjson::StringRef("options"), + RapidJsonUtil::SerializeValue(options_map, &allocator).Move(), allocator); + if (src.HasMember("comment") && src["comment"].IsString()) { + rapidjson::Value comment; + comment.CopyFrom(src["comment"], allocator); + out.AddMember(rapidjson::StringRef("comment"), comment.Move(), allocator); + } + // The server's audit time keeps the conversion deterministic; 0 (the epoch) + // when the server did not report an "updatedAt". + out.AddMember(rapidjson::StringRef("timeMillis"), + response.GetAuditFields().updated_at.value_or(0), allocator); + table_schema_json = RestUtil::JsonToString(out); + } catch (const std::exception& e) { + return Status::Invalid("failed to convert rest table schema: ", e.what()); + } + return TableSchema::CreateFromJson(table_schema_json); +} + +Result> RestCatalog::LoadDataTableSchema( + const Identifier& data_identifier, const std::optional& branch, + std::string* table_path) const { + PAIMON_ASSIGN_OR_RAISE(GetTableResponse response, api_->GetTable(data_identifier)); + if (table_path != nullptr) { + *table_path = response.GetPath(); + } + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr schema, ToTableSchema(response, branch)); + return std::shared_ptr(std::move(schema)); +} + +Result> RestCatalog::LoadTableSchema(const Identifier& identifier) const { + // Serve the global system tables of the "sys" database locally, like + // FileSystemCatalog. + if (CatalogUtils::IsSystemDatabase(identifier.GetDatabaseName())) { + PAIMON_ASSIGN_OR_RAISE(bool supported, + GlobalSystemTableLoader::IsSupported(identifier.GetTableName(), + api_->GetMergedOptions())); + if (!supported) { + return Status::NotExist(fmt::format("{} not exist", identifier.ToString())); + } + GlobalSystemTableContext context; + context.catalog = this; + context.fs = fs_; + context.warehouse = warehouse_; + context.catalog_options = api_->GetMergedOptions(); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr system_table, + GlobalSystemTableLoader::Load(identifier.GetTableName(), context)); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr arrow_schema, + system_table->ArrowSchema()); + return std::make_shared(std::move(arrow_schema)); + } + PAIMON_ASSIGN_OR_RAISE(bool is_system_table, identifier.IsSystemTable()); + PAIMON_ASSIGN_OR_RAISE(std::optional branch, identifier.GetBranchName()); + branch = NormalizeBranch(std::move(branch)); + PAIMON_ASSIGN_OR_RAISE(Identifier load_identifier, ToLoadIdentifier(identifier)); + if (is_system_table) { + PAIMON_ASSIGN_OR_RAISE(std::optional system_table_name, + identifier.GetSystemTableName()); + if (!system_table_name || !SystemTableLoader::IsSupported(system_table_name.value())) { + return Status::NotExist(fmt::format("{} not exist", identifier.ToString())); + } + std::string table_path; + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr latest_schema, + LoadDataTableSchema(load_identifier, branch, &table_path)); + std::map dynamic_options; + if (branch) { + dynamic_options[Options::BRANCH] = branch.value(); + } + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr system_table, + SystemTableLoader::Load(system_table_name.value(), fs_, table_path, + latest_schema, dynamic_options)); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr arrow_schema, + system_table->ArrowSchema()); + return std::make_shared(std::move(arrow_schema)); + } + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr schema, + LoadDataTableSchema(load_identifier, branch, nullptr)); + return std::static_pointer_cast(schema); +} + +Result> RestCatalog::GetTable(const Identifier& identifier) const { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr schema, LoadTableSchema(identifier)); + return std::make_shared
(schema, identifier.GetDatabaseName(), identifier.GetTableName()); +} + +std::string RestCatalog::GetRootPath() const { + return warehouse_; +} + +std::shared_ptr RestCatalog::GetFileSystem() const { + return fs_; +} + +Result> RestCatalog::ListSnapshots(const Identifier& identifier, + const std::string& branch) const { + PAIMON_RETURN_NOT_OK(CatalogUtils::CheckNotBranch(identifier, "listSnapshots")); + PAIMON_RETURN_NOT_OK(CatalogUtils::CheckNotSystemTable(identifier, "listSnapshots")); + std::optional normalized_branch = + NormalizeBranch(branch.empty() ? std::nullopt : std::make_optional(branch)); + std::string object_name = + normalized_branch ? BranchObjectName(identifier.GetTableName(), normalized_branch.value()) + : identifier.GetTableName(); + Identifier load_identifier(identifier.GetDatabaseName(), object_name); + // The Catalog interface has no pagination, so all pages are fetched; the server + // does not order snapshots across pages while the contract requires ascending ids. + PAIMON_ASSIGN_OR_RAISE(std::vector snapshots, api_->ListSnapshots(load_identifier)); + std::sort(snapshots.begin(), snapshots.end(), + [](const Snapshot& a, const Snapshot& b) { return a.Id() < b.Id(); }); + std::vector result; + result.reserve(snapshots.size()); + for (const auto& snapshot : snapshots) { + result.push_back(snapshot.ToSnapshotInfo()); + } + return result; +} + +} // namespace paimon diff --git a/src/paimon/rest/rest_catalog.h b/src/paimon/rest/rest_catalog.h new file mode 100644 index 00000000..984aad70 --- /dev/null +++ b/src/paimon/rest/rest_catalog.h @@ -0,0 +1,101 @@ +/* + * Copyright 2026-present Alibaba Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include +#include +#include +#include + +#include "paimon/catalog/catalog.h" +#include "paimon/logging.h" +#include "paimon/rest/rest_api.h" +#include "paimon/result.h" +#include "paimon/status.h" + +struct ArrowSchema; + +namespace paimon { +class FileSystem; +class TableSchema; + +/// A catalog backed by a REST catalog server. Metadata operations are delegated to +/// `RestApi`; table data is accessed through the file system configured by the +/// server-merged options. +class RestCatalog : public Catalog { + public: + /// Creates the catalog: fetches and merges "/v1/config" from the server configured + /// by `CatalogOptions::URI`, then builds the file system from the merged options. + /// + /// @param warehouse The warehouse identifier sent to the server; may be empty. + static Result> Create( + const std::string& warehouse, const std::map& options, + const std::shared_ptr& file_system, + const RestHttpClient::Config& http_config = RestHttpClient::Config()); + + Status CreateDatabase(const std::string& name, + const std::map& options, + bool ignore_if_exists) override; + Status CreateTable(const Identifier& identifier, ArrowSchema* c_schema, + const std::vector& partition_keys, + const std::vector& primary_keys, + const std::map& options, + bool ignore_if_exists) override; + Status DropDatabase(const std::string& name, bool ignore_if_not_exists, bool cascade) override; + Status DropTable(const Identifier& identifier, bool ignore_if_not_exists) override; + Status RenameTable(const Identifier& from_table, const Identifier& to_table, + bool ignore_if_not_exists) override; + Result> ListDatabases() const override; + Result> ListTables(const std::string& db_name) const override; + Result DatabaseExists(const std::string& db_name) const override; + Result TableExists(const Identifier& identifier) const override; + std::string GetDatabaseLocation(const std::string& db_name) const override; + Result GetTableLocation(const Identifier& identifier) const override; + Result> LoadTableSchema(const Identifier& identifier) const override; + std::string GetRootPath() const override; + std::shared_ptr GetFileSystem() const override; + Result> GetTable(const Identifier& identifier) const override; + Result> ListSnapshots(const Identifier& identifier, + const std::string& branch) const override; + + /// Options merged with the server side config. + const std::map& GetOptions() const override; + + private: + RestCatalog(std::unique_ptr api, const std::shared_ptr& fs, + const std::string& warehouse); + + /// Loads the table from the server and converts the response to a `TableSchema` + /// (options are enriched with the table path, audit info and branch). + Result> LoadDataTableSchema( + const Identifier& data_identifier, const std::optional& branch, + std::string* table_path) const; + + static Result> ToTableSchema( + const GetTableResponse& response, const std::optional& branch); + + std::unique_ptr api_; + std::shared_ptr fs_; + std::string warehouse_; + /// The "table-default." options of the merged config, applied to `CreateTable` + /// options when absent. + std::map table_default_options_; + std::shared_ptr logger_; +}; + +} // namespace paimon diff --git a/src/paimon/rest/rest_catalog_test.cpp b/src/paimon/rest/rest_catalog_test.cpp new file mode 100644 index 00000000..242104fb --- /dev/null +++ b/src/paimon/rest/rest_catalog_test.cpp @@ -0,0 +1,1047 @@ +/* + * Copyright 2026-present Alibaba Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "paimon/rest/rest_catalog.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "arrow/api.h" +#include "arrow/c/bridge.h" +#include "fmt/format.h" +#include "gtest/gtest.h" +#include "paimon/catalog/catalog.h" +#include "paimon/catalog/table.h" +#include "paimon/catalog_options.h" +#include "paimon/common/utils/string_utils.h" +#include "paimon/core/schema/table_schema.h" +#include "paimon/defs.h" +#include "paimon/rest/mock_rest_server.h" +#include "paimon/rest/rest_api.h" +#include "paimon/schema/schema.h" +#include "paimon/testing/utils/testharness.h" + +namespace paimon::test { + +namespace { + +constexpr const char kToken[] = "test-token"; +constexpr const char kPrefix[] = "paimon"; +constexpr const char kWarehouse[] = "wh1"; + +// The in-memory catalog state behind the mock rest server. +struct MockCatalogState { + struct TableData { + std::string schema_json; + int64_t schema_id = 0; + std::string path; + }; + std::map> databases; + // headers of the last request, with lower-cased names + std::map last_headers; + // when set, every request except "/v1/config" fails with this http code + std::optional force_error_code; + // guards all fields above: the handler runs on the server's accept thread while + // tests seed and inspect the state + std::mutex mutex; +}; + +MockRestServer::Response JsonResponse(int32_t code, const std::string& body) { + MockRestServer::Response response; + response.code = code; + response.body = body; + return response; +} + +MockRestServer::Response MockError(int32_t code, const std::string& resource_type, + const std::string& resource_name, const std::string& message) { + ErrorResponse error(resource_type, resource_name, message, code); + return JsonResponse(code, error.ToJsonString().value()); +} + +std::string SnapshotJson(int64_t id) { + return fmt::format( + R"({{"version":3,"id":{},"schemaId":0,"baseManifestList":"bml","deltaManifestList":"dml",)" + R"("commitUser":"user1","commitIdentifier":1,"commitKind":"APPEND","timeMillis":100,)" + R"("totalRecordCount":10,"deltaRecordCount":1}})", + id); +} + +std::string TableResponseJson(const std::string& name, const MockCatalogState::TableData& table) { + return fmt::format( + R"({{"id":"1","name":"{}","path":"{}","isExternal":false,"schemaId":{},"schema":{},)" + R"("owner":"owner1","updatedAt":123}})", + name, table.path, table.schema_id, table.schema_json); +} + +// Serves `names` one item per page to exercise the pagination loop of the client. +std::pair, std::optional> PageOf( + const std::vector& names, const MockRestServer::Request& request) { + size_t index = 0; + auto token_iter = request.query_params.find(RestApi::kQueryParamPageToken); + if (token_iter != request.query_params.end()) { + // A malformed token must not throw: an exception on the accept thread would + // terminate the test binary. + index = static_cast( + StringUtils::StringToValue(token_iter->second).value_or(0)); + } + std::vector page; + std::optional next_page_token; + if (index < names.size()) { + page.push_back(names[index]); + if (index + 1 < names.size()) { + next_page_token = std::to_string(index + 1); + } + } + return {page, next_page_token}; +} + +// Implements the subset of the rest catalog protocol used by `RestCatalog` on top of +// `MockCatalogState`. +MockRestServer::Response HandleCatalogRequest(MockCatalogState* state, + const MockRestServer::Request& request) { + std::lock_guard lock(state->mutex); + state->last_headers = request.headers; + auto auth_iter = request.headers.find("authorization"); + if (auth_iter == request.headers.end() || + auth_iter->second != std::string("Bearer ") + kToken) { + return MockError(401, "", "", "invalid token"); + } + if (request.path == "/v1/config") { + auto warehouse_iter = request.query_params.find("warehouse"); + if (warehouse_iter == request.query_params.end() || warehouse_iter->second != kWarehouse) { + return MockError(400, "", "", "unexpected warehouse"); + } + ConfigResponse config( + {{RestApi::kOptionUrlPrefix, kPrefix}, + {"header.x-server-header", "from-config"}, + {"table-default.write-only", "true"}, + {"table-default.bucket", "8"}}, + {{"server-override", "from-server"}, {"header.x-shared-header", "from-config"}}); + return JsonResponse(200, config.ToJsonString().value()); + } + if (state->force_error_code) { + return MockError(state->force_error_code.value(), "", "", "injected failure"); + } + const std::string base = std::string("/v1/") + kPrefix; + if (request.path.rfind(base, 0) != 0) { + return MockError(404, "", "", "unknown path " + request.path); + } + std::string rest = request.path.substr(base.size()); + + if (rest == "/databases") { + if (request.method == "GET") { + std::vector names; + for (const auto& [name, tables] : state->databases) { + names.push_back(name); + } + auto [page, next_page_token] = PageOf(names, request); + ListDatabasesResponse response(page, next_page_token); + return JsonResponse(200, response.ToJsonString().value()); + } + if (request.method == "POST") { + CreateDatabaseRequest create_request("", {}); + if (!RapidJsonUtil::FromJsonString(request.body, &create_request).ok()) { + return MockError(400, "", "", "bad create database request"); + } + if (state->databases.count(create_request.GetName()) > 0) { + return MockError(409, ErrorResponse::kResourceTypeDatabase, + create_request.GetName(), "database already exists"); + } + state->databases[create_request.GetName()] = {}; + return JsonResponse(200, ""); + } + return MockError(400, "", "", "unsupported method"); + } + + if (rest == "/tables/rename" && request.method == "POST") { + RenameTableRequest rename_request("", "", "", ""); + if (!RapidJsonUtil::FromJsonString(request.body, &rename_request).ok()) { + return MockError(400, "", "", "bad rename table request"); + } + auto db_iter = state->databases.find(rename_request.GetSourceDatabase()); + if (db_iter == state->databases.end() || + db_iter->second.count(rename_request.GetSourceTable()) == 0) { + return MockError(404, ErrorResponse::kResourceTypeTable, + rename_request.GetSourceTable(), "table not found"); + } + auto& dest_tables = state->databases[rename_request.GetDestinationDatabase()]; + if (dest_tables.count(rename_request.GetDestinationTable()) > 0) { + return MockError(409, ErrorResponse::kResourceTypeTable, + rename_request.GetDestinationTable(), "table already exists"); + } + dest_tables[rename_request.GetDestinationTable()] = + db_iter->second[rename_request.GetSourceTable()]; + db_iter->second.erase(rename_request.GetSourceTable()); + return JsonResponse(200, ""); + } + + const std::string databases_prefix = "/databases/"; + if (rest.rfind(databases_prefix, 0) != 0) { + return MockError(404, "", "", "unknown path " + request.path); + } + std::string remainder = rest.substr(databases_prefix.size()); + size_t tables_pos = remainder.find("/tables"); + + if (tables_pos == std::string::npos) { + const std::string& db_name = remainder; + auto db_iter = state->databases.find(db_name); + if (request.method == "GET") { + if (db_iter == state->databases.end()) { + return MockError(404, ErrorResponse::kResourceTypeDatabase, db_name, + "database not found"); + } + std::string body = fmt::format( + R"({{"id":"1","name":"{}","location":"{}/{}.db","options":{{"dbk":"dbv"}}}})", + db_name, kWarehouse, db_name); + return JsonResponse(200, body); + } + if (request.method == "DELETE") { + if (db_iter == state->databases.end()) { + return MockError(404, ErrorResponse::kResourceTypeDatabase, db_name, + "database not found"); + } + state->databases.erase(db_iter); + return JsonResponse(200, ""); + } + return MockError(400, "", "", "unsupported method"); + } + + std::string db_name = remainder.substr(0, tables_pos); + auto db_iter = state->databases.find(db_name); + if (db_iter == state->databases.end()) { + return MockError(404, ErrorResponse::kResourceTypeDatabase, db_name, "database not found"); + } + auto& tables = db_iter->second; + std::string table_part = remainder.substr(tables_pos + std::strlen("/tables")); + + if (table_part.empty()) { + if (request.method == "GET") { + std::vector names; + for (const auto& [name, table] : tables) { + names.push_back(name); + } + auto [page, next_page_token] = PageOf(names, request); + ListTablesResponse response(page, next_page_token); + return JsonResponse(200, response.ToJsonString().value()); + } + if (request.method == "POST") { + CreateTableRequest create_request("", "", ""); + if (!RapidJsonUtil::FromJsonString(request.body, &create_request).ok()) { + return MockError(400, "", "", "bad create table request"); + } + if (tables.count(create_request.GetTable()) > 0) { + return MockError(409, ErrorResponse::kResourceTypeTable, create_request.GetTable(), + "table already exists"); + } + MockCatalogState::TableData table; + table.schema_json = create_request.GetSchemaJson(); + table.schema_id = 0; + table.path = fmt::format("{}/{}.db/{}", kWarehouse, db_name, create_request.GetTable()); + tables[create_request.GetTable()] = table; + return JsonResponse(200, ""); + } + return MockError(400, "", "", "unsupported method"); + } + + // "/{table}" or "/{table}/snapshots" + std::string table_name = table_part.substr(1); + bool list_snapshots = false; + const std::string snapshots_suffix = "/snapshots"; + if (table_name.size() > snapshots_suffix.size() && + table_name.compare(table_name.size() - snapshots_suffix.size(), snapshots_suffix.size(), + snapshots_suffix) == 0) { + table_name = table_name.substr(0, table_name.size() - snapshots_suffix.size()); + list_snapshots = true; + } + auto table_iter = tables.find(table_name); + if (table_iter == tables.end()) { + return MockError(404, ErrorResponse::kResourceTypeTable, table_name, "table not found"); + } + if (list_snapshots) { + // two pages, out of order to exercise pagination and sorting + auto token_iter = request.query_params.find(RestApi::kQueryParamPageToken); + if (token_iter == request.query_params.end()) { + return JsonResponse( + 200, fmt::format(R"({{"snapshots":[{}],"nextPageToken":"1"}})", SnapshotJson(2))); + } + return JsonResponse(200, fmt::format(R"({{"snapshots":[{}]}})", SnapshotJson(1))); + } + if (request.method == "GET") { + return JsonResponse(200, TableResponseJson(table_name, table_iter->second)); + } + if (request.method == "DELETE") { + tables.erase(table_iter); + return JsonResponse(200, ""); + } + return MockError(400, "", "", "unsupported method"); +} + +} // namespace + +class RestCatalogTest : public ::testing::Test { + protected: + void SetUp() override { + state_ = std::make_shared(); + ASSERT_OK_AND_ASSIGN( + server_, MockRestServer::Start([state = state_](const MockRestServer::Request& req) { + return HandleCatalogRequest(state.get(), req); + })); + options_ = { + {CatalogOptions::METASTORE, "rest"}, + {CatalogOptions::URI, server_->GetBaseUri()}, + {CatalogOptions::TOKEN_PROVIDER, "bear"}, + {CatalogOptions::TOKEN, kToken}, + {Options::FILE_SYSTEM, "local"}, + // mock_format is linked statically into the test binary, so its factory is + // registered in the binary's own registry even when the real format plugin + // dylibs register into a different one (macOS two-level namespace) + {Options::FILE_FORMAT, "mock_format"}, + {Options::MANIFEST_FORMAT, "mock_format"}, + {"header.x-client-header", "from-client"}, + {"header.x-shared-header", "from-client"}, + }; + } + + void TearDown() override { + if (server_) { + server_->Stop(); + } + } + + Result> CreateRestCatalog() { + return RestCatalog::Create(kWarehouse, options_, nullptr); + } + + Status CreateSampleTable(Catalog* catalog, const Identifier& identifier, + bool ignore_if_exists = false) { + std::shared_ptr schema = + arrow::schema({arrow::field("f0", arrow::int32(), /*nullable=*/false), + arrow::field("f1", arrow::utf8())}); + struct ArrowSchema c_schema; + if (!arrow::ExportSchema(*schema, &c_schema).ok()) { + return Status::Invalid("failed to export arrow schema"); + } + Status status = + catalog->CreateTable(identifier, &c_schema, /*partition_keys=*/{}, + /*primary_keys=*/{"f0"}, {{"bucket", "2"}}, ignore_if_exists); + // CreateTable takes ownership of the exported schema only once it reaches + // arrow::ImportSchema, which an identifier rejected by its checks never does + if (c_schema.release != nullptr) { + c_schema.release(&c_schema); + } + return status; + } + + // Seeds `schema_json` as table `table_name` of "db1" behind the mock server and + // expects loading the table to fail with an Invalid status carrying + // `expected_message`. + void ExpectBrokenSchemaRejected(Catalog* catalog, const std::string& table_name, + const std::string& schema_json, + const std::string& expected_message) { + MockCatalogState::TableData table_data; + table_data.schema_json = schema_json; + table_data.path = "wh1/db1.db/" + table_name; + { + std::lock_guard lock(state_->mutex); + state_->databases["db1"][table_name] = table_data; + } + Status status = catalog->GetTable(Identifier("db1", table_name)).status(); + ASSERT_TRUE(status.IsInvalid()) << status.ToString(); + ASSERT_NOK_WITH_MSG(status, expected_message); + } + + std::shared_ptr state_; + std::unique_ptr server_; + std::map options_; +}; + +TEST_F(RestCatalogTest, CreateMergesServerConfig) { + ASSERT_OK_AND_ASSIGN(std::unique_ptr catalog, CreateRestCatalog()); + const std::map& merged = catalog->GetOptions(); + ASSERT_EQ(kPrefix, merged.at(RestApi::kOptionUrlPrefix)); + ASSERT_EQ("from-server", merged.at("server-override")); + ASSERT_EQ(kWarehouse, catalog->GetRootPath()); + ASSERT_NE(nullptr, catalog->GetFileSystem()); +} + +TEST_F(RestCatalogTest, CatalogFactoryMetastoreDispatch) { + ASSERT_OK_AND_ASSIGN(std::unique_ptr catalog, Catalog::Create(kWarehouse, options_)); + ASSERT_OK_AND_ASSIGN(std::vector databases, catalog->ListDatabases()); + ASSERT_TRUE(databases.empty()); + options_[CatalogOptions::METASTORE] = "something-else"; + Status status = Catalog::Create(kWarehouse, options_).status(); + ASSERT_TRUE(status.IsInvalid()) << status.ToString(); + ASSERT_NOK_WITH_MSG(status, "unsupported metastore"); +} + +TEST_F(RestCatalogTest, CreateWithWrongTokenFails) { + options_[CatalogOptions::TOKEN] = "wrong-token"; + ASSERT_NOK_WITH_MSG(CreateRestCatalog().status(), "not authorized"); +} + +TEST_F(RestCatalogTest, CreateRejectsInvalidOptions) { + // all rejected by client side validation, before any request reaches the server + const std::map valid_options = options_; + + options_.erase(CatalogOptions::URI); + ASSERT_NOK_WITH_MSG(CreateRestCatalog().status(), "'uri' must be configured"); + + options_ = valid_options; + options_.erase(CatalogOptions::TOKEN_PROVIDER); + ASSERT_NOK_WITH_MSG(CreateRestCatalog().status(), "'token.provider' must be configured"); + + options_ = valid_options; + options_[CatalogOptions::TOKEN_PROVIDER] = "dlf"; + Status unsupported_provider = CreateRestCatalog().status(); + ASSERT_TRUE(unsupported_provider.IsNotImplemented()) << unsupported_provider.ToString(); + ASSERT_NOK_WITH_MSG(unsupported_provider, "unsupported token provider"); + + options_ = valid_options; + options_.erase(CatalogOptions::TOKEN); + ASSERT_NOK_WITH_MSG(CreateRestCatalog().status(), "bear token provider"); +} + +TEST_F(RestCatalogTest, DatabaseOperations) { + ASSERT_OK_AND_ASSIGN(std::unique_ptr catalog, CreateRestCatalog()); + + ASSERT_OK(catalog->CreateDatabase("db1", {}, /*ignore_if_exists=*/false)); + ASSERT_OK(catalog->CreateDatabase("db2", {}, /*ignore_if_exists=*/false)); + Status duplicated = catalog->CreateDatabase("db1", {}, /*ignore_if_exists=*/false); + ASSERT_TRUE(duplicated.IsExist()) << duplicated.ToString(); + ASSERT_OK(catalog->CreateDatabase("db1", {}, /*ignore_if_exists=*/true)); + + // the mock server returns one database per page + ASSERT_OK_AND_ASSIGN(std::vector databases, catalog->ListDatabases()); + ASSERT_EQ((std::vector{"db1", "db2"}), databases); + + ASSERT_OK_AND_ASSIGN(bool exists, catalog->DatabaseExists("db1")); + ASSERT_TRUE(exists); + ASSERT_OK_AND_ASSIGN(exists, catalog->DatabaseExists("db3")); + ASSERT_FALSE(exists); + + ASSERT_EQ("wh1/db1.db", catalog->GetDatabaseLocation("db1")); + ASSERT_EQ("", catalog->GetDatabaseLocation("db3")); + + ASSERT_OK(catalog->DropDatabase("db2", /*ignore_if_not_exists=*/false, /*cascade=*/false)); + ASSERT_OK(catalog->DropDatabase("db2", /*ignore_if_not_exists=*/true, /*cascade=*/false)); + Status missing = catalog->DropDatabase("db2", /*ignore_if_not_exists=*/false, + /*cascade=*/false); + ASSERT_TRUE(missing.IsNotExist()) << missing.ToString(); + + ASSERT_OK(CreateSampleTable(catalog.get(), Identifier("db1", "t1"))); + ASSERT_NOK_WITH_MSG( + catalog->DropDatabase("db1", /*ignore_if_not_exists=*/false, /*cascade=*/false), + "non-empty database"); + // cascade drop skips the emptiness check + ASSERT_OK(catalog->DropDatabase("db1", /*ignore_if_not_exists=*/false, /*cascade=*/true)); + ASSERT_OK_AND_ASSIGN(exists, catalog->DatabaseExists("db1")); + ASSERT_FALSE(exists); +} + +TEST_F(RestCatalogTest, TableOperations) { + ASSERT_OK_AND_ASSIGN(std::unique_ptr catalog, CreateRestCatalog()); + ASSERT_OK(catalog->CreateDatabase("db1", {}, /*ignore_if_exists=*/false)); + Identifier identifier("db1", "t1"); + + Status missing_db = CreateSampleTable(catalog.get(), Identifier("db_missing", "t1")); + ASSERT_TRUE(missing_db.IsNotExist()) << missing_db.ToString(); + + ASSERT_OK(CreateSampleTable(catalog.get(), identifier)); + Status duplicated = CreateSampleTable(catalog.get(), identifier); + ASSERT_TRUE(duplicated.IsExist()) << duplicated.ToString(); + ASSERT_OK(CreateSampleTable(catalog.get(), identifier, /*ignore_if_exists=*/true)); + + ASSERT_OK_AND_ASSIGN(std::vector tables, catalog->ListTables("db1")); + ASSERT_EQ((std::vector{"t1"}), tables); + Status list_missing = catalog->ListTables("db_missing").status(); + ASSERT_TRUE(list_missing.IsNotExist()) << list_missing.ToString(); + + ASSERT_OK_AND_ASSIGN(bool exists, catalog->TableExists(identifier)); + ASSERT_TRUE(exists); + ASSERT_OK_AND_ASSIGN(exists, catalog->TableExists(Identifier("db1", "t2"))); + ASSERT_FALSE(exists); + + ASSERT_OK_AND_ASSIGN(std::string location, catalog->GetTableLocation(identifier)); + ASSERT_EQ("wh1/db1.db/t1", location); + + ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, catalog->GetTable(identifier)); + ASSERT_EQ("t1", table->Name()); + std::shared_ptr schema = + std::dynamic_pointer_cast(table->LatestSchema()); + ASSERT_NE(nullptr, schema); + ASSERT_EQ((std::vector{"f0", "f1"}), schema->FieldNames()); + ASSERT_EQ((std::vector{"f0"}), schema->PrimaryKeys()); + ASSERT_EQ(0, schema->Id()); + // options are enriched with the table path and audit info from the server + ASSERT_EQ("wh1/db1.db/t1", schema->Options().at("path")); + ASSERT_EQ("owner1", schema->Options().at("owner")); + + // "table-default." options of the merged config apply only where the caller left the + // option unset: "write-only" is taken from the config, "bucket" keeps the value passed + // to CreateTable instead of the configured "table-default.bucket" of 8 + ASSERT_EQ("true", schema->Options().at("write-only")); + ASSERT_EQ("2", schema->Options().at("bucket")); + + // timeMillis is backed by the server's audit "updatedAt" instead of the current + // time, keeping the conversion deterministic + std::shared_ptr table_schema = + std::dynamic_pointer_cast(table->LatestSchema()); + ASSERT_NE(nullptr, table_schema); + ASSERT_EQ(123, table_schema->TimeMillis()); + + ASSERT_OK_AND_ASSIGN(std::shared_ptr loaded_schema, + catalog->LoadTableSchema(identifier)); + ASSERT_EQ((std::vector{"f0", "f1"}), loaded_schema->FieldNames()); + Status schema_missing = catalog->LoadTableSchema(Identifier("db1", "t2")).status(); + ASSERT_TRUE(schema_missing.IsNotExist()) << schema_missing.ToString(); + + // a schema comment of the server response is carried into the table schema + MockCatalogState::TableData commented; + commented.schema_json = R"({"fields": [{"id": 0, "name": "f0", "type": "INT NOT NULL"}],)" + R"( "partitionKeys": [], "primaryKeys": [], "options": {},)" + R"( "comment": "a table comment"})"; + commented.path = "wh1/db1.db/commented"; + { + std::lock_guard lock(state_->mutex); + state_->databases["db1"]["commented"] = commented; + } + ASSERT_OK_AND_ASSIGN(std::shared_ptr
commented_table, + catalog->GetTable(Identifier("db1", "commented"))); + ASSERT_EQ("a table comment", commented_table->LatestSchema()->Comment().value_or("")); + + ASSERT_OK(catalog->RenameTable(identifier, Identifier("db1", "t2"), + /*ignore_if_not_exists=*/false)); + ASSERT_OK_AND_ASSIGN(exists, catalog->TableExists(Identifier("db1", "t2"))); + ASSERT_TRUE(exists); + ASSERT_OK(catalog->RenameTable(identifier, Identifier("db1", "t3"), + /*ignore_if_not_exists=*/true)); + Status rename_missing = catalog->RenameTable(identifier, Identifier("db1", "t3"), + /*ignore_if_not_exists=*/false); + ASSERT_TRUE(rename_missing.IsNotExist()) << rename_missing.ToString(); + + ASSERT_OK(catalog->DropTable(Identifier("db1", "t2"), /*ignore_if_not_exists=*/false)); + ASSERT_OK(catalog->DropTable(Identifier("db1", "t2"), /*ignore_if_not_exists=*/true)); + Status drop_missing = catalog->DropTable(Identifier("db1", "t2"), + /*ignore_if_not_exists=*/false); + ASSERT_TRUE(drop_missing.IsNotExist()) << drop_missing.ToString(); +} + +TEST_F(RestCatalogTest, ClientAndServerHeadersAreSent) { + ASSERT_OK_AND_ASSIGN(std::unique_ptr catalog, CreateRestCatalog()); + ASSERT_OK_AND_ASSIGN(std::vector databases, catalog->ListDatabases()); + // "header." options from both the client and the merged server config are sent as + // http headers on every request + { + std::lock_guard lock(state_->mutex); + ASSERT_EQ("from-client", state_->last_headers.at("x-client-header")); + ASSERT_EQ("from-config", state_->last_headers.at("x-server-header")); + // when the client and the server config set the same "header." option, the + // merged config wins (overrides > client options > defaults) + ASSERT_EQ("from-config", state_->last_headers.at("x-shared-header")); + ASSERT_EQ(std::string("Bearer ") + kToken, state_->last_headers.at("authorization")); + } + // a request carrying a body declares the json content type (set before the auth + // headers are merged, so a signing auth provider covers it) + ASSERT_OK(catalog->CreateDatabase("db1", {}, /*ignore_if_exists=*/false)); + { + std::lock_guard lock(state_->mutex); + ASSERT_EQ("application/json", state_->last_headers.at("content-type")); + } +} + +TEST_F(RestCatalogTest, SystemTableSchema) { + ASSERT_OK_AND_ASSIGN(std::unique_ptr catalog, CreateRestCatalog()); + ASSERT_OK(catalog->CreateDatabase("db1", {}, /*ignore_if_exists=*/false)); + ASSERT_OK(CreateSampleTable(catalog.get(), Identifier("db1", "t1"))); + + // the "options" system table has a static schema and needs no file system access + Identifier system_identifier("db1", "t1$options"); + ASSERT_OK_AND_ASSIGN(bool exists, catalog->TableExists(system_identifier)); + ASSERT_TRUE(exists); + ASSERT_OK_AND_ASSIGN(exists, catalog->TableExists(Identifier("db1", "t9$options"))); + ASSERT_FALSE(exists); + ASSERT_OK_AND_ASSIGN(exists, catalog->TableExists(Identifier("db1", "t1$unsupported"))); + ASSERT_FALSE(exists); + + ASSERT_OK_AND_ASSIGN(std::shared_ptr schema, + catalog->LoadTableSchema(system_identifier)); + ASSERT_EQ((std::vector{"key", "value"}), schema->FieldNames()); + + ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, catalog->GetTable(system_identifier)); + ASSERT_EQ("t1$options", table->Name()); + ASSERT_EQ((std::vector{"key", "value"}), table->LatestSchema()->FieldNames()); + + Status unsupported = catalog->LoadTableSchema(Identifier("db1", "t1$unsupported")).status(); + ASSERT_TRUE(unsupported.IsNotExist()) << unsupported.ToString(); +} + +TEST_F(RestCatalogTest, BranchTableLoadsBranchSchemaFromServer) { + ASSERT_OK_AND_ASSIGN(std::unique_ptr catalog, CreateRestCatalog()); + ASSERT_OK(catalog->CreateDatabase("db1", {}, /*ignore_if_exists=*/false)); + ASSERT_OK(CreateSampleTable(catalog.get(), Identifier("db1", "t1"))); + + // the identifier sent to the server keeps the branch, so the server resolves it and + // returns the branch's own schema. The path it reports is the data table root, not + // the branch subdirectory: readers derive "/branch/branch-" from the + // branch option, so a branch path here would be applied twice + MockCatalogState::TableData branch_data; + branch_data.schema_json = R"({"fields": [{"id": 0, "name": "b0", "type": "INT NOT NULL"}],)" + R"( "partitionKeys": [], "primaryKeys": [], "options": {}})"; + branch_data.schema_id = 3; + branch_data.path = "wh1/db1.db/t1"; + { + std::lock_guard lock(state_->mutex); + state_->databases["db1"]["t1$branch_b1"] = branch_data; + } + + Identifier branch_identifier("db1", "t1$branch_b1"); + ASSERT_OK_AND_ASSIGN(bool exists, catalog->TableExists(branch_identifier)); + ASSERT_TRUE(exists); + // a branch the server does not know is missing instead of silently falling back to + // the main table + ASSERT_OK_AND_ASSIGN(exists, catalog->TableExists(Identifier("db1", "t1$branch_missing"))); + ASSERT_FALSE(exists); + + ASSERT_OK_AND_ASSIGN(std::string location, catalog->GetTableLocation(branch_identifier)); + ASSERT_EQ("wh1/db1.db/t1", location); + + ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, catalog->GetTable(branch_identifier)); + ASSERT_EQ("t1$branch_b1", table->Name()); + std::shared_ptr schema = + std::dynamic_pointer_cast(table->LatestSchema()); + ASSERT_NE(nullptr, schema); + // the branch's own schema and schema id, not the main table's + ASSERT_EQ((std::vector{"b0"}), schema->FieldNames()); + ASSERT_EQ(3, schema->Id()); + ASSERT_EQ("b1", schema->Options().at(Options::BRANCH)); + + // the default branch is addressed as the bare table: "t1$branch_main" resolves to + // "t1" and carries no branch option + ASSERT_OK_AND_ASSIGN(std::shared_ptr
main_table, + catalog->GetTable(Identifier("db1", "t1$branch_main"))); + std::shared_ptr main_schema = + std::dynamic_pointer_cast(main_table->LatestSchema()); + ASSERT_NE(nullptr, main_schema); + ASSERT_EQ((std::vector{"f0", "f1"}), main_schema->FieldNames()); + ASSERT_EQ(0, main_schema->Options().count(Options::BRANCH)); + + // the default branch is matched ignoring case, as in the Java client, so + // "t1$branch_MAIN" addresses the bare table too + ASSERT_OK_AND_ASSIGN(std::shared_ptr
main_case_table, + catalog->GetTable(Identifier("db1", "t1$branch_MAIN"))); + std::shared_ptr main_case_schema = + std::dynamic_pointer_cast(main_case_table->LatestSchema()); + ASSERT_NE(nullptr, main_case_schema); + ASSERT_EQ((std::vector{"f0", "f1"}), main_case_schema->FieldNames()); + ASSERT_EQ(0, main_case_schema->Options().count(Options::BRANCH)); + + // a system table on a branch resolves against the branch's data table: the system + // suffix is stripped while the branch stays in the identifier sent to the server + ASSERT_OK_AND_ASSIGN(std::shared_ptr options_schema, + catalog->LoadTableSchema(Identifier("db1", "t1$branch_b1$options"))); + ASSERT_EQ((std::vector{"key", "value"}), options_schema->FieldNames()); + // a missing branch fails through the system table path too instead of silently + // resolving against the main table + Status missing_branch = + catalog->LoadTableSchema(Identifier("db1", "t1$branch_missing$options")).status(); + ASSERT_TRUE(missing_branch.IsNotExist()) << missing_branch.ToString(); + + ASSERT_NOK_WITH_MSG(catalog->DropTable(branch_identifier, /*ignore_if_not_exists=*/false), + "branch table"); + ASSERT_NOK_WITH_MSG(catalog->RenameTable(branch_identifier, Identifier("db1", "t2"), + /*ignore_if_not_exists=*/false), + "branch table"); + ASSERT_NOK_WITH_MSG(CreateSampleTable(catalog.get(), Identifier("db1", "t2$branch_b1")), + "branch table"); +} + +TEST_F(RestCatalogTest, NestedSchemaHighestFieldId) { + ASSERT_OK_AND_ASSIGN(std::unique_ptr catalog, CreateRestCatalog()); + ASSERT_OK(catalog->CreateDatabase("db1", {}, /*ignore_if_exists=*/false)); + MockCatalogState::TableData table_data; + table_data.schema_json = R"({ + "fields": [ + {"id": 0, "name": "f0", "type": "INT NOT NULL"}, + {"id": 1, "name": "s", "type": {"type": "ROW", + "fields": [{"id": 3, "name": "inner", "type": "INT"}]}}, + {"id": 2, "name": "arr", "type": {"type": "ARRAY", + "element": {"type": "ROW", + "fields": [{"id": 7, "name": "deep", "type": "BIGINT"}]}}}, + {"id": 4, "name": "m", "type": {"type": "MAP", + "key": {"type": "ROW NOT NULL", + "fields": [{"id": 8, "name": "k", "type": "INT NOT NULL"}]}, + "value": {"type": "ROW", + "fields": [{"id": 9, "name": "v", "type": "INT"}]}}} + ], + "partitionKeys": [], + "primaryKeys": [], + "options": {} + })"; + table_data.schema_id = 5; + table_data.path = "wh1/db1.db/nested"; + { + std::lock_guard lock(state_->mutex); + state_->databases["db1"]["nested"] = table_data; + } + + ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, + catalog->GetTable(Identifier("db1", "nested"))); + std::shared_ptr schema = + std::dynamic_pointer_cast(table->LatestSchema()); + ASSERT_NE(nullptr, schema); + ASSERT_EQ(5, schema->Id()); + // 9 lives inside the value row of the map: ROW, ARRAY element and MAP key/value + // must all be traversed + ASSERT_EQ(9, schema->HighestFieldId()); + ASSERT_EQ((std::vector{"f0", "s", "arr", "m"}), schema->FieldNames()); +} + +TEST_F(RestCatalogTest, ListTablesPaged) { + ASSERT_OK_AND_ASSIGN(std::unique_ptr catalog, CreateRestCatalog()); + ASSERT_OK(catalog->CreateDatabase("db1", {}, /*ignore_if_exists=*/false)); + ASSERT_OK(CreateSampleTable(catalog.get(), Identifier("db1", "t1"))); + ASSERT_OK(CreateSampleTable(catalog.get(), Identifier("db1", "t2"))); + ASSERT_OK(CreateSampleTable(catalog.get(), Identifier("db1", "t3"))); + ASSERT_OK_AND_ASSIGN(std::vector tables, catalog->ListTables("db1")); + ASSERT_EQ((std::vector{"t1", "t2", "t3"}), tables); +} + +TEST_F(RestCatalogTest, BrokenSchemaRejected) { + ASSERT_OK_AND_ASSIGN(std::unique_ptr catalog, CreateRestCatalog()); + ASSERT_OK(catalog->CreateDatabase("db1", {}, /*ignore_if_exists=*/false)); + ExpectBrokenSchemaRejected(catalog.get(), "duplicate", + R"({"fields": [{"id": 0, "name": "f0", "type": "INT NOT NULL"},)" + R"( {"id": 0, "name": "f1", "type": "STRING"}],)" + R"( "partitionKeys": [], "primaryKeys": [], "options": {}})", + "duplicated"); + // an id inside a nested row colliding with an outer field id is a duplicate too + ExpectBrokenSchemaRejected(catalog.get(), "duplicate_nested", + R"({"fields": [{"id": 0, "name": "f0", "type": "INT NOT NULL"},)" + R"( {"id": 1, "name": "s", "type": {"type": "ROW",)" + R"( "fields": [{"id": 0, "name": "inner", "type": "INT"}]}}],)" + R"( "partitionKeys": [], "primaryKeys": [], "options": {}})", + "duplicated"); + // a field without an integer id must fail the conversion instead of being + // silently skipped when computing highestFieldId + ExpectBrokenSchemaRejected(catalog.get(), "no_id", + R"({"fields": [{"name": "f0", "type": "INT NOT NULL"}],)" + R"( "partitionKeys": [], "primaryKeys": [], "options": {}})", + "integer id"); + ExpectBrokenSchemaRejected(catalog.get(), "string_id", + R"({"fields": [{"id": "0", "name": "f0", "type": "INT NOT NULL"}],)" + R"( "partitionKeys": [], "primaryKeys": [], "options": {}})", + "integer id"); + // a field that is not an object fails too instead of being silently skipped + ExpectBrokenSchemaRejected(catalog.get(), "non_object", + R"({"fields": [1],)" + R"( "partitionKeys": [], "primaryKeys": [], "options": {}})", + "must be an object"); + // a missing or wrong-typed member is a visible failure instead of silently + // defaulting to an empty value (e.g. loading a partitioned table as + // unpartitioned) + ExpectBrokenSchemaRejected(catalog.get(), "missing_partition_keys", + R"({"fields": [{"id": 0, "name": "f0", "type": "INT NOT NULL"}],)" + R"( "primaryKeys": [], "options": {}})", + "missing 'partitionKeys'"); + ExpectBrokenSchemaRejected(catalog.get(), "missing_primary_keys", + R"({"fields": [{"id": 0, "name": "f0", "type": "INT NOT NULL"}],)" + R"( "partitionKeys": [], "options": {}})", + "missing 'primaryKeys'"); + ExpectBrokenSchemaRejected(catalog.get(), "missing_options", + R"({"fields": [{"id": 0, "name": "f0", "type": "INT NOT NULL"}],)" + R"( "partitionKeys": [], "primaryKeys": []})", + "missing 'options'"); + ExpectBrokenSchemaRejected(catalog.get(), "wrong_typed_partition_keys", + R"({"fields": [{"id": 0, "name": "f0", "type": "INT NOT NULL"}],)" + R"( "partitionKeys": {}, "primaryKeys": [], "options": {}})", + "'partitionKeys' is not an array"); + ExpectBrokenSchemaRejected(catalog.get(), "wrong_typed_primary_keys", + R"({"fields": [{"id": 0, "name": "f0", "type": "INT NOT NULL"}],)" + R"( "partitionKeys": [], "primaryKeys": "f0", "options": {}})", + "'primaryKeys' is not an array"); + ExpectBrokenSchemaRejected(catalog.get(), "wrong_typed_options", + R"({"fields": [{"id": 0, "name": "f0", "type": "INT NOT NULL"}],)" + R"( "partitionKeys": [], "primaryKeys": [], "options": []})", + "'options' is not an object"); +} + +TEST_F(RestCatalogTest, PartitionKeysRoundTrip) { + ASSERT_OK_AND_ASSIGN(std::unique_ptr catalog, CreateRestCatalog()); + ASSERT_OK(catalog->CreateDatabase("db1", {}, /*ignore_if_exists=*/false)); + std::shared_ptr schema = + arrow::schema({arrow::field("f0", arrow::int32(), /*nullable=*/false), + arrow::field("f1", arrow::utf8(), /*nullable=*/false)}); + struct ArrowSchema c_schema; + ASSERT_TRUE(arrow::ExportSchema(*schema, &c_schema).ok()); + ASSERT_OK(catalog->CreateTable(Identifier("db1", "pt"), &c_schema, + /*partition_keys=*/{"f1"}, /*primary_keys=*/{}, {}, + /*ignore_if_exists=*/false)); + // the partition keys survive both the create request and the load response conversion + ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, catalog->GetTable(Identifier("db1", "pt"))); + std::shared_ptr loaded = + std::dynamic_pointer_cast(table->LatestSchema()); + ASSERT_NE(nullptr, loaded); + ASSERT_EQ((std::vector{"f1"}), loaded->PartitionKeys()); +} + +TEST_F(RestCatalogTest, ServerErrorIsPropagatedNotMappedToAbsent) { + ASSERT_OK_AND_ASSIGN(std::unique_ptr catalog, CreateRestCatalog()); + { + std::lock_guard lock(state_->mutex); + state_->force_error_code = 500; + } + // a server failure surfaces as an error instead of "does not exist" + Status db_status = catalog->DatabaseExists("db1").status(); + ASSERT_NOK_WITH_MSG(db_status, "server error"); + Status table_status = catalog->TableExists(Identifier("db1", "t1")).status(); + ASSERT_NOK_WITH_MSG(table_status, "server error"); +} + +TEST_F(RestCatalogTest, SystemTableChecks) { + ASSERT_OK_AND_ASSIGN(std::unique_ptr catalog, CreateRestCatalog()); + ASSERT_NOK_WITH_MSG(catalog->CreateDatabase("sys", {}, false), "system database"); + ASSERT_NOK_WITH_MSG(catalog->DropDatabase("sys", false, false), "system database"); + ASSERT_NOK_WITH_MSG(catalog->DropTable(Identifier("sys", "t"), false), "system table"); + ASSERT_NOK_WITH_MSG( + catalog->RenameTable(Identifier("db1", "t1$snapshots"), Identifier("db1", "t2"), false), + "system table"); +} + +TEST_F(RestCatalogTest, SystemDatabase) { + ASSERT_OK_AND_ASSIGN(std::unique_ptr catalog, CreateRestCatalog()); + // the "sys" database and its global system tables are resolved locally without + // contacting the server, like in FileSystemCatalog + ASSERT_OK_AND_ASSIGN(bool exists, catalog->DatabaseExists("sys")); + ASSERT_TRUE(exists); + + ASSERT_OK_AND_ASSIGN(std::vector sys_tables, catalog->ListTables("sys")); + ASSERT_TRUE(std::find(sys_tables.begin(), sys_tables.end(), "tables") != sys_tables.end()); + + ASSERT_OK_AND_ASSIGN(exists, catalog->TableExists(Identifier("sys", "tables"))); + ASSERT_TRUE(exists); + ASSERT_OK_AND_ASSIGN(exists, catalog->TableExists(Identifier("sys", "unsupported"))); + ASSERT_FALSE(exists); + + ASSERT_OK_AND_ASSIGN(std::shared_ptr schema, + catalog->LoadTableSchema(Identifier("sys", "tables"))); + ASSERT_FALSE(schema->FieldNames().empty()); + Status missing = catalog->LoadTableSchema(Identifier("sys", "unsupported")).status(); + ASSERT_TRUE(missing.IsNotExist()) << missing.ToString(); +} + +TEST_F(RestCatalogTest, ListSnapshots) { + ASSERT_OK_AND_ASSIGN(std::unique_ptr catalog, CreateRestCatalog()); + ASSERT_OK(catalog->CreateDatabase("db1", {}, /*ignore_if_exists=*/false)); + Identifier identifier("db1", "t1"); + ASSERT_OK(CreateSampleTable(catalog.get(), identifier)); + + ASSERT_OK_AND_ASSIGN(std::vector snapshots, + catalog->ListSnapshots(identifier, "")); + ASSERT_EQ(2, snapshots.size()); + // fetched via two pages and sorted by snapshot id + ASSERT_EQ(1, snapshots[0].snapshot_id); + ASSERT_EQ(2, snapshots[1].snapshot_id); + ASSERT_EQ("user1", snapshots[0].commit_user); + ASSERT_EQ(SnapshotInfo::CommitKind::APPEND, snapshots[0].commit_kind); + + // the default branch is addressed as the bare table, so passing it explicitly + // equals the branch-less call + ASSERT_OK_AND_ASSIGN(std::vector main_snapshots, + catalog->ListSnapshots(identifier, "main")); + ASSERT_EQ(2, main_snapshots.size()); + // the match ignores case, as in the Java client, so "MAIN" is the default branch too + ASSERT_OK_AND_ASSIGN(std::vector main_case_snapshots, + catalog->ListSnapshots(identifier, "MAIN")); + ASSERT_EQ(2, main_case_snapshots.size()); + + Status missing = catalog->ListSnapshots(Identifier("db1", "t9"), "").status(); + ASSERT_TRUE(missing.IsNotExist()) << missing.ToString(); + + // a non-main branch is sent under its branch object name, so the server resolves + // the branch and lists its own snapshots + MockCatalogState::TableData branch_data; + branch_data.schema_json = R"({"fields": [{"id": 0, "name": "f0", "type": "INT NOT NULL"}],)" + R"( "partitionKeys": [], "primaryKeys": [], "options": {}})"; + branch_data.path = "wh1/db1.db/t1"; + { + std::lock_guard lock(state_->mutex); + state_->databases["db1"]["t1$branch_b1"] = branch_data; + } + ASSERT_OK_AND_ASSIGN(std::vector branch_snapshots, + catalog->ListSnapshots(identifier, "b1")); + ASSERT_EQ(2, branch_snapshots.size()); + // a branch the server does not know is missing instead of silently falling back + // to the main table + Status missing_branch = catalog->ListSnapshots(identifier, "b_missing").status(); + ASSERT_TRUE(missing_branch.IsNotExist()) << missing_branch.ToString(); + + // a branch must be passed as the branch argument, not encoded in the identifier + ASSERT_NOK_WITH_MSG(catalog->ListSnapshots(Identifier("db1", "t1$branch_b1"), "").status(), + "branch table"); +} + +TEST(RestApiErrorTest, ErrorToStatus) { + RestHttpClient::Response response; + response.code = 404; + response.body = R"({"message": "no table", "resourceType": "TABLE", "resourceName": "t1"})"; + response.headers["x-request-id"] = "req-123"; + Status status = RestApi::ErrorToStatus(response); + ASSERT_TRUE(status.IsNotExist()); + ASSERT_TRUE(status.ToString().find("requestId:req-123") != std::string::npos) + << status.ToString(); + + response.code = 409; + Status exist_status = RestApi::ErrorToStatus(response); + ASSERT_TRUE(exist_status.IsExist()); + + response.code = 501; + response.body = ""; + ASSERT_TRUE(RestApi::ErrorToStatus(response).IsNotImplemented()); + + // a body that is not an error object at all is reported as such, since the body + // itself is never echoed + response.code = 500; + response.body = "not-a-json"; + Status unparsable = RestApi::ErrorToStatus(response); + ASSERT_NOK_WITH_MSG(unparsable, "server error"); + ASSERT_NOK_WITH_MSG(unparsable, "unparsable error response body (http status 500)"); + ASSERT_EQ(std::string::npos, unparsable.ToString().find("not-a-json")) << unparsable.ToString(); + + // the code of the error body wins over the http status when they disagree (e.g. a + // gateway rewriting the status) + response.code = 500; + response.body = R"({"message": "gone", "code": 404})"; + ASSERT_TRUE(RestApi::ErrorToStatus(response).IsNotExist()); + + // an error object without a message is told apart from an unparsable body, and the + // resource info is kept + response.code = 404; + response.body = R"({"resourceType": "TABLE", "resourceName": "t1"})"; + Status empty_message = RestApi::ErrorToStatus(response); + ASSERT_NOK_WITH_MSG(empty_message, "empty error message (http status 404)"); + ASSERT_TRUE(empty_message.ToString().find("resource name: t1") != std::string::npos) + << empty_message.ToString(); + + // server messages that may embed secrets are redacted as a whole + response.code = 400; + response.body = R"({"message": "bad option password=abc123", "code": 400})"; + Status redacted = RestApi::ErrorToStatus(response); + ASSERT_TRUE(redacted.IsInvalid()) << redacted.ToString(); + ASSERT_TRUE(redacted.ToString().find("abc123") == std::string::npos) << redacted.ToString(); + ASSERT_TRUE(redacted.ToString().find("******") != std::string::npos) << redacted.ToString(); + + // any header carrying a request id is used when x-request-id is absent + response.code = 404; + response.body = ""; + response.headers.clear(); + response.headers["x-amz-request-id"] = "amz-1"; + Status fallback = RestApi::ErrorToStatus(response); + ASSERT_TRUE(fallback.ToString().find("requestId:amz-1") != std::string::npos) + << fallback.ToString(); + + // the "unknown" placeholder is not a real request id + response.headers.clear(); + response.headers["x-request-id"] = "unknown"; + Status unknown_id = RestApi::ErrorToStatus(response); + ASSERT_TRUE(unknown_id.ToString().find("requestId") == std::string::npos) + << unknown_id.ToString(); + + // 401/403 map to IOError; the mapped code is carried as a status detail so + // callers can distinguish them + response.headers.clear(); + response.code = 401; + Status not_authorized = RestApi::ErrorToStatus(response); + ASSERT_NOK_WITH_MSG(not_authorized, "not authorized"); + ASSERT_NE(nullptr, not_authorized.detail()); + ASSERT_EQ(std::string(RestErrorDetail::kTypeId), not_authorized.detail()->type_id()); + ASSERT_EQ(401, std::static_pointer_cast(not_authorized.detail())->GetCode()); + response.code = 403; + Status forbidden = RestApi::ErrorToStatus(response); + ASSERT_NOK_WITH_MSG(forbidden, "forbidden"); + ASSERT_EQ(403, std::static_pointer_cast(forbidden.detail())->GetCode()); + + // 503 and the codes without an own mapping (e.g. 429) become IOError with a + // message naming the code + response.code = 503; + ASSERT_NOK_WITH_MSG(RestApi::ErrorToStatus(response), "service unavailable"); + response.code = 429; + ASSERT_NOK_WITH_MSG(RestApi::ErrorToStatus(response), "rest request failed with code 429"); + response.code = 418; + ASSERT_NOK_WITH_MSG(RestApi::ErrorToStatus(response), "rest request failed with code 418"); +} + +TEST(RestApiErrorTest, MalformedSuccessBodyFails) { + // a 200 response whose body is not the expected json must fail, not crash or + // return partial data + ASSERT_OK_AND_ASSIGN(std::unique_ptr server, + MockRestServer::Start([](const MockRestServer::Request& request) { + MockRestServer::Response response; + response.body = "not-a-json"; + return response; + })); + std::map options = { + {CatalogOptions::URI, server->GetBaseUri()}, + {CatalogOptions::TOKEN_PROVIDER, "bear"}, + {CatalogOptions::TOKEN, kToken}, + }; + ASSERT_OK_AND_ASSIGN(std::unique_ptr api, + RestApi::Create(options, "", /*config_required=*/false)); + Status list_status = api->ListDatabases().status(); + ASSERT_NOK(list_status); + // the body is not echoed into the error: a successful response may contain + // credentials + ASSERT_EQ(std::string::npos, list_status.ToString().find("not-a-json")) + << list_status.ToString(); + ASSERT_NOK(api->GetTable(Identifier("db1", "t1")).status()); + + Status config_status = RestApi::Create(options, "", /*config_required=*/true).status(); + ASSERT_NOK(config_status); + ASSERT_EQ(std::string::npos, config_status.ToString().find("not-a-json")) + << config_status.ToString(); +} + +TEST(RestApiErrorTest, PagedListingStopsOnEmptyPageWithToken) { + // a server that keeps returning a page token with no data must not loop forever + std::atomic request_count{0}; + ASSERT_OK_AND_ASSIGN(std::unique_ptr server, + MockRestServer::Start([&](const MockRestServer::Request& request) { + request_count++; + MockRestServer::Response response; + response.body = R"({"databases":[],"nextPageToken":"more"})"; + return response; + })); + std::map options = { + {CatalogOptions::URI, server->GetBaseUri()}, + {CatalogOptions::TOKEN_PROVIDER, "bear"}, + {CatalogOptions::TOKEN, kToken}, + }; + ASSERT_OK_AND_ASSIGN(std::unique_ptr api, + RestApi::Create(options, "", /*config_required=*/false)); + ASSERT_OK_AND_ASSIGN(std::vector databases, api->ListDatabases()); + ASSERT_TRUE(databases.empty()); + ASSERT_EQ(1, request_count.load()); +} + +} // namespace paimon::test diff --git a/src/paimon/rest/rest_http_client.cpp b/src/paimon/rest/rest_http_client.cpp new file mode 100644 index 00000000..b17b0a28 --- /dev/null +++ b/src/paimon/rest/rest_http_client.cpp @@ -0,0 +1,462 @@ +/* + * Copyright 2026-present Alibaba Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "paimon/rest/rest_http_client.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "curl/curl.h" +#include "fmt/format.h" +#include "paimon/common/utils/http_client.h" +#include "paimon/common/utils/scope_guard.h" +#include "paimon/common/utils/string_utils.h" +#include "paimon/common/utils/url_utils.h" +#include "paimon/rest/rest_util.h" + +namespace paimon { + +namespace { + +size_t WriteBodyCallback(char* data, size_t size, size_t nmemb, void* user_data) { + auto* body = static_cast(user_data); + body->append(data, size * nmemb); + return size * nmemb; +} + +size_t WriteHeaderCallback(char* data, size_t size, size_t nmemb, void* user_data) { + auto* headers = static_cast(user_data); + size_t total = size * nmemb; + // A status line starts the header block of the next response on the connection + // (e.g. after a followed redirect); dropping the previous headers leaves only the + // final response's. + if (total >= 5 && std::strncmp(data, "HTTP/", 5) == 0) { + headers->clear(); + } + ParseHttpHeaderLine(data, total, headers); + return total; +} + +// Only the methods `Execute` supports are classified; POST is the sole non-idempotent +// one among them. +bool IsIdempotent(const std::string& method) { + return method == "GET" || method == "DELETE"; +} + +bool IsRetriableCode(int64_t code) { + return code == HttpStatus::kTooManyRequests || code == HttpStatus::kServiceUnavailable; +} + +// Parses the RFC 1123 form of an http date, e.g. "Wed, 21 Oct 2015 07:28:00 GMT", into +// unix epoch seconds. Month names are matched against a fixed English table so the +// result does not depend on the process locale. +std::optional ParseHttpDateSeconds(const std::string& value) { + std::vector parts = StringUtils::Split(value, " ", /*ignore_empty=*/true); + if (parts.size() != 6 || parts[5] != "GMT" || parts[0].size() != 4 || parts[0][3] != ',') { + return std::nullopt; + } + static constexpr const char* kMonths[] = {"Jan", "Feb", "Mar", "Apr", "May", "Jun", + "Jul", "Aug", "Sep", "Oct", "Nov", "Dec"}; + int64_t month = 0; + for (size_t i = 0; i < 12; i++) { + if (parts[2] == kMonths[i]) { + month = static_cast(i) + 1; + break; + } + } + std::optional day = StringUtils::StringToValue(parts[1]); + std::optional year = StringUtils::StringToValue(parts[3]); + std::vector time_parts = StringUtils::Split(parts[4], ":", + /*ignore_empty=*/false); + if (month == 0 || !day || !year || time_parts.size() != 3) { + return std::nullopt; + } + std::optional hour = StringUtils::StringToValue(time_parts[0]); + std::optional minute = StringUtils::StringToValue(time_parts[1]); + std::optional second = StringUtils::StringToValue(time_parts[2]); + if (!hour || !minute || !second || day.value() < 1 || day.value() > 31 || year.value() < 1970 || + year.value() > 9999 || hour.value() > 23 || minute.value() > 59 || second.value() > 60) { + return std::nullopt; + } + // Days between 1970-01-01 and the given civil date ("days from civil" algorithm). + int64_t y = year.value() - (month <= 2 ? 1 : 0); + int64_t era = y / 400; + int64_t yoe = y - era * 400; + int64_t doy = (153 * (month + (month > 2 ? -3 : 9)) + 2) / 5 + day.value() - 1; + int64_t doe = yoe * 365 + yoe / 4 - yoe / 100 + doy; + int64_t days = era * 146097 + doe - 719468; + return days * 86400 + hour.value() * 3600 + minute.value() * 60 + second.value(); +} + +// Allowlist of the transient transport failures: an established connection breaking +// mid-request (send/receive failures, HTTP/2 stream errors) or a response truncated +// mid-body. The request reached a live server, so retrying an idempotent method is +// likely to succeed. Every other CURLcode (resolution/connect failures, timeouts, TLS +// failures, malformed urls, redirect loops, ...) is permanent and costs one attempt. +bool IsRetriableTransportError(CURLcode code) { + switch (code) { + case CURLE_SEND_ERROR: + case CURLE_RECV_ERROR: + case CURLE_PARTIAL_FILE: + case CURLE_HTTP2: + case CURLE_HTTP2_STREAM: + return true; + default: + return false; + } +} + +// libcurl sends no User-Agent of its own, so requests would reach the server unnamed. A +// "header.User-Agent" option still wins: an explicit request header overrides +// CURLOPT_USERAGENT. +constexpr const char kDefaultUserAgent[] = "paimon-cpp"; + +// A valid HTTP header field name per RFC 7230: one or more tchar. +bool IsValidHeaderName(const std::string& name) { + if (name.empty()) { + return false; + } + static constexpr char kTcharSymbols[] = "!#$%&'*+-.^_`|~"; + for (char c : name) { + if (c != '\0' && (std::isalnum(static_cast(c)) || + std::strchr(kTcharSymbols, c) != nullptr)) { + continue; + } + return false; + } + return true; +} + +// libcurl sends header strings verbatim, so a value containing CR, LF or NUL could +// inject additional header lines into the request. +bool IsValidHeaderValue(const std::string& value) { + return value.find_first_of("\r\n") == std::string::npos && + value.find('\0') == std::string::npos; +} + +} // namespace + +// A libcurl easy handle owns the connection cache of the connections it opened, so +// returning it to the pool after a request keeps the connection alive for the next one: +// re-creating a handle per request would pay a TCP and TLS handshake every time. A handle +// must not be used by two threads at once, hence handing it out exclusively. +class RestHttpClient::HandlePool { + public: + HandlePool() : curl_guard_(EnsureCurlGlobalInit()) {} + + ~HandlePool() { + for (CURL* handle : handles_) { + curl_easy_cleanup(handle); + } + } + + CURL* Acquire() const { + std::scoped_lock lock(mutex_); + if (handles_.empty()) { + return curl_easy_init(); + } + CURL* handle = handles_.back(); + handles_.pop_back(); + return handle; + } + + void Release(CURL* handle) const { + // Resetting drops the options of the finished request while keeping the + // connection cache, so the next request starts from a clean handle. + curl_easy_reset(handle); + std::scoped_lock lock(mutex_); + handles_.push_back(handle); + } + + private: + std::shared_ptr curl_guard_; + mutable std::mutex mutex_; + mutable std::vector handles_; +}; + +RestHttpClient::RestHttpClient(const std::string& base_uri, const Config& config) + : handle_pool_(std::make_unique()), + base_uri_(base_uri), + config_(config), + logger_(Logger::GetLogger("RestHttpClient")) {} + +RestHttpClient::~RestHttpClient() = default; + +Result> RestHttpClient::Create(const std::string& base_uri) { + return Create(base_uri, Config()); +} + +Result> RestHttpClient::Create(const std::string& base_uri, + const Config& config) { + if (base_uri.empty()) { + return Status::Invalid("uri of the http client is empty"); + } + return std::unique_ptr(new RestHttpClient(NormalizeUri(base_uri), config)); +} + +std::string RestHttpClient::NormalizeUri(const std::string& uri) { + std::string normalized = uri; + StringUtils::Trim(&normalized); + while (!normalized.empty() && normalized.back() == '/') { + normalized.pop_back(); + } + if (normalized.rfind("http://", 0) != 0 && normalized.rfind("https://", 0) != 0) { + normalized = "http://" + normalized; + } + return normalized; +} + +std::string RestHttpClient::BuildQueryString( + const std::map& query_params) { + std::string query; + for (const auto& [key, value] : query_params) { + if (!query.empty()) { + query.push_back('&'); + } + query.append(UrlUtils::EncodeString(key)); + query.push_back('='); + query.append(UrlUtils::EncodeString(value)); + } + return query; +} + +Result RestHttpClient::ExecuteOnce( + const std::string& method, const std::string& url, + const std::map& headers, const std::string& body, + bool* transport_retriable) const { + CURL* curl = handle_pool_->Acquire(); + if (curl == nullptr) { + return Status::IOError("failed to create curl handle"); + } + ScopeGuard release_curl([this, curl] { handle_pool_->Release(curl); }); + Response response; + curl_easy_setopt(curl, CURLOPT_URL, url.c_str()); + curl_easy_setopt(curl, CURLOPT_NOSIGNAL, 1L); + curl_easy_setopt(curl, CURLOPT_USERAGENT, kDefaultUserAgent); + // TLS 1.0/1.1 are not accepted, as in the Java client. The curl constants of the + // options below are `int`, while curl reads a `long` from the variadic argument. + curl_easy_setopt(curl, CURLOPT_SSLVERSION, + static_cast(CURL_SSLVERSION_TLSv1_2)); // NOLINT(runtime/int) + curl_easy_setopt(curl, CURLOPT_CONNECTTIMEOUT_MS, + static_cast(config_.connect_timeout_ms)); // NOLINT(runtime/int) + curl_easy_setopt(curl, CURLOPT_TIMEOUT_MS, + static_cast(config_.request_timeout_ms)); // NOLINT(runtime/int) + curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, WriteBodyCallback); + curl_easy_setopt(curl, CURLOPT_WRITEDATA, &response.body); + curl_easy_setopt(curl, CURLOPT_HEADERFUNCTION, WriteHeaderCallback); + curl_easy_setopt(curl, CURLOPT_HEADERDATA, &response.headers); + // Follow redirects transparently, restricted to http(s) targets. Without + // CURLOPT_POSTREDIR a 301/302 would replay a body-carrying request as a bodyless + // GET. A 303 is left to become a GET, which is what it is defined to mean. + curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L); + curl_easy_setopt(curl, CURLOPT_MAXREDIRS, 50L); + curl_easy_setopt(curl, CURLOPT_POSTREDIR, + static_cast(CURL_REDIR_POST_301 | // NOLINT(runtime/int) + CURL_REDIR_POST_302)); +#if CURL_AT_LEAST_VERSION(7, 85, 0) + curl_easy_setopt(curl, CURLOPT_REDIR_PROTOCOLS_STR, "http,https"); +#else + curl_easy_setopt(curl, CURLOPT_REDIR_PROTOCOLS, + static_cast(CURLPROTO_HTTP | CURLPROTO_HTTPS)); // NOLINT(runtime/int) +#endif + + if (method == "POST") { + curl_easy_setopt(curl, CURLOPT_POST, 1L); + curl_easy_setopt(curl, CURLOPT_POSTFIELDS, body.c_str()); + curl_easy_setopt(curl, CURLOPT_POSTFIELDSIZE, + static_cast(body.size())); // NOLINT(runtime/int) + } else if (method == "DELETE") { + curl_easy_setopt(curl, CURLOPT_CUSTOMREQUEST, "DELETE"); + if (!body.empty()) { + curl_easy_setopt(curl, CURLOPT_POSTFIELDS, body.c_str()); + curl_easy_setopt(curl, CURLOPT_POSTFIELDSIZE, + static_cast(body.size())); // NOLINT(runtime/int) + } + } + + struct curl_slist* header_list = nullptr; + ScopeGuard free_headers([&header_list] { curl_slist_free_all(header_list); }); + auto append_header = [&header_list](const std::string& line) -> bool { + struct curl_slist* updated_list = curl_slist_append(header_list, line.c_str()); + if (updated_list == nullptr) { + return false; + } + header_list = updated_list; + return true; + }; + // An empty "Expect:" header disables libcurl's automatic "Expect: 100-continue". + bool headers_ok = append_header("Expect:"); + for (const auto& [name, value] : headers) { + headers_ok = headers_ok && append_header(name + ": " + value); + } + if (!headers_ok) { + return Status::IOError("failed to create http headers"); + } + curl_easy_setopt(curl, CURLOPT_HTTPHEADER, header_list); + + CURLcode curl_code = curl_easy_perform(curl); + if (curl_code != CURLE_OK) { + *transport_retriable = IsRetriableTransportError(curl_code); + // Neither the url nor the response body is echoed in errors: both may carry + // credentials. + return Status::IOError( + fmt::format("http {} request failed: {}", method, curl_easy_strerror(curl_code))); + } + // curl writes a `long` through the pointer, so the type cannot be narrowed here. + // NOLINTNEXTLINE(google-runtime-int) + long http_code = 0; // NOLINT(runtime/int) + curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &http_code); + response.code = http_code; + return response; +} + +std::optional RestHttpClient::ComputeRetryDelayMs(const Config& config, + int32_t execution_count, + const Response* response, + int64_t now_epoch_seconds, + int64_t remaining_budget_ms) { + if (remaining_budget_ms <= 0) { + return std::nullopt; + } + if (response != nullptr) { + auto iter = response->headers.find("retry-after"); + if (iter != response->headers.end()) { + std::optional retry_after_ms; + std::optional retry_after_seconds = + StringUtils::StringToValue(iter->second); + if (retry_after_seconds) { + // Clamped so that a misbehaving server cannot overflow the conversion. + constexpr int64_t kMaxSeconds = std::numeric_limits::max() / 1000; + retry_after_ms = + std::clamp(retry_after_seconds.value(), -kMaxSeconds, kMaxSeconds) * 1000; + } else { + std::optional date_seconds = ParseHttpDateSeconds(iter->second); + if (date_seconds) { + retry_after_ms = (date_seconds.value() - now_epoch_seconds) * 1000; + } + } + // Non-positive values fall through to the exponential backoff. A delay + // beyond the remaining budget stops retrying entirely: sleeping less than + // requested would just hit the throttle again. `retry_max_delay_ms` is + // deliberately not applied here, as in the Java client, so a longer + // `Retry-After` is honored whenever it fits the budget. + if (retry_after_ms && retry_after_ms.value() > 0) { + if (retry_after_ms.value() > remaining_budget_ms) { + return std::nullopt; + } + return retry_after_ms.value(); + } + } + } + int64_t multiplier = static_cast(1) << std::clamp(execution_count - 1, 0, 6); + int64_t delay_ms = config.retry_base_delay_ms * multiplier; + if (delay_ms > 0) { + static thread_local std::mt19937 generator( + std::random_device{}()); // NOLINT(whitespace/braces) + std::uniform_int_distribution jitter(0, delay_ms / 10); + delay_ms += jitter(generator); + } + delay_ms = std::min(delay_ms, config.retry_max_delay_ms); + if (delay_ms > remaining_budget_ms) { + return std::nullopt; + } + return delay_ms; +} + +std::optional RestHttpClient::GetRetryDelayMs(int32_t execution_count, + const Response* response, + int64_t remaining_budget_ms) const { + return ComputeRetryDelayMs(config_, execution_count, response, + static_cast(std::time(nullptr)), remaining_budget_ms); +} + +Result RestHttpClient::Execute( + const std::string& method, const std::string& path, + const std::map& query_params, + const std::map& headers, const std::string& body) const { + if (method != "GET" && method != "POST" && method != "DELETE") { + return Status::Invalid(fmt::format("unsupported http method: {}", method)); + } + for (const auto& [name, value] : headers) { + // Neither the name nor the value is echoed in errors: the name may contain + // the very control bytes this check rejects, the value may carry credentials. + if (!IsValidHeaderName(name)) { + return Status::Invalid("invalid http header name"); + } + if (!IsValidHeaderValue(value)) { + return Status::Invalid(fmt::format("invalid http header value for '{}'", name)); + } + } + std::string url = base_uri_ + path; + if (!query_params.empty()) { + url += "?" + BuildQueryString(query_params); + } + bool idempotent = IsIdempotent(method); + std::chrono::steady_clock::time_point start = std::chrono::steady_clock::now(); + int32_t execution_count = 0; + while (true) { + execution_count++; + bool transport_retriable = false; + Result result = ExecuteOnce(method, url, headers, body, &transport_retriable); + bool retriable; + if (result.ok()) { + retriable = IsRetriableCode(result.value().code); + } else { + retriable = idempotent && transport_retriable; + } + int64_t elapsed_ms = std::chrono::duration_cast( + std::chrono::steady_clock::now() - start) + .count(); + std::optional delay_ms; + if (retriable && execution_count <= config_.max_retries) { + delay_ms = GetRetryDelayMs(execution_count, result.ok() ? &result.value() : nullptr, + config_.retry_timeout_ms - elapsed_ms); + } + if (!delay_ms) { + if (result.ok()) { + const std::string request_id = RestUtil::ExtractRequestId(result.value().headers); + PAIMON_LOG_DEBUG( + logger_, "[rest] requestId:%s method:%s path:%s status:%lld duration:%lldms", + request_id.c_str(), method.c_str(), path.c_str(), + static_cast(result.value().code), // NOLINT(runtime/int) + static_cast(elapsed_ms)); // NOLINT(runtime/int) + } + return result; + } + std::string reason = result.ok() ? fmt::format("http status {}", result.value().code) + : result.status().ToString(); + PAIMON_LOG_WARN(logger_, "[rest] retrying method:%s path:%s in %lld ms (retry %d/%d): %s", + method.c_str(), path.c_str(), + static_cast(delay_ms.value()), // NOLINT(runtime/int) + execution_count, config_.max_retries, reason.c_str()); + if (delay_ms.value() > 0) { + std::this_thread::sleep_for(std::chrono::milliseconds(delay_ms.value())); + } + } +} + +} // namespace paimon diff --git a/src/paimon/rest/rest_http_client.h b/src/paimon/rest/rest_http_client.h new file mode 100644 index 00000000..07712795 --- /dev/null +++ b/src/paimon/rest/rest_http_client.h @@ -0,0 +1,149 @@ +/* + * Copyright 2026-present Alibaba Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include +#include +#include +#include + +#include "paimon/logging.h" +#include "paimon/result.h" +#include "paimon/status.h" + +namespace paimon { + +/// HTTP status codes used by the REST catalog. +struct HttpStatus { + static constexpr int64_t kOk = 200; + static constexpr int64_t kAccepted = 202; + static constexpr int64_t kNoContent = 204; + static constexpr int64_t kBadRequest = 400; + static constexpr int64_t kUnauthorized = 401; + static constexpr int64_t kForbidden = 403; + static constexpr int64_t kNotFound = 404; + static constexpr int64_t kConflict = 409; + static constexpr int64_t kTooManyRequests = 429; + static constexpr int64_t kInternalServerError = 500; + static constexpr int64_t kNotImplemented = 501; + static constexpr int64_t kServiceUnavailable = 503; +}; + +/// A blocking HTTP client for the REST catalog based on libcurl. HTTP 429/503 +/// responses are retried for all methods, transient transport errors only for +/// idempotent ones, with exponential backoff honoring a positive `Retry-After` +/// response header (delta-seconds or HTTP-date form). A backoff sleep is bounded by +/// `retry_max_delay_ms` and the whole request by `retry_timeout_ms`; a `Retry-After` +/// beyond the remaining budget stops retrying rather than shortening the sleep. +/// Redirects to http(s) targets are followed transparently, keeping the method and +/// body of POST/DELETE requests. +class RestHttpClient { + public: + struct Config { + int64_t connect_timeout_ms = 180 * 1000; + int64_t request_timeout_ms = 180 * 1000; + int32_t max_retries = 5; + int64_t retry_base_delay_ms = 1000; + /// Upper bound for a single backoff sleep; a longer `Retry-After` is still + /// honored, bounded only by `retry_timeout_ms`. + int64_t retry_max_delay_ms = 64 * 1000; + /// Overall budget for one `Execute` call, covering all attempts and retry + /// sleeps; a retry whose delay does not fit the remaining budget is not + /// attempted. + int64_t retry_timeout_ms = 5 * 60 * 1000; + }; + + struct Response { + int64_t code = 0; + std::string body; + /// Response headers with lower-cased names. + std::map headers; + + bool IsSuccessful() const { + return code == HttpStatus::kOk || code == HttpStatus::kAccepted || + code == HttpStatus::kNoContent; + } + }; + + /// Creates a client against `base_uri`. The uri is normalized: surrounding + /// whitespace and all trailing '/' are stripped and "http://" is prepended when no + /// scheme is present. + static Result> Create(const std::string& base_uri); + static Result> Create(const std::string& base_uri, + const Config& config); + + ~RestHttpClient(); + + /// Executes `method` ("GET", "POST" or "DELETE") on `path` (already url-encoded, + /// starting with '/'); query parameter keys and values are url-encoded internally. + /// Header names must be valid HTTP tokens and values must not contain CR, LF or + /// NUL; a violating header fails the request before anything is sent. Returns the + /// final response, which may carry a non-2xx code, or an error status when the + /// request could not be transported at all. Only transient transport errors (an + /// established connection breaking mid-request or a truncated response body) are + /// retried; every other transport failure fails immediately. + Result Execute(const std::string& method, const std::string& path, + const std::map& query_params, + const std::map& headers, + const std::string& body) const; + + const std::string& GetBaseUri() const { + return base_uri_; + } + + static std::string NormalizeUri(const std::string& uri); + + /// Builds "k1=v1&k2=v2" with url-encoded keys and values. + static std::string BuildQueryString(const std::map& query_params); + + /// Computes the delay before the retry following the `execution_count`-th attempt + /// (counted from 1), in ms, or `nullopt` when retrying should stop. A positive + /// `Retry-After` header of `response` (delta-seconds, or HTTP-date resolved against + /// `now_epoch_seconds`) wins and is never shortened: beyond `remaining_budget_ms` + /// it yields `nullopt`. Otherwise exponential backoff with up to 10% jitter + /// applies, clamped to `retry_max_delay_ms`, yielding `nullopt` when even the + /// clamped delay exceeds `remaining_budget_ms`. `response` is null when the attempt + /// failed with a transport error. + static std::optional ComputeRetryDelayMs(const Config& config, int32_t execution_count, + const Response* response, + int64_t now_epoch_seconds, + int64_t remaining_budget_ms); + + private: + RestHttpClient(const std::string& base_uri, const Config& config); + + /// Pool of the libcurl easy handles used by `ExecuteOnce`, also keeping libcurl's + /// global state alive for the lifetime of the client. + class HandlePool; + + /// On a transport failure, `transport_retriable` reports whether the failure kind + /// may be retried; see `Execute` for which kinds are not. + Result ExecuteOnce(const std::string& method, const std::string& url, + const std::map& headers, + const std::string& body, bool* transport_retriable) const; + + std::optional GetRetryDelayMs(int32_t execution_count, const Response* response, + int64_t remaining_budget_ms) const; + + std::unique_ptr handle_pool_; + std::string base_uri_; + Config config_; + std::shared_ptr logger_; +}; + +} // namespace paimon diff --git a/src/paimon/rest/rest_http_client_test.cpp b/src/paimon/rest/rest_http_client_test.cpp new file mode 100644 index 00000000..34d07fe6 --- /dev/null +++ b/src/paimon/rest/rest_http_client_test.cpp @@ -0,0 +1,565 @@ +/* + * Copyright 2026-present Alibaba Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "paimon/rest/rest_http_client.h" + +#include +#include +#include +#include +#include +#include + +#include "gtest/gtest.h" +#include "paimon/rest/mock_rest_server.h" +#include "paimon/testing/utils/testharness.h" + +namespace paimon::test { + +namespace { +RestHttpClient::Config FastRetryConfig(int32_t max_retries) { + RestHttpClient::Config config; + config.max_retries = max_retries; + config.retry_base_delay_ms = 1; + return config; +} +} // namespace + +TEST(RestHttpClientTest, NormalizeUri) { + ASSERT_EQ("http://localhost:80", RestHttpClient::NormalizeUri("localhost:80/")); + ASSERT_EQ("http://localhost", RestHttpClient::NormalizeUri("http://localhost//")); + ASSERT_EQ("https://foo.bar", RestHttpClient::NormalizeUri(" https://foo.bar ")); +} + +TEST(RestHttpClientTest, BuildQueryString) { + ASSERT_EQ("a=1&b=x+y%2Fz", RestHttpClient::BuildQueryString({{"a", "1"}, {"b", "x y/z"}})); +} + +TEST(RestHttpClientTest, GetWithHeadersAndQuery) { + // the handler runs on the mock server's accept thread, so every state it shares with + // the test body is mutex guarded, here and in the tests below + std::mutex mutex; + MockRestServer::Request last_request; + ASSERT_OK_AND_ASSIGN(std::unique_ptr server, + MockRestServer::Start([&](const MockRestServer::Request& request) { + std::lock_guard lock(mutex); + last_request = request; + MockRestServer::Response response; + response.body = R"({"ok": true})"; + return response; + })); + ASSERT_OK_AND_ASSIGN(std::unique_ptr client, + RestHttpClient::Create(server->GetBaseUri())); + ASSERT_OK_AND_ASSIGN(RestHttpClient::Response response, + client->Execute("GET", "/v1/config", {{"warehouse", "wh 1"}}, + {{"Authorization", "Bearer token1"}}, "")); + ASSERT_EQ(200, response.code); + ASSERT_EQ(R"({"ok": true})", response.body); + ASSERT_EQ("application/json", response.headers.at("content-type")); + std::lock_guard lock(mutex); + ASSERT_EQ("GET", last_request.method); + ASSERT_EQ("/v1/config", last_request.path); + ASSERT_EQ("wh 1", last_request.query_params.at("warehouse")); + ASSERT_EQ("Bearer token1", last_request.headers.at("authorization")); +} + +TEST(RestHttpClientTest, PostBody) { + std::mutex mutex; + MockRestServer::Request last_request; + ASSERT_OK_AND_ASSIGN(std::unique_ptr server, + MockRestServer::Start([&](const MockRestServer::Request& request) { + std::lock_guard lock(mutex); + last_request = request; + return MockRestServer::Response(); + })); + ASSERT_OK_AND_ASSIGN(std::unique_ptr client, + RestHttpClient::Create(server->GetBaseUri())); + ASSERT_OK_AND_ASSIGN( + RestHttpClient::Response response, + client->Execute("POST", "/v1/databases", {}, {{"Content-Type", "application/json"}}, + R"({"name": "db1"})")); + ASSERT_EQ(200, response.code); + std::lock_guard lock(mutex); + ASSERT_EQ("POST", last_request.method); + ASSERT_EQ(R"({"name": "db1"})", last_request.body); + ASSERT_EQ("application/json", last_request.headers.at("content-type")); +} + +TEST(RestHttpClientTest, NotFoundIsNotRetried) { + std::atomic request_count{0}; + ASSERT_OK_AND_ASSIGN(std::unique_ptr server, + MockRestServer::Start([&](const MockRestServer::Request& request) { + request_count++; + MockRestServer::Response response; + response.code = 404; + response.body = R"({"message": "not found", "code": 404})"; + return response; + })); + ASSERT_OK_AND_ASSIGN(std::unique_ptr client, + RestHttpClient::Create(server->GetBaseUri(), FastRetryConfig(5))); + ASSERT_OK_AND_ASSIGN(RestHttpClient::Response response, + client->Execute("GET", "/v1/databases/db1", {}, {}, "")); + ASSERT_EQ(404, response.code); + ASSERT_FALSE(response.IsSuccessful()); + ASSERT_EQ(1, request_count.load()); +} + +TEST(RestHttpClientTest, ServiceUnavailableIsRetried) { + std::atomic request_count{0}; + ASSERT_OK_AND_ASSIGN(std::unique_ptr server, + MockRestServer::Start([&](const MockRestServer::Request& request) { + MockRestServer::Response response; + if (request_count++ < 2) { + response.code = 503; + } else { + response.body = R"({"ok": true})"; + } + return response; + })); + ASSERT_OK_AND_ASSIGN(std::unique_ptr client, + RestHttpClient::Create(server->GetBaseUri(), FastRetryConfig(5))); + // 429/503 responses are retried even for non-idempotent POST. + ASSERT_OK_AND_ASSIGN(RestHttpClient::Response response, + client->Execute("POST", "/v1/databases", {}, {}, "{}")); + ASSERT_EQ(200, response.code); + ASSERT_EQ(3, request_count.load()); +} + +TEST(RestHttpClientTest, RetryAfterHeaderIsHonored) { + std::atomic request_count{0}; + ASSERT_OK_AND_ASSIGN(std::unique_ptr server, + MockRestServer::Start([&](const MockRestServer::Request& request) { + MockRestServer::Response response; + if (request_count++ == 0) { + response.code = 429; + response.headers["Retry-After"] = "1"; + } + return response; + })); + // the backoff base is large so that the elapsed time below proves the Retry-After + // header took precedence: without it this test would sleep for a minute + RestHttpClient::Config config; + config.max_retries = 5; + config.retry_base_delay_ms = 60 * 1000; + ASSERT_OK_AND_ASSIGN(std::unique_ptr client, + RestHttpClient::Create(server->GetBaseUri(), config)); + auto start = std::chrono::steady_clock::now(); + ASSERT_OK_AND_ASSIGN(RestHttpClient::Response response, + client->Execute("GET", "/v1/databases", {}, {}, "")); + auto elapsed = std::chrono::steady_clock::now() - start; + ASSERT_EQ(200, response.code); + ASSERT_EQ(2, request_count.load()); + ASSERT_GE(elapsed, std::chrono::seconds(1)); + ASSERT_LT(elapsed, std::chrono::seconds(30)); +} + +TEST(RestHttpClientTest, ComputeRetryDelayMs) { + RestHttpClient::Config config; + config.retry_base_delay_ms = 1000; + constexpr int64_t kBudgetMs = 1000 * 1000; + // without a usable Retry-After header: exponential backoff with up to 10% jitter, + // the multiplier capped at 2^6 + std::optional delay = + RestHttpClient::ComputeRetryDelayMs(config, 1, nullptr, 0, kBudgetMs); + ASSERT_TRUE(delay); + ASSERT_GE(delay.value(), 1000); + ASSERT_LE(delay.value(), 1100); + delay = RestHttpClient::ComputeRetryDelayMs(config, 3, nullptr, 0, kBudgetMs); + ASSERT_TRUE(delay); + ASSERT_GE(delay.value(), 4000); + ASSERT_LE(delay.value(), 4400); + // the capped multiplier lands exactly on the default per-retry bound, so the + // jitter is clamped away + delay = RestHttpClient::ComputeRetryDelayMs(config, 100, nullptr, 0, kBudgetMs); + ASSERT_EQ(64000, delay); + // an out-of-contract execution count degrades to the first-attempt delay instead + // of shifting by a negative amount + delay = RestHttpClient::ComputeRetryDelayMs(config, 0, nullptr, 0, kBudgetMs); + ASSERT_TRUE(delay); + ASSERT_GE(delay.value(), 1000); + ASSERT_LE(delay.value(), 1100); + + // the delta-seconds form takes precedence over the backoff, without jitter + RestHttpClient::Response response; + response.headers["retry-after"] = "7"; + ASSERT_EQ(7000, RestHttpClient::ComputeRetryDelayMs(config, 1, &response, 0, kBudgetMs)); + + // the HTTP-date form is resolved against the given "now" + constexpr int64_t kDateEpoch = 1445412480; // Wed, 21 Oct 2015 07:28:00 GMT + response.headers["retry-after"] = "Wed, 21 Oct 2015 07:28:00 GMT"; + ASSERT_EQ(5000, + RestHttpClient::ComputeRetryDelayMs(config, 1, &response, kDateEpoch - 5, kBudgetMs)); + + // a date in the past falls back to the backoff + delay = RestHttpClient::ComputeRetryDelayMs(config, 1, &response, kDateEpoch + 1, kBudgetMs); + ASSERT_TRUE(delay); + ASSERT_GE(delay.value(), 1000); + ASSERT_LE(delay.value(), 1100); +} + +TEST(RestHttpClientTest, RetryDelayBounds) { + RestHttpClient::Config config; + config.retry_base_delay_ms = 1000; + config.retry_max_delay_ms = 10 * 1000; + RestHttpClient::Response response; + + // the per-retry bound applies to the backoff only: a Retry-After beyond it is + // still honored as long as it fits the remaining budget + response.headers["retry-after"] = "11"; + ASSERT_EQ(11000, RestHttpClient::ComputeRetryDelayMs(config, 1, &response, 0, 1000 * 1000)); + // a Retry-After beyond the remaining budget stops retrying instead of sleeping + // less than the server requested + response.headers["retry-after"] = "9"; + ASSERT_FALSE(RestHttpClient::ComputeRetryDelayMs(config, 1, &response, 0, 8000)); + ASSERT_EQ(9000, RestHttpClient::ComputeRetryDelayMs(config, 1, &response, 0, 9000)); + + // the backoff is clamped to the per-retry bound... + ASSERT_EQ(10000, RestHttpClient::ComputeRetryDelayMs(config, 100, nullptr, 0, 1000 * 1000)); + // ...and stops retrying when even the clamped delay does not fit the budget + ASSERT_FALSE(RestHttpClient::ComputeRetryDelayMs(config, 100, nullptr, 0, 9999)); + // an exhausted budget stops retrying before any header is consulted + ASSERT_FALSE(RestHttpClient::ComputeRetryDelayMs(config, 1, &response, 0, 0)); +} + +TEST(RestHttpClientTest, ExcessiveRetryAfterStopsRetrying) { + std::atomic request_count{0}; + ASSERT_OK_AND_ASSIGN(std::unique_ptr server, + MockRestServer::Start([&](const MockRestServer::Request& request) { + request_count++; + MockRestServer::Response response; + response.code = 503; + // an hour is far beyond the default overall retry budget + response.headers["Retry-After"] = "3600"; + return response; + })); + ASSERT_OK_AND_ASSIGN(std::unique_ptr client, + RestHttpClient::Create(server->GetBaseUri(), FastRetryConfig(5))); + ASSERT_OK_AND_ASSIGN(RestHttpClient::Response response, + client->Execute("GET", "/v1/databases", {}, {}, "")); + // the 503 is returned as-is after a single attempt instead of five shortened + // sleeps the server did not ask for + ASSERT_EQ(503, response.code); + ASSERT_EQ(1, request_count.load()); +} + +TEST(RestHttpClientTest, InvalidRetryAfterFallsBackToBackoff) { + std::atomic request_count{0}; + ASSERT_OK_AND_ASSIGN(std::unique_ptr server, + MockRestServer::Start([&](const MockRestServer::Request& request) { + MockRestServer::Response response; + int32_t count = request_count++; + if (count == 0) { + response.code = 429; + // non-positive values are ignored + response.headers["Retry-After"] = "0"; + } else if (count == 2) { + response.code = 429; + // neither delta-seconds nor a valid http date + response.headers["Retry-After"] = "Sat, 32 Foo 2015 99:99:99 GMT"; + } + return response; + })); + ASSERT_OK_AND_ASSIGN(std::unique_ptr client, + RestHttpClient::Create(server->GetBaseUri(), FastRetryConfig(5))); + ASSERT_OK_AND_ASSIGN(RestHttpClient::Response response, + client->Execute("GET", "/v1/databases", {}, {}, "")); + ASSERT_EQ(200, response.code); + ASSERT_EQ(2, request_count.load()); + ASSERT_OK_AND_ASSIGN(response, client->Execute("GET", "/v1/databases", {}, {}, "")); + ASSERT_EQ(200, response.code); + ASSERT_EQ(4, request_count.load()); +} + +TEST(RestHttpClientTest, RetriesExhausted) { + std::atomic request_count{0}; + ASSERT_OK_AND_ASSIGN(std::unique_ptr server, + MockRestServer::Start([&](const MockRestServer::Request& request) { + request_count++; + MockRestServer::Response response; + response.code = 503; + return response; + })); + ASSERT_OK_AND_ASSIGN(std::unique_ptr client, + RestHttpClient::Create(server->GetBaseUri(), FastRetryConfig(2))); + ASSERT_OK_AND_ASSIGN(RestHttpClient::Response response, + client->Execute("GET", "/v1/databases", {}, {}, "")); + ASSERT_EQ(503, response.code); + // initial attempt + 2 retries + ASSERT_EQ(3, request_count.load()); +} + +TEST(RestHttpClientTest, EmptyReplyIsNotRetried) { + std::atomic request_count{0}; + ASSERT_OK_AND_ASSIGN(std::unique_ptr server, + MockRestServer::Start([&](const MockRestServer::Request& request) { + request_count++; + MockRestServer::Response response; + // closing without a response maps to CURLE_GOT_NOTHING, a + // non-retriable transport error that, unlike timeouts or + // TLS failures, is deterministic in a unit test + response.close_without_response = true; + return response; + })); + ASSERT_OK_AND_ASSIGN(std::unique_ptr client, + RestHttpClient::Create(server->GetBaseUri(), FastRetryConfig(5))); + ASSERT_NOK(client->Execute("GET", "/v1/databases", {}, {}, "").status()); + ASSERT_EQ(1, request_count.load()); +} + +TEST(RestHttpClientTest, ConnectionRefusedIsNotRetried) { + // a closed port yields CURLE_COULDNT_CONNECT, another of the non-retriable + // transport errors; the port of a stopped mock server is known to be closed + ASSERT_OK_AND_ASSIGN(std::unique_ptr server, + MockRestServer::Start([](const MockRestServer::Request& request) { + return MockRestServer::Response(); + })); + std::string base_uri = server->GetBaseUri(); + server->Stop(); + // the backoff base is large so that the elapsed time below proves the failure was + // not retried: a single retry would sleep for a minute + RestHttpClient::Config config; + config.max_retries = 5; + config.retry_base_delay_ms = 60 * 1000; + ASSERT_OK_AND_ASSIGN(std::unique_ptr client, + RestHttpClient::Create(base_uri, config)); + auto start = std::chrono::steady_clock::now(); + ASSERT_NOK(client->Execute("GET", "/v1/databases", {}, {}, "").status()); + auto elapsed = std::chrono::steady_clock::now() - start; + ASSERT_LT(elapsed, std::chrono::seconds(30)); +} + +TEST(RestHttpClientTest, DeleteWithBodyIsSent) { + std::mutex mutex; + MockRestServer::Request last_request; + ASSERT_OK_AND_ASSIGN(std::unique_ptr server, + MockRestServer::Start([&](const MockRestServer::Request& request) { + std::lock_guard lock(mutex); + last_request = request; + return MockRestServer::Response(); + })); + ASSERT_OK_AND_ASSIGN(std::unique_ptr client, + RestHttpClient::Create(server->GetBaseUri())); + // a body on DELETE keeps the DELETE method line instead of degrading to POST + ASSERT_OK_AND_ASSIGN( + RestHttpClient::Response response, + client->Execute("DELETE", "/v1/databases/db1", {}, {{"Content-Type", "application/json"}}, + R"({"purge": true})")); + ASSERT_EQ(200, response.code); + std::lock_guard lock(mutex); + ASSERT_EQ("DELETE", last_request.method); + ASSERT_EQ(R"({"purge": true})", last_request.body); +} + +TEST(RestHttpClientTest, ReusedHandleDoesNotCarryRequestState) { + // the client pools its curl handles to keep connections alive, so a request must not + // inherit the method or the body of the request that used the handle before it + std::mutex mutex; + std::vector requests; + ASSERT_OK_AND_ASSIGN(std::unique_ptr server, + MockRestServer::Start([&](const MockRestServer::Request& request) { + std::lock_guard lock(mutex); + requests.push_back(request); + return MockRestServer::Response(); + })); + ASSERT_OK_AND_ASSIGN(std::unique_ptr client, + RestHttpClient::Create(server->GetBaseUri())); + ASSERT_OK(client->Execute("POST", "/v1/databases", {}, {}, R"({"name": "db1"})").status()); + ASSERT_OK( + client->Execute("DELETE", "/v1/databases/db1", {}, {}, R"({"purge": true})").status()); + ASSERT_OK(client->Execute("GET", "/v1/databases", {}, {}, "").status()); + std::lock_guard lock(mutex); + ASSERT_EQ(3, requests.size()); + ASSERT_EQ("GET", requests[2].method); + ASSERT_TRUE(requests[2].body.empty()) << requests[2].body; +} + +TEST(RestHttpClientTest, TransportErrorOmitsUrlAndQuery) { + ASSERT_OK_AND_ASSIGN(std::unique_ptr server, + MockRestServer::Start([](const MockRestServer::Request& request) { + MockRestServer::Response response; + response.close_without_response = true; + return response; + })); + ASSERT_OK_AND_ASSIGN(std::unique_ptr client, + RestHttpClient::Create(server->GetBaseUri())); + Status status = + client->Execute("GET", "/v1/secret-path", {{"sig", "topsecret1"}}, {}, "").status(); + ASSERT_NOK(status); + // the error must not echo the path or query values, which may carry credentials + // (e.g. a presigned url) + ASSERT_EQ(std::string::npos, status.ToString().find("secret-path")) << status.ToString(); + ASSERT_EQ(std::string::npos, status.ToString().find("topsecret1")) << status.ToString(); +} + +TEST(RestHttpClientTest, TruncatedResponseIsRetriedForGet) { + std::atomic request_count{0}; + ASSERT_OK_AND_ASSIGN(std::unique_ptr server, + MockRestServer::Start([&](const MockRestServer::Request& request) { + MockRestServer::Response response; + response.body = R"({"ok": true})"; + if (request_count++ == 0) { + // cutting the response off mid-body is a retriable + // transport error, and GET is idempotent + response.missing_body_bytes = 5; + } + return response; + })); + ASSERT_OK_AND_ASSIGN(std::unique_ptr client, + RestHttpClient::Create(server->GetBaseUri(), FastRetryConfig(5))); + ASSERT_OK_AND_ASSIGN(RestHttpClient::Response response, + client->Execute("GET", "/v1/databases", {}, {}, "")); + ASSERT_EQ(200, response.code); + ASSERT_EQ(R"({"ok": true})", response.body); + ASSERT_EQ(2, request_count.load()); +} + +TEST(RestHttpClientTest, TruncatedResponseIsRetriedForDelete) { + std::atomic request_count{0}; + ASSERT_OK_AND_ASSIGN(std::unique_ptr server, + MockRestServer::Start([&](const MockRestServer::Request& request) { + MockRestServer::Response response; + if (request_count++ == 0) { + // DELETE is the other idempotent method (see the GET + // test above) + response.missing_body_bytes = 5; + response.body = R"({"ok": true})"; + } + return response; + })); + ASSERT_OK_AND_ASSIGN(std::unique_ptr client, + RestHttpClient::Create(server->GetBaseUri(), FastRetryConfig(5))); + ASSERT_OK_AND_ASSIGN(RestHttpClient::Response response, + client->Execute("DELETE", "/v1/databases/db1", {}, {}, "")); + ASSERT_EQ(200, response.code); + ASSERT_EQ(2, request_count.load()); +} + +TEST(RestHttpClientTest, TruncatedResponseIsNotRetriedForPost) { + std::atomic request_count{0}; + ASSERT_OK_AND_ASSIGN(std::unique_ptr server, + MockRestServer::Start([&](const MockRestServer::Request& request) { + request_count++; + MockRestServer::Response response; + response.body = R"({"ok": true})"; + // the same retriable transport error kind as in the GET + // test above, but POST is not idempotent + response.missing_body_bytes = 5; + return response; + })); + ASSERT_OK_AND_ASSIGN(std::unique_ptr client, + RestHttpClient::Create(server->GetBaseUri(), FastRetryConfig(5))); + ASSERT_NOK(client->Execute("POST", "/v1/databases", {}, {}, "{}").status()); + ASSERT_EQ(1, request_count.load()); +} + +TEST(RestHttpClientTest, RedirectIsFollowed) { + std::atomic request_count{0}; + ASSERT_OK_AND_ASSIGN(std::unique_ptr server, + MockRestServer::Start([&](const MockRestServer::Request& request) { + MockRestServer::Response response; + if (request_count++ == 0) { + response.code = 302; + response.headers["Location"] = "/v1/config"; + } else { + response.body = R"({"ok": true})"; + } + return response; + })); + ASSERT_OK_AND_ASSIGN(std::unique_ptr client, + RestHttpClient::Create(server->GetBaseUri())); + ASSERT_OK_AND_ASSIGN(RestHttpClient::Response response, + client->Execute("GET", "/old", {}, {}, "")); + ASSERT_EQ(200, response.code); + ASSERT_EQ(R"({"ok": true})", response.body); + ASSERT_EQ(2, request_count.load()); + // only the final response's headers are reported, not the redirect's + ASSERT_EQ(0, response.headers.count("location")); +} + +TEST(RestHttpClientTest, PostRedirectKeepsMethodAndBody) { + std::mutex mutex; + MockRestServer::Request last_request; + ASSERT_OK_AND_ASSIGN(std::unique_ptr server, + MockRestServer::Start([&](const MockRestServer::Request& request) { + MockRestServer::Response response; + if (request.path == "/old") { + // 302 is one of the codes that would degrade a POST + // to a bodyless GET without CURLOPT_POSTREDIR + response.code = 302; + response.headers["Location"] = "/v1/databases"; + return response; + } + std::lock_guard lock(mutex); + last_request = request; + return response; + })); + ASSERT_OK_AND_ASSIGN(std::unique_ptr client, + RestHttpClient::Create(server->GetBaseUri())); + ASSERT_OK_AND_ASSIGN(RestHttpClient::Response response, + client->Execute("POST", "/old", {}, {}, R"({"name": "db1"})")); + ASSERT_EQ(200, response.code); + std::lock_guard lock(mutex); + ASSERT_EQ("/v1/databases", last_request.path); + ASSERT_EQ("POST", last_request.method); + ASSERT_EQ(R"({"name": "db1"})", last_request.body); +} + +TEST(RestHttpClientTest, RedirectLoopIsNotRetried) { + std::atomic request_count{0}; + ASSERT_OK_AND_ASSIGN(std::unique_ptr server, + MockRestServer::Start([&](const MockRestServer::Request& request) { + request_count++; + MockRestServer::Response response; + response.code = 302; + response.headers["Location"] = "/loop"; + return response; + })); + ASSERT_OK_AND_ASSIGN(std::unique_ptr client, + RestHttpClient::Create(server->GetBaseUri(), FastRetryConfig(5))); + ASSERT_NOK(client->Execute("GET", "/loop", {}, {}, "").status()); + // one attempt follows at most CURLOPT_MAXREDIRS (50) redirects and the resulting + // CURLE_TOO_MANY_REDIRECTS is a permanent transport error: retrying it would + // multiply the request count by the retry schedule + ASSERT_LE(request_count.load(), 51); +} + +TEST(RestHttpClientTest, InvalidHeadersAreRejected) { + // headers are validated before anything is sent, so no server is needed + ASSERT_OK_AND_ASSIGN(std::unique_ptr client, + RestHttpClient::Create("http://127.0.0.1:1")); + ASSERT_NOK_WITH_MSG(client->Execute("GET", "/", {}, {{"Bad Name", "v"}}, "").status(), + "invalid http header name"); + ASSERT_NOK_WITH_MSG( + client->Execute("GET", "/", {}, {{"Evil\r\nInjected: x", "v"}}, "").status(), + "invalid http header name"); + ASSERT_NOK_WITH_MSG(client->Execute("GET", "/", {}, {{"", "v"}}, "").status(), + "invalid http header name"); + ASSERT_NOK_WITH_MSG( + client->Execute("GET", "/", {}, {{"X-Ok", "a\r\nInjected: b"}}, "").status(), + "invalid http header value for 'X-Ok'"); + ASSERT_NOK_WITH_MSG( + client->Execute("GET", "/", {}, {{"X-Ok", std::string("a\0b", 3)}}, "").status(), + "invalid http header value for 'X-Ok'"); +} + +TEST(RestHttpClientTest, UnsupportedMethod) { + ASSERT_OK_AND_ASSIGN(std::unique_ptr client, + RestHttpClient::Create("http://127.0.0.1:1")); + ASSERT_NOK_WITH_MSG(client->Execute("PATCH", "/", {}, {}, "").status(), + "unsupported http method"); +} + +} // namespace paimon::test diff --git a/src/paimon/rest/rest_messages.cpp b/src/paimon/rest/rest_messages.cpp new file mode 100644 index 00000000..324a8408 --- /dev/null +++ b/src/paimon/rest/rest_messages.cpp @@ -0,0 +1,386 @@ +/* + * Copyright 2026-present Alibaba Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "paimon/rest/rest_messages.h" + +#include + +#include "paimon/rest/rest_util.h" + +namespace paimon { + +namespace { + +constexpr const char kFieldMessage[] = "message"; +constexpr const char kFieldResourceType[] = "resourceType"; +constexpr const char kFieldResourceName[] = "resourceName"; +constexpr const char kFieldCode[] = "code"; +constexpr const char kFieldDefaults[] = "defaults"; +constexpr const char kFieldOverrides[] = "overrides"; +constexpr const char kFieldOwner[] = "owner"; +constexpr const char kFieldCreatedAt[] = "createdAt"; +constexpr const char kFieldCreatedBy[] = "createdBy"; +constexpr const char kFieldUpdatedAt[] = "updatedAt"; +constexpr const char kFieldUpdatedBy[] = "updatedBy"; +constexpr const char kFieldName[] = "name"; +constexpr const char kFieldOptions[] = "options"; +constexpr const char kFieldId[] = "id"; +constexpr const char kFieldLocation[] = "location"; +constexpr const char kFieldDatabases[] = "databases"; +constexpr const char kFieldTables[] = "tables"; +constexpr const char kFieldSnapshots[] = "snapshots"; +constexpr const char kFieldNextPageToken[] = "nextPageToken"; +constexpr const char kFieldPath[] = "path"; +constexpr const char kFieldIsExternal[] = "isExternal"; +constexpr const char kFieldSchemaId[] = "schemaId"; +constexpr const char kFieldSchema[] = "schema"; +constexpr const char kFieldIdentifier[] = "identifier"; +constexpr const char kFieldDatabase[] = "database"; +constexpr const char kFieldObject[] = "object"; +constexpr const char kFieldSource[] = "source"; +constexpr const char kFieldDestination[] = "destination"; + +void AddOptionalStringMember(rapidjson::Value* obj, const char* key, + const std::optional& value, + rapidjson::Document::AllocatorType* allocator) { + if (value) { + obj->AddMember(rapidjson::StringRef(key), + RapidJsonUtil::SerializeValue(value.value(), allocator).Move(), *allocator); + } +} + +rapidjson::Value SerializeIdentifier(const std::string& database, const std::string& table, + rapidjson::Document::AllocatorType* allocator) { + rapidjson::Value obj(rapidjson::kObjectType); + obj.AddMember(rapidjson::StringRef(kFieldDatabase), + RapidJsonUtil::SerializeValue(database, allocator).Move(), *allocator); + obj.AddMember(rapidjson::StringRef(kFieldObject), + RapidJsonUtil::SerializeValue(table, allocator).Move(), *allocator); + return obj; +} + +void DeserializeIdentifier(const rapidjson::Value& obj, const char* key, std::string* database, + std::string* table) { + if (!obj.IsObject() || !obj.HasMember(key) || !obj[key].IsObject()) { + throw std::invalid_argument(std::string("member '") + key + + "' must exist and be an object"); + } + const rapidjson::Value& identifier = obj[key]; + *database = RapidJsonUtil::DeserializeKeyValue(identifier, kFieldDatabase); + *table = RapidJsonUtil::DeserializeKeyValue(identifier, kFieldObject); +} + +std::string DeserializeRawJsonMember(const rapidjson::Value& obj, const char* key) { + if (!obj.IsObject() || !obj.HasMember(key) || !obj[key].IsObject()) { + throw std::invalid_argument(std::string("member '") + key + + "' must exist and be an object"); + } + return RestUtil::JsonToString(obj[key]); +} + +} // namespace + +rapidjson::Value ErrorResponse::ToJson(rapidjson::Document::AllocatorType* allocator) const + noexcept(false) { + rapidjson::Value obj(rapidjson::kObjectType); + obj.AddMember(rapidjson::StringRef(kFieldMessage), + RapidJsonUtil::SerializeValue(message_, allocator).Move(), *allocator); + if (!resource_type_.empty()) { + obj.AddMember(rapidjson::StringRef(kFieldResourceType), + RapidJsonUtil::SerializeValue(resource_type_, allocator).Move(), *allocator); + } + if (!resource_name_.empty()) { + obj.AddMember(rapidjson::StringRef(kFieldResourceName), + RapidJsonUtil::SerializeValue(resource_name_, allocator).Move(), *allocator); + } + obj.AddMember(rapidjson::StringRef(kFieldCode), + RapidJsonUtil::SerializeValue(code_, allocator).Move(), *allocator); + return obj; +} + +void ErrorResponse::FromJson(const rapidjson::Value& obj) noexcept(false) { + message_ = RapidJsonUtil::DeserializeKeyValue(obj, kFieldMessage, std::string()); + resource_type_ = + RapidJsonUtil::DeserializeKeyValue(obj, kFieldResourceType, std::string()); + resource_name_ = + RapidJsonUtil::DeserializeKeyValue(obj, kFieldResourceName, std::string()); + code_ = RapidJsonUtil::DeserializeKeyValue(obj, kFieldCode, 0); +} + +rapidjson::Value ConfigResponse::ToJson(rapidjson::Document::AllocatorType* allocator) const + noexcept(false) { + rapidjson::Value obj(rapidjson::kObjectType); + obj.AddMember(rapidjson::StringRef(kFieldDefaults), + RapidJsonUtil::SerializeValue(defaults_, allocator).Move(), *allocator); + obj.AddMember(rapidjson::StringRef(kFieldOverrides), + RapidJsonUtil::SerializeValue(overrides_, allocator).Move(), *allocator); + return obj; +} + +namespace { +// The server may send null values in `defaults`/`overrides`; a null carries no value to +// merge, so it is collected in `null_keys` (when given) instead of failing the parse. +std::map DeserializeStringMap(const rapidjson::Value& obj, + const char* key, + std::set* null_keys) { + std::map result; + if (!obj.IsObject() || !obj.HasMember(key) || obj[key].IsNull()) { + return result; + } + const rapidjson::Value& map_value = obj[key]; + if (!map_value.IsObject()) { + throw std::invalid_argument(std::string("member '") + key + "' must be an object"); + } + for (auto iter = map_value.MemberBegin(); iter != map_value.MemberEnd(); ++iter) { + if (iter->value.IsNull()) { + if (null_keys != nullptr) { + null_keys->insert(iter->name.GetString()); + } + continue; + } + if (!iter->value.IsString()) { + throw std::invalid_argument(std::string("member '") + key + + "' must only contain string values"); + } + result[iter->name.GetString()] = iter->value.GetString(); + } + return result; +} +} // namespace + +void ConfigResponse::FromJson(const rapidjson::Value& obj) noexcept(false) { + removed_keys_.clear(); + // A null default carries no value and never unsets anything; only a null override + // removes a key. + defaults_ = DeserializeStringMap(obj, kFieldDefaults, nullptr); + overrides_ = DeserializeStringMap(obj, kFieldOverrides, &removed_keys_); +} + +std::map ConfigResponse::Merge( + const std::map& client_options) const { + std::map merged = defaults_; + for (const auto& [key, value] : client_options) { + merged[key] = value; + } + for (const auto& [key, value] : overrides_) { + merged[key] = value; + } + for (const std::string& key : removed_keys_) { + merged.erase(key); + } + return merged; +} + +void RestAuditFields::ParseFrom(const rapidjson::Value& obj) { + owner = RapidJsonUtil::DeserializeKeyValue>(obj, kFieldOwner, + std::nullopt); + created_at = RapidJsonUtil::DeserializeKeyValue>(obj, kFieldCreatedAt, + std::nullopt); + created_by = RapidJsonUtil::DeserializeKeyValue>( + obj, kFieldCreatedBy, std::nullopt); + updated_at = RapidJsonUtil::DeserializeKeyValue>(obj, kFieldUpdatedAt, + std::nullopt); + updated_by = RapidJsonUtil::DeserializeKeyValue>( + obj, kFieldUpdatedBy, std::nullopt); +} + +void RestAuditFields::AddTo(rapidjson::Value* obj, + rapidjson::Document::AllocatorType* allocator) const { + AddOptionalStringMember(obj, kFieldOwner, owner, allocator); + if (created_at) { + obj->AddMember(rapidjson::StringRef(kFieldCreatedAt), + RapidJsonUtil::SerializeValue(created_at.value(), allocator).Move(), + *allocator); + } + AddOptionalStringMember(obj, kFieldCreatedBy, created_by, allocator); + if (updated_at) { + obj->AddMember(rapidjson::StringRef(kFieldUpdatedAt), + RapidJsonUtil::SerializeValue(updated_at.value(), allocator).Move(), + *allocator); + } + AddOptionalStringMember(obj, kFieldUpdatedBy, updated_by, allocator); +} + +void RestAuditFields::PutAuditOptionsTo(std::map* options) const { + if (owner) { + (*options)[kFieldOwner] = owner.value(); + } + if (created_at) { + (*options)[kFieldCreatedAt] = std::to_string(created_at.value()); + } + if (created_by) { + (*options)[kFieldCreatedBy] = created_by.value(); + } + if (updated_at) { + (*options)[kFieldUpdatedAt] = std::to_string(updated_at.value()); + } + if (updated_by) { + (*options)[kFieldUpdatedBy] = updated_by.value(); + } +} + +rapidjson::Value CreateDatabaseRequest::ToJson(rapidjson::Document::AllocatorType* allocator) const + noexcept(false) { + rapidjson::Value obj(rapidjson::kObjectType); + obj.AddMember(rapidjson::StringRef(kFieldName), + RapidJsonUtil::SerializeValue(name_, allocator).Move(), *allocator); + obj.AddMember(rapidjson::StringRef(kFieldOptions), + RapidJsonUtil::SerializeValue(options_, allocator).Move(), *allocator); + return obj; +} + +void CreateDatabaseRequest::FromJson(const rapidjson::Value& obj) noexcept(false) { + name_ = RapidJsonUtil::DeserializeKeyValue(obj, kFieldName); + options_ = RapidJsonUtil::DeserializeKeyValue>( + obj, kFieldOptions, {}); +} + +rapidjson::Value GetDatabaseResponse::ToJson(rapidjson::Document::AllocatorType* allocator) const + noexcept(false) { + rapidjson::Value obj(rapidjson::kObjectType); + obj.AddMember(rapidjson::StringRef(kFieldId), + RapidJsonUtil::SerializeValue(id_, allocator).Move(), *allocator); + obj.AddMember(rapidjson::StringRef(kFieldName), + RapidJsonUtil::SerializeValue(name_, allocator).Move(), *allocator); + obj.AddMember(rapidjson::StringRef(kFieldLocation), + RapidJsonUtil::SerializeValue(location_, allocator).Move(), *allocator); + obj.AddMember(rapidjson::StringRef(kFieldOptions), + RapidJsonUtil::SerializeValue(options_, allocator).Move(), *allocator); + audit_.AddTo(&obj, allocator); + return obj; +} + +void GetDatabaseResponse::FromJson(const rapidjson::Value& obj) noexcept(false) { + id_ = RapidJsonUtil::DeserializeKeyValue(obj, kFieldId, std::string()); + name_ = RapidJsonUtil::DeserializeKeyValue(obj, kFieldName); + location_ = RapidJsonUtil::DeserializeKeyValue(obj, kFieldLocation, std::string()); + options_ = RapidJsonUtil::DeserializeKeyValue>( + obj, kFieldOptions, {}); + audit_.ParseFrom(obj); +} + +rapidjson::Value ListDatabasesResponse::ToJson(rapidjson::Document::AllocatorType* allocator) const + noexcept(false) { + rapidjson::Value obj(rapidjson::kObjectType); + obj.AddMember(rapidjson::StringRef(kFieldDatabases), + RapidJsonUtil::SerializeValue(databases_, allocator).Move(), *allocator); + AddOptionalStringMember(&obj, kFieldNextPageToken, next_page_token_, allocator); + return obj; +} + +void ListDatabasesResponse::FromJson(const rapidjson::Value& obj) noexcept(false) { + databases_ = + RapidJsonUtil::DeserializeKeyValue>(obj, kFieldDatabases, {}); + next_page_token_ = RapidJsonUtil::DeserializeKeyValue>( + obj, kFieldNextPageToken, std::nullopt); +} + +rapidjson::Value ListTablesResponse::ToJson(rapidjson::Document::AllocatorType* allocator) const + noexcept(false) { + rapidjson::Value obj(rapidjson::kObjectType); + obj.AddMember(rapidjson::StringRef(kFieldTables), + RapidJsonUtil::SerializeValue(tables_, allocator).Move(), *allocator); + AddOptionalStringMember(&obj, kFieldNextPageToken, next_page_token_, allocator); + return obj; +} + +void ListTablesResponse::FromJson(const rapidjson::Value& obj) noexcept(false) { + tables_ = RapidJsonUtil::DeserializeKeyValue>(obj, kFieldTables, {}); + next_page_token_ = RapidJsonUtil::DeserializeKeyValue>( + obj, kFieldNextPageToken, std::nullopt); +} + +rapidjson::Value ListSnapshotsResponse::ToJson(rapidjson::Document::AllocatorType* allocator) const + noexcept(false) { + rapidjson::Value obj(rapidjson::kObjectType); + obj.AddMember(rapidjson::StringRef(kFieldSnapshots), + RapidJsonUtil::SerializeValue(snapshots_, allocator).Move(), *allocator); + AddOptionalStringMember(&obj, kFieldNextPageToken, next_page_token_, allocator); + return obj; +} + +void ListSnapshotsResponse::FromJson(const rapidjson::Value& obj) noexcept(false) { + snapshots_ = + RapidJsonUtil::DeserializeKeyValue>(obj, kFieldSnapshots, {}); + next_page_token_ = RapidJsonUtil::DeserializeKeyValue>( + obj, kFieldNextPageToken, std::nullopt); +} + +rapidjson::Value GetTableResponse::ToJson(rapidjson::Document::AllocatorType* allocator) const + noexcept(false) { + rapidjson::Value obj(rapidjson::kObjectType); + obj.AddMember(rapidjson::StringRef(kFieldId), + RapidJsonUtil::SerializeValue(id_, allocator).Move(), *allocator); + obj.AddMember(rapidjson::StringRef(kFieldDatabase), + RapidJsonUtil::SerializeValue(database_, allocator).Move(), *allocator); + obj.AddMember(rapidjson::StringRef(kFieldName), + RapidJsonUtil::SerializeValue(name_, allocator).Move(), *allocator); + obj.AddMember(rapidjson::StringRef(kFieldPath), + RapidJsonUtil::SerializeValue(path_, allocator).Move(), *allocator); + obj.AddMember(rapidjson::StringRef(kFieldIsExternal), + RapidJsonUtil::SerializeValue(is_external_, allocator).Move(), *allocator); + obj.AddMember(rapidjson::StringRef(kFieldSchemaId), + RapidJsonUtil::SerializeValue(schema_id_, allocator).Move(), *allocator); + obj.AddMember(rapidjson::StringRef(kFieldSchema), + RestUtil::ParseToValue(schema_json_, allocator).Move(), *allocator); + audit_.AddTo(&obj, allocator); + return obj; +} + +void GetTableResponse::FromJson(const rapidjson::Value& obj) noexcept(false) { + id_ = RapidJsonUtil::DeserializeKeyValue(obj, kFieldId, std::string()); + database_ = RapidJsonUtil::DeserializeKeyValue(obj, kFieldDatabase, std::string()); + name_ = RapidJsonUtil::DeserializeKeyValue(obj, kFieldName, std::string()); + path_ = RapidJsonUtil::DeserializeKeyValue(obj, kFieldPath); + is_external_ = RapidJsonUtil::DeserializeKeyValue(obj, kFieldIsExternal, false); + schema_id_ = RapidJsonUtil::DeserializeKeyValue(obj, kFieldSchemaId); + schema_json_ = DeserializeRawJsonMember(obj, kFieldSchema); + audit_.ParseFrom(obj); +} + +rapidjson::Value CreateTableRequest::ToJson(rapidjson::Document::AllocatorType* allocator) const + noexcept(false) { + rapidjson::Value obj(rapidjson::kObjectType); + obj.AddMember(rapidjson::StringRef(kFieldIdentifier), + SerializeIdentifier(database_, table_, allocator).Move(), *allocator); + obj.AddMember(rapidjson::StringRef(kFieldSchema), + RestUtil::ParseToValue(schema_json_, allocator).Move(), *allocator); + return obj; +} + +void CreateTableRequest::FromJson(const rapidjson::Value& obj) noexcept(false) { + DeserializeIdentifier(obj, kFieldIdentifier, &database_, &table_); + schema_json_ = DeserializeRawJsonMember(obj, kFieldSchema); +} + +rapidjson::Value RenameTableRequest::ToJson(rapidjson::Document::AllocatorType* allocator) const + noexcept(false) { + rapidjson::Value obj(rapidjson::kObjectType); + obj.AddMember(rapidjson::StringRef(kFieldSource), + SerializeIdentifier(source_database_, source_table_, allocator).Move(), + *allocator); + obj.AddMember(rapidjson::StringRef(kFieldDestination), + SerializeIdentifier(destination_database_, destination_table_, allocator).Move(), + *allocator); + return obj; +} + +void RenameTableRequest::FromJson(const rapidjson::Value& obj) noexcept(false) { + DeserializeIdentifier(obj, kFieldSource, &source_database_, &source_table_); + DeserializeIdentifier(obj, kFieldDestination, &destination_database_, &destination_table_); +} + +} // namespace paimon diff --git a/src/paimon/rest/rest_messages.h b/src/paimon/rest/rest_messages.h new file mode 100644 index 00000000..b943e6e1 --- /dev/null +++ b/src/paimon/rest/rest_messages.h @@ -0,0 +1,389 @@ +/* + * Copyright 2026-present Alibaba Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include +#include +#include +#include +#include + +#include "paimon/common/utils/jsonizable.h" +#include "paimon/common/utils/rapidjson_util.h" +#include "paimon/core/snapshot.h" +#include "rapidjson/allocators.h" +#include "rapidjson/document.h" +#include "rapidjson/rapidjson.h" + +namespace paimon { + +// Request and response objects of the REST catalog protocol. The JSON field names must +// stay aligned with the REST catalog open api. + +class ErrorResponse : public Jsonizable { + public: + static constexpr const char* kResourceTypeDatabase = "DATABASE"; + static constexpr const char* kResourceTypeTable = "TABLE"; + + ErrorResponse(const std::string& resource_type, const std::string& resource_name, + const std::string& message, int32_t code) + : resource_type_(resource_type), + resource_name_(resource_name), + message_(message), + code_(code) {} + + rapidjson::Value ToJson(rapidjson::Document::AllocatorType* allocator) const + noexcept(false) override; + void FromJson(const rapidjson::Value& obj) noexcept(false) override; + + const std::string& GetResourceType() const { + return resource_type_; + } + const std::string& GetResourceName() const { + return resource_name_; + } + const std::string& GetMessage() const { + return message_; + } + int32_t GetCode() const { + return code_; + } + + ErrorResponse() = default; + + private: + std::string resource_type_; + std::string resource_name_; + std::string message_; + int32_t code_ = 0; +}; + +/// The response of "/v1/config". +class ConfigResponse : public Jsonizable { + public: + ConfigResponse(const std::map& defaults, + const std::map& overrides) + : defaults_(defaults), overrides_(overrides) {} + + rapidjson::Value ToJson(rapidjson::Document::AllocatorType* allocator) const + noexcept(false) override; + void FromJson(const rapidjson::Value& obj) noexcept(false) override; + + /// Merges with the client options; the precedence is + /// `overrides` > `client_options` > `defaults`. A key sent as null in `overrides` is + /// removed from the result, which is how the server unsets a client option; a null in + /// `defaults` is ignored. + std::map Merge( + const std::map& client_options) const; + + const std::map& GetDefaults() const { + return defaults_; + } + const std::map& GetOverrides() const { + return overrides_; + } + + ConfigResponse() = default; + + private: + std::map defaults_; + std::map overrides_; + /// Keys the server sent as null in `overrides`; `Merge` removes them from the result + /// and `ToJson` does not re-emit them. + std::set removed_keys_; +}; + +/// The audit fields shared by database/table responses. +struct RestAuditFields { + std::optional owner; + std::optional created_at; + std::optional created_by; + std::optional updated_at; + std::optional updated_by; + + void ParseFrom(const rapidjson::Value& obj); + void AddTo(rapidjson::Value* obj, rapidjson::Document::AllocatorType* allocator) const; + /// Adds the present fields to `options`, keyed by their JSON field names. + void PutAuditOptionsTo(std::map* options) const; +}; + +class CreateDatabaseRequest : public Jsonizable { + public: + CreateDatabaseRequest(const std::string& name, + const std::map& options) + : name_(name), options_(options) {} + + rapidjson::Value ToJson(rapidjson::Document::AllocatorType* allocator) const + noexcept(false) override; + void FromJson(const rapidjson::Value& obj) noexcept(false) override; + + const std::string& GetName() const { + return name_; + } + const std::map& GetOptions() const { + return options_; + } + + CreateDatabaseRequest() = default; + + private: + std::string name_; + std::map options_; +}; + +class GetDatabaseResponse : public Jsonizable { + public: + GetDatabaseResponse(const std::string& id, const std::string& name, const std::string& location, + const std::map& options) + : id_(id), name_(name), location_(location), options_(options) {} + + rapidjson::Value ToJson(rapidjson::Document::AllocatorType* allocator) const + noexcept(false) override; + void FromJson(const rapidjson::Value& obj) noexcept(false) override; + + const std::string& GetId() const { + return id_; + } + const std::string& GetName() const { + return name_; + } + const std::string& GetLocation() const { + return location_; + } + const std::map& GetOptions() const { + return options_; + } + const RestAuditFields& GetAuditFields() const { + return audit_; + } + + GetDatabaseResponse() = default; + + private: + std::string id_; + std::string name_; + std::string location_; + std::map options_; + RestAuditFields audit_; +}; + +class ListDatabasesResponse : public Jsonizable { + public: + using ItemType = std::string; + + ListDatabasesResponse(const std::vector& databases, + const std::optional& next_page_token) + : databases_(databases), next_page_token_(next_page_token) {} + + rapidjson::Value ToJson(rapidjson::Document::AllocatorType* allocator) const + noexcept(false) override; + void FromJson(const rapidjson::Value& obj) noexcept(false) override; + + const std::vector& Data() const { + return databases_; + } + const std::optional& NextPageToken() const { + return next_page_token_; + } + + ListDatabasesResponse() = default; + + private: + std::vector databases_; + std::optional next_page_token_; +}; + +class ListTablesResponse : public Jsonizable { + public: + using ItemType = std::string; + + ListTablesResponse(const std::vector& tables, + const std::optional& next_page_token) + : tables_(tables), next_page_token_(next_page_token) {} + + rapidjson::Value ToJson(rapidjson::Document::AllocatorType* allocator) const + noexcept(false) override; + void FromJson(const rapidjson::Value& obj) noexcept(false) override; + + const std::vector& Data() const { + return tables_; + } + const std::optional& NextPageToken() const { + return next_page_token_; + } + + ListTablesResponse() = default; + + private: + std::vector tables_; + std::optional next_page_token_; +}; + +/// Each element is a snapshot in the same JSON layout as the snapshot files. +class ListSnapshotsResponse : public Jsonizable { + public: + using ItemType = Snapshot; + + ListSnapshotsResponse(const std::vector& snapshots, + const std::optional& next_page_token) + : snapshots_(snapshots), next_page_token_(next_page_token) {} + + rapidjson::Value ToJson(rapidjson::Document::AllocatorType* allocator) const + noexcept(false) override; + void FromJson(const rapidjson::Value& obj) noexcept(false) override; + + const std::vector& Data() const { + return snapshots_; + } + const std::optional& NextPageToken() const { + return next_page_token_; + } + + ListSnapshotsResponse() = default; + + private: + std::vector snapshots_; + std::optional next_page_token_; +}; + +/// The nested `schema` object (fields/partitionKeys/primaryKeys/options/comment) is +/// kept as a raw JSON string and converted to a `TableSchema` by the catalog. +class GetTableResponse : public Jsonizable { + public: + GetTableResponse(const std::string& id, const std::string& database, const std::string& name, + const std::string& path, bool is_external, int64_t schema_id, + const std::string& schema_json) + : id_(id), + database_(database), + name_(name), + path_(path), + is_external_(is_external), + schema_id_(schema_id), + schema_json_(schema_json) {} + + rapidjson::Value ToJson(rapidjson::Document::AllocatorType* allocator) const + noexcept(false) override; + void FromJson(const rapidjson::Value& obj) noexcept(false) override; + + const std::string& GetId() const { + return id_; + } + /// The database of the table. The server reports it so that a table addressed by its + /// id alone can be resolved; it is empty on the endpoints that address a table by + /// database and name, which are the ones `RestCatalog` uses. + const std::string& GetDatabase() const { + return database_; + } + const std::string& GetName() const { + return name_; + } + const std::string& GetPath() const { + return path_; + } + bool IsExternal() const { + return is_external_; + } + int64_t GetSchemaId() const { + return schema_id_; + } + const std::string& GetSchemaJson() const { + return schema_json_; + } + const RestAuditFields& GetAuditFields() const { + return audit_; + } + + GetTableResponse() = default; + + private: + std::string id_; + std::string database_; + std::string name_; + std::string path_; + bool is_external_ = false; + int64_t schema_id_ = 0; + std::string schema_json_; + RestAuditFields audit_; +}; + +/// `schema_json` uses the same schema JSON layout as `GetTableResponse`. +class CreateTableRequest : public Jsonizable { + public: + CreateTableRequest(const std::string& database, const std::string& table, + const std::string& schema_json) + : database_(database), table_(table), schema_json_(schema_json) {} + + rapidjson::Value ToJson(rapidjson::Document::AllocatorType* allocator) const + noexcept(false) override; + void FromJson(const rapidjson::Value& obj) noexcept(false) override; + + const std::string& GetDatabase() const { + return database_; + } + const std::string& GetTable() const { + return table_; + } + const std::string& GetSchemaJson() const { + return schema_json_; + } + + CreateTableRequest() = default; + + private: + std::string database_; + std::string table_; + std::string schema_json_; +}; + +class RenameTableRequest : public Jsonizable { + public: + RenameTableRequest(const std::string& source_database, const std::string& source_table, + const std::string& destination_database, + const std::string& destination_table) + : source_database_(source_database), + source_table_(source_table), + destination_database_(destination_database), + destination_table_(destination_table) {} + + rapidjson::Value ToJson(rapidjson::Document::AllocatorType* allocator) const + noexcept(false) override; + void FromJson(const rapidjson::Value& obj) noexcept(false) override; + + const std::string& GetSourceDatabase() const { + return source_database_; + } + const std::string& GetSourceTable() const { + return source_table_; + } + const std::string& GetDestinationDatabase() const { + return destination_database_; + } + const std::string& GetDestinationTable() const { + return destination_table_; + } + + RenameTableRequest() = default; + + private: + std::string source_database_; + std::string source_table_; + std::string destination_database_; + std::string destination_table_; +}; + +} // namespace paimon diff --git a/src/paimon/rest/rest_messages_test.cpp b/src/paimon/rest/rest_messages_test.cpp new file mode 100644 index 00000000..7a29b77c --- /dev/null +++ b/src/paimon/rest/rest_messages_test.cpp @@ -0,0 +1,267 @@ +/* + * Copyright 2026-present Alibaba Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "paimon/rest/rest_messages.h" + +#include +#include + +#include "gtest/gtest.h" +#include "paimon/common/utils/rapidjson_util.h" +#include "paimon/testing/utils/testharness.h" +#include "rapidjson/document.h" + +namespace paimon::test { + +TEST(RestMessagesTest, ErrorResponseRoundTrip) { + ErrorResponse response(ErrorResponse::kResourceTypeDatabase, "db1", "database db1 not found", + 404); + ASSERT_OK_AND_ASSIGN(std::string json, response.ToJsonString()); + ASSERT_OK_AND_ASSIGN(ErrorResponse parsed, ErrorResponse::FromJsonString(json)); + ASSERT_EQ("DATABASE", parsed.GetResourceType()); + ASSERT_EQ("db1", parsed.GetResourceName()); + ASSERT_EQ("database db1 not found", parsed.GetMessage()); + ASSERT_EQ(404, parsed.GetCode()); +} + +TEST(RestMessagesTest, ErrorResponseLenientParse) { + ASSERT_OK_AND_ASSIGN(ErrorResponse parsed, ErrorResponse::FromJsonString("{}")); + ASSERT_EQ("", parsed.GetMessage()); + ASSERT_EQ(0, parsed.GetCode()); +} + +TEST(RestMessagesTest, ConfigResponseMerge) { + // null values may be sent by the server: a null default carries nothing to merge, + // a null override unsets the key + std::string json = R"({ + "defaults": {"prefix": "server-prefix", "a": "default-a", "b": "default-b", + "nullable": null, "unset-default": "default-value"}, + "overrides": {"c": "override-c", "a": "override-a", + "unset-client": null, "unset-default": null, "unset-absent": null} + })"; + ASSERT_OK_AND_ASSIGN(ConfigResponse config, ConfigResponse::FromJsonString(json)); + std::map client = { + {"a", "client-a"}, {"b", "client-b"}, {"d", "client-d"}, {"unset-client", "client-value"}}; + std::map merged = config.Merge(client); + // overrides > client options > defaults + ASSERT_EQ("override-a", merged["a"]); + ASSERT_EQ("client-b", merged["b"]); + ASSERT_EQ("override-c", merged["c"]); + ASSERT_EQ("client-d", merged["d"]); + ASSERT_EQ("server-prefix", merged["prefix"]); + ASSERT_EQ(0, merged.count("nullable")); + // a null override beats the client option and the default; on an unknown key it is + // a no-op + ASSERT_EQ(0, merged.count("unset-client")); + ASSERT_EQ(0, merged.count("unset-default")); + ASSERT_EQ(0, merged.count("unset-absent")); +} + +TEST(RestMessagesTest, ListResponsesParse) { + ASSERT_OK_AND_ASSIGN(ListDatabasesResponse databases, + ListDatabasesResponse::FromJsonString( + R"({"databases": ["db1", "db2"], "nextPageToken": "token1"})")); + ASSERT_EQ((std::vector{"db1", "db2"}), databases.Data()); + ASSERT_EQ("token1", databases.NextPageToken().value()); + + ASSERT_OK_AND_ASSIGN(ListTablesResponse tables, + ListTablesResponse::FromJsonString(R"({"tables": ["t1"]})")); + ASSERT_EQ((std::vector{"t1"}), tables.Data()); + ASSERT_FALSE(tables.NextPageToken().has_value()); + + // nextPageToken may be serialized as explicit null by the server + ASSERT_OK_AND_ASSIGN( + ListTablesResponse null_token, + ListTablesResponse::FromJsonString(R"({"tables": [], "nextPageToken": null})")); + ASSERT_FALSE(null_token.NextPageToken().has_value()); +} + +TEST(RestMessagesTest, ListSnapshotsResponseParse) { + std::string json = R"({ + "snapshots": [{ + "version": 3, + "id": 7, + "schemaId": 2, + "baseManifestList": "manifest-list-1", + "deltaManifestList": "manifest-list-2", + "commitUser": "user1", + "commitIdentifier": 9, + "commitKind": "APPEND", + "timeMillis": 1234567, + "totalRecordCount": 100, + "deltaRecordCount": 10, + "watermark": 42 + }, { + "version": 3, + "id": 8, + "schemaId": 2, + "baseManifestList": "manifest-list-3", + "deltaManifestList": "manifest-list-4", + "commitUser": "user1", + "commitIdentifier": 10, + "commitKind": "COMPACT", + "timeMillis": 1234568, + "totalRecordCount": 100, + "deltaRecordCount": 0 + }], + "nextPageToken": null + })"; + ASSERT_OK_AND_ASSIGN(ListSnapshotsResponse response, + ListSnapshotsResponse::FromJsonString(json)); + ASSERT_EQ(2, response.Data().size()); + const Snapshot& snapshot = response.Data()[0]; + ASSERT_EQ(7, snapshot.Id()); + ASSERT_EQ(2, snapshot.SchemaId()); + ASSERT_EQ("user1", snapshot.CommitUser()); + ASSERT_EQ(1234567, snapshot.TimeMillis()); + SnapshotInfo info = snapshot.ToSnapshotInfo(); + ASSERT_EQ(SnapshotInfo::CommitKind::APPEND, info.commit_kind); + ASSERT_EQ(100, info.total_record_count.value()); + ASSERT_EQ(42, info.watermark.value()); + SnapshotInfo compact_info = response.Data()[1].ToSnapshotInfo(); + ASSERT_EQ(SnapshotInfo::CommitKind::COMPACT, compact_info.commit_kind); + ASSERT_FALSE(compact_info.watermark.has_value()); +} + +TEST(RestMessagesTest, GetDatabaseResponseParse) { + std::string json = R"({ + "id": "10", + "name": "db1", + "location": "/warehouse/db1.db", + "options": {"k1": "v1"}, + "owner": "owner1", + "createdAt": 100, + "createdBy": "creator", + "updatedAt": 200, + "updatedBy": "updater" + })"; + ASSERT_OK_AND_ASSIGN(GetDatabaseResponse response, GetDatabaseResponse::FromJsonString(json)); + ASSERT_EQ("db1", response.GetName()); + ASSERT_EQ("/warehouse/db1.db", response.GetLocation()); + ASSERT_EQ("v1", response.GetOptions().at("k1")); + std::map options; + response.GetAuditFields().PutAuditOptionsTo(&options); + ASSERT_EQ("owner1", options.at("owner")); + ASSERT_EQ("100", options.at("createdAt")); + ASSERT_EQ("updater", options.at("updatedBy")); +} + +TEST(RestMessagesTest, GetTableResponseParse) { + std::string json = R"({ + "id": "42", + "database": "db1", + "name": "t1", + "path": "/warehouse/db1.db/t1", + "isExternal": false, + "schemaId": 3, + "schema": { + "fields": [ + {"id": 0, "name": "f0", "type": "INT NOT NULL"}, + {"id": 1, "name": "f1", "type": "STRING"} + ], + "partitionKeys": [], + "primaryKeys": ["f0"], + "options": {"bucket": "2"}, + "comment": "a table" + }, + "updatedAt": 300 + })"; + ASSERT_OK_AND_ASSIGN(GetTableResponse response, GetTableResponse::FromJsonString(json)); + ASSERT_EQ("42", response.GetId()); + ASSERT_EQ("db1", response.GetDatabase()); + ASSERT_EQ("t1", response.GetName()); + ASSERT_EQ("/warehouse/db1.db/t1", response.GetPath()); + ASSERT_FALSE(response.IsExternal()); + ASSERT_EQ(3, response.GetSchemaId()); + ASSERT_EQ(300, response.GetAuditFields().updated_at.value()); + rapidjson::Document schema; + schema.Parse(response.GetSchemaJson().c_str()); + ASSERT_FALSE(schema.HasParseError()); + ASSERT_EQ(2u, schema["fields"].Size()); + ASSERT_STREQ("f0", schema["fields"][0]["name"].GetString()); +} + +TEST(RestMessagesTest, UnknownFieldsAreIgnored) { + // forward compatibility: fields a newer server adds must be ignored + std::string json = R"({ + "id": "42", "name": "t1", "path": "p", "schemaId": 3, + "schema": {"fields": []}, + "futureField": {"nested": true}, "anotherUnknown": [1, 2] + })"; + ASSERT_OK_AND_ASSIGN(GetTableResponse response, GetTableResponse::FromJsonString(json)); + ASSERT_EQ("t1", response.GetName()); + ASSERT_EQ(3, response.GetSchemaId()); + // the endpoints addressing a table by database and name report no "database" + ASSERT_EQ("", response.GetDatabase()); +} + +TEST(RestMessagesTest, MissingRequiredFieldsFail) { + // "path", "schemaId" and "schema" are required + ASSERT_NOK(GetTableResponse::FromJsonString(R"({"id": "42", "name": "t1"})").status()); + // a non-object identifier is rejected + CreateTableRequest bad_identifier("", "", ""); + ASSERT_NOK(RapidJsonUtil::FromJsonString(R"({"identifier": "not-an-object", "schema": {}})", + &bad_identifier)); +} + +TEST(RestMessagesTest, CreateTableRequestSerialize) { + CreateTableRequest request( + "db1", "t1", R"({"fields": [], "partitionKeys": [], "primaryKeys": [], "options": {}})"); + ASSERT_OK_AND_ASSIGN(std::string json, request.ToJsonString()); + rapidjson::Document doc; + doc.Parse(json.c_str()); + ASSERT_FALSE(doc.HasParseError()); + ASSERT_STREQ("db1", doc["identifier"]["database"].GetString()); + ASSERT_STREQ("t1", doc["identifier"]["object"].GetString()); + ASSERT_TRUE(doc["schema"].IsObject()); + ASSERT_TRUE(doc["schema"]["fields"].IsArray()); + + ASSERT_OK_AND_ASSIGN(CreateTableRequest parsed, CreateTableRequest::FromJsonString(json)); + ASSERT_EQ("db1", parsed.GetDatabase()); + ASSERT_EQ("t1", parsed.GetTable()); +} + +TEST(RestMessagesTest, InvalidSchemaJsonErrorOmitsPayload) { + CreateTableRequest request("db1", "t1", R"({"fields": [], "options": {"token": "top-secret")"); + Status status = request.ToJsonString().status(); + ASSERT_NOK_WITH_MSG(status, "invalid json"); + // the payload may carry credentials, so it must not be echoed back in the error + ASSERT_EQ(std::string::npos, status.ToString().find("top-secret")) << status.ToString(); +} + +TEST(RestMessagesTest, RenameTableRequestSerialize) { + RenameTableRequest request("db1", "t1", "db1", "t2"); + ASSERT_OK_AND_ASSIGN(std::string json, request.ToJsonString()); + rapidjson::Document doc; + doc.Parse(json.c_str()); + ASSERT_FALSE(doc.HasParseError()); + ASSERT_STREQ("t1", doc["source"]["object"].GetString()); + ASSERT_STREQ("t2", doc["destination"]["object"].GetString()); + + ASSERT_OK_AND_ASSIGN(RenameTableRequest parsed, RenameTableRequest::FromJsonString(json)); + ASSERT_EQ("db1", parsed.GetSourceDatabase()); + ASSERT_EQ("t2", parsed.GetDestinationTable()); +} + +TEST(RestMessagesTest, CreateDatabaseRequestRoundTrip) { + CreateDatabaseRequest request("db1", {{"k1", "v1"}}); + ASSERT_OK_AND_ASSIGN(std::string json, request.ToJsonString()); + ASSERT_OK_AND_ASSIGN(CreateDatabaseRequest parsed, CreateDatabaseRequest::FromJsonString(json)); + ASSERT_EQ("db1", parsed.GetName()); + ASSERT_EQ("v1", parsed.GetOptions().at("k1")); +} + +} // namespace paimon::test diff --git a/src/paimon/rest/rest_util.cpp b/src/paimon/rest/rest_util.cpp new file mode 100644 index 00000000..b887d256 --- /dev/null +++ b/src/paimon/rest/rest_util.cpp @@ -0,0 +1,74 @@ +/* + * Copyright 2026-present Alibaba Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "paimon/rest/rest_util.h" + +#include + +#include "fmt/format.h" +#include "rapidjson/error/en.h" +#include "rapidjson/stringbuffer.h" +#include "rapidjson/writer.h" + +namespace paimon { + +std::map RestUtil::ExtractPrefixMap( + const std::map& options, const std::string& prefix) { + std::map result; + for (const auto& [key, value] : options) { + if (key.size() > prefix.size() && key.compare(0, prefix.size(), prefix) == 0) { + result[key.substr(prefix.size())] = value; + } + } + return result; +} + +std::string RestUtil::ExtractRequestId(const std::map& headers) { + auto iter = headers.find(kRequestIdHeader); + if (iter != headers.end() && !iter->second.empty()) { + return iter->second; + } + for (const auto& [name, value] : headers) { + if (!value.empty() && name.find("request-id") != std::string::npos) { + return value; + } + } + return kUnknownRequestId; +} + +std::string RestUtil::JsonToString(const rapidjson::Value& value) { + rapidjson::StringBuffer buffer; + rapidjson::Writer writer(buffer); + value.Accept(writer); + return buffer.GetString(); +} + +rapidjson::Value RestUtil::ParseToValue(const std::string& json, + rapidjson::Document::AllocatorType* allocator) { + rapidjson::Document doc; + doc.Parse(json.c_str()); + if (doc.HasParseError()) { + // The payload may carry credentials and be arbitrarily large; report only the error. + throw std::invalid_argument(fmt::format("invalid json: {} (at offset {})", + rapidjson::GetParseError_En(doc.GetParseError()), + doc.GetErrorOffset())); + } + rapidjson::Value value; + value.CopyFrom(doc, *allocator); + return value; +} + +} // namespace paimon diff --git a/src/paimon/rest/rest_util.h b/src/paimon/rest/rest_util.h new file mode 100644 index 00000000..407217c6 --- /dev/null +++ b/src/paimon/rest/rest_util.h @@ -0,0 +1,61 @@ +/* + * Copyright 2026-present Alibaba Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include + +#include "rapidjson/allocators.h" +#include "rapidjson/document.h" +#include "rapidjson/rapidjson.h" + +namespace paimon { + +/// Utilities for the REST catalog; URL encoding and decoding live in `UrlUtils`, +/// credential redaction in `SensitiveConfigUtils`. +class RestUtil { + public: + RestUtil() = delete; + ~RestUtil() = delete; + + /// Header under which the rest server reports the id of a request. + static constexpr const char* kRequestIdHeader = "x-request-id"; + /// Placeholder `ExtractRequestId` returns when no header carries a request id. + static constexpr const char* kUnknownRequestId = "unknown"; + + /// Extract all options whose key starts with `prefix`, with the prefix stripped from + /// the resulting keys. A key exactly equal to the prefix is dropped instead of + /// yielding an empty key. + static std::map ExtractPrefixMap( + const std::map& options, const std::string& prefix); + + /// Returns the request id tying a log line or an error to the server side trace: the + /// `kRequestIdHeader` value, falling back to any other header whose name contains + /// "request-id" (a gateway may report it under its own name, e.g. "x-amz-request-id"), + /// then to `kUnknownRequestId`. `headers` must have lower-cased names. + static std::string ExtractRequestId(const std::map& headers); + + /// Serialize a rapidjson value to a compact JSON string. + static std::string JsonToString(const rapidjson::Value& value); + + /// Parse a JSON string into a rapidjson value owned by `allocator`. + /// Throws `std::invalid_argument` on parse error, consistent with `Jsonizable`. + static rapidjson::Value ParseToValue(const std::string& json, + rapidjson::Document::AllocatorType* allocator); +}; + +} // namespace paimon diff --git a/src/paimon/rest/rest_util_test.cpp b/src/paimon/rest/rest_util_test.cpp new file mode 100644 index 00000000..e76baf15 --- /dev/null +++ b/src/paimon/rest/rest_util_test.cpp @@ -0,0 +1,47 @@ +/* + * Copyright 2026-present Alibaba Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "paimon/rest/rest_util.h" + +#include +#include + +#include "gtest/gtest.h" + +namespace paimon::test { + +TEST(RestUtilTest, ExtractPrefixMap) { + std::map options = { + {"header.k1", "v1"}, {"header.k2", "v2"}, {"other", "v3"}, {"header.", "v4"}}; + std::map expected = {{"k1", "v1"}, {"k2", "v2"}}; + ASSERT_EQ(expected, RestUtil::ExtractPrefixMap(options, "header.")); +} + +TEST(RestUtilTest, ExtractRequestId) { + ASSERT_EQ("req-1", RestUtil::ExtractRequestId({{"x-request-id", "req-1"}})); + // a gateway may report the request id under a name of its own + ASSERT_EQ("amz-1", RestUtil::ExtractRequestId({{"x-amz-request-id", "amz-1"}})); + // the dedicated header wins over a gateway one + ASSERT_EQ("req-1", RestUtil::ExtractRequestId( + {{"x-request-id", "req-1"}, {"x-amz-request-id", "amz-1"}})); + // an empty value carries no id and falls through like an absent header + ASSERT_EQ("amz-1", + RestUtil::ExtractRequestId({{"x-request-id", ""}, {"x-amz-request-id", "amz-1"}})); + ASSERT_EQ(RestUtil::kUnknownRequestId, RestUtil::ExtractRequestId({})); + ASSERT_EQ(RestUtil::kUnknownRequestId, RestUtil::ExtractRequestId({{"content-type", "json"}})); +} + +} // namespace paimon::test diff --git a/test/inte/read_inte_test.cpp b/test/inte/read_inte_test.cpp index 9ebf2d60..4ad82d0c 100644 --- a/test/inte/read_inte_test.cpp +++ b/test/inte/read_inte_test.cpp @@ -3915,7 +3915,14 @@ TEST(SystemTableReadInteTest, TestReadGlobalCatalogOptions) { {Options::FILE_SYSTEM, "local"}, {Options::FILE_FORMAT, "orc"}, {CatalogOptionsSystemTable::kEnabledOption, "true"}, - {"custom.catalog.option", "test-value"}}; + {"custom.catalog.option", "test-value"}, + {"token", "bearer-credential-1"}, + {"dlf.access-key-secret", "aksecret-1"}, + {"fs.s3a.access.key", "s3akey-1"}, + {"fs.azure.account-key.store1", "azkey-1"}, + {"client.credential", "cred-1"}, + {"fs.azure.sas.container", "sas-1"}, + {"dlf.access-key-id", "an-access-key-id-1"}}; auto dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); std::string warehouse = PathUtil::JoinPath(dir->Str(), "warehouse"); @@ -3937,6 +3944,26 @@ TEST(SystemTableReadInteTest, TestReadGlobalCatalogOptions) { } ASSERT_EQ(result_map["file-system"], "local"); ASSERT_EQ(result_map["file.format"], "orc"); + // credential-carrying options stay listed but their values are masked; the key match + // ignores separators, so "fs.s3a.access.key" hits the "accesskey" marker + ASSERT_EQ(result_map["token"], "******"); + ASSERT_EQ(result_map["dlf.access-key-secret"], "******"); + ASSERT_EQ(result_map["fs.s3a.access.key"], "******"); + ASSERT_EQ(result_map["fs.azure.account-key.store1"], "******"); + ASSERT_EQ(result_map["client.credential"], "******"); + ASSERT_EQ(result_map["fs.azure.sas.container"], "******"); + // an identifier-like key keeps a four character tail, which names the credential in a + // support case without disclosing it + ASSERT_EQ(result_map["dlf.access-key-id"], "****id-1"); + for (const auto& [key, value] : result_map) { + ASSERT_EQ(std::string::npos, value.find("bearer-credential-1")) << key; + ASSERT_EQ(std::string::npos, value.find("aksecret-1")) << key; + ASSERT_EQ(std::string::npos, value.find("s3akey-1")) << key; + ASSERT_EQ(std::string::npos, value.find("azkey-1")) << key; + ASSERT_EQ(std::string::npos, value.find("cred-1")) << key; + ASSERT_EQ(std::string::npos, value.find("sas-1")) << key; + ASSERT_EQ(std::string::npos, value.find("an-access-key-id-1")) << key; + } } TEST(SystemTableReadInteTest, TestReadGlobalAllTableOptions) {