From e69ff7054375a223980779652c36e9e9a0bd3446 Mon Sep 17 00:00:00 2001 From: Li Feiyang Date: Sat, 27 Sep 2025 18:31:51 +0800 Subject: [PATCH 1/4] refactor: Restructure catalog module --- example/demo_example.cc | 2 +- src/iceberg/CMakeLists.txt | 2 +- src/iceberg/catalog/CMakeLists.txt | 1 + src/iceberg/catalog/memory/CMakeLists.txt | 18 +++ .../catalog/{ => memory}/in_memory_catalog.cc | 2 +- .../catalog/{ => memory}/in_memory_catalog.h | 0 test/CMakeLists.txt | 18 ++- test/in_memory_catalog_test.cc | 2 +- test/rest_catalog_test.cc | 111 ++++++++++++++++++ 9 files changed, 151 insertions(+), 5 deletions(-) create mode 100644 src/iceberg/catalog/memory/CMakeLists.txt rename src/iceberg/catalog/{ => memory}/in_memory_catalog.cc (99%) rename src/iceberg/catalog/{ => memory}/in_memory_catalog.h (100%) create mode 100644 test/rest_catalog_test.cc diff --git a/example/demo_example.cc b/example/demo_example.cc index d20f48c0..c333c797 100644 --- a/example/demo_example.cc +++ b/example/demo_example.cc @@ -21,7 +21,7 @@ #include "iceberg/arrow/arrow_file_io.h" #include "iceberg/avro/avro_register.h" -#include "iceberg/catalog/in_memory_catalog.h" +#include "iceberg/catalog/memory/in_memory_catalog.h" #include "iceberg/parquet/parquet_register.h" #include "iceberg/table.h" #include "iceberg/table_scan.h" diff --git a/src/iceberg/CMakeLists.txt b/src/iceberg/CMakeLists.txt index c8fb0772..055f9b5f 100644 --- a/src/iceberg/CMakeLists.txt +++ b/src/iceberg/CMakeLists.txt @@ -18,7 +18,7 @@ set(ICEBERG_INCLUDES "$" "$") set(ICEBERG_SOURCES - catalog/in_memory_catalog.cc + catalog/memory/in_memory_catalog.cc expression/expression.cc expression/literal.cc file_reader.cc diff --git a/src/iceberg/catalog/CMakeLists.txt b/src/iceberg/catalog/CMakeLists.txt index 9e11eee3..bcfb5f0c 100644 --- a/src/iceberg/catalog/CMakeLists.txt +++ b/src/iceberg/catalog/CMakeLists.txt @@ -16,6 +16,7 @@ # under the License. iceberg_install_all_headers(iceberg/catalog) +add_subdirectory(memory) if(ICEBERG_BUILD_REST) add_subdirectory(rest) diff --git a/src/iceberg/catalog/memory/CMakeLists.txt b/src/iceberg/catalog/memory/CMakeLists.txt new file mode 100644 index 00000000..306a4a7f --- /dev/null +++ b/src/iceberg/catalog/memory/CMakeLists.txt @@ -0,0 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +iceberg_install_all_headers(iceberg/catalog/memory) diff --git a/src/iceberg/catalog/in_memory_catalog.cc b/src/iceberg/catalog/memory/in_memory_catalog.cc similarity index 99% rename from src/iceberg/catalog/in_memory_catalog.cc rename to src/iceberg/catalog/memory/in_memory_catalog.cc index 0215b908..08a9822c 100644 --- a/src/iceberg/catalog/in_memory_catalog.cc +++ b/src/iceberg/catalog/memory/in_memory_catalog.cc @@ -17,7 +17,7 @@ * under the License. */ -#include "iceberg/catalog/in_memory_catalog.h" +#include "iceberg/catalog/memory/in_memory_catalog.h" #include #include // IWYU pragma: keep diff --git a/src/iceberg/catalog/in_memory_catalog.h b/src/iceberg/catalog/memory/in_memory_catalog.h similarity index 100% rename from src/iceberg/catalog/in_memory_catalog.h rename to src/iceberg/catalog/memory/in_memory_catalog.h diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index d70b4f85..bcd62032 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -15,13 +15,23 @@ # specific language governing permissions and limitations # under the License. +fetchcontent_declare(cpp-httplib + GIT_REPOSITORY https://github.com/yhirose/cpp-httplib.git + GIT_TAG 89c932f313c6437c38f2982869beacc89c2f2246 #release-0.26.0 +) + fetchcontent_declare(googletest GIT_REPOSITORY https://github.com/google/googletest.git GIT_TAG b514bdc898e2951020cbdca1304b75f5950d1f59 # release-1.15.2 FIND_PACKAGE_ARGS NAMES GTest) -fetchcontent_makeavailable(googletest) + +if(ICEBERG_BUILD_REST_CATALOG) + fetchcontent_makeavailable(cpp-httplib googletest) +else() + fetchcontent_makeavailable(googletest) +endif() set(ICEBERG_TEST_RESOURCES "${CMAKE_SOURCE_DIR}/test/resources") @@ -131,3 +141,9 @@ if(ICEBERG_BUILD_BUNDLE) add_iceberg_test(scan_test USE_BUNDLE SOURCES file_scan_task_test.cc) endif() + +if(ICEBERG_BUILD_REST_CATALOG) + add_iceberg_test(rest_catalog_test SOURCES rest_catalog_test.cc) + target_link_libraries(rest_catalog_test PRIVATE iceberg_rest_catalog_static) + target_include_directories(rest_catalog_test PRIVATE ${cpp-httplib_SOURCE_DIR}) +endif() diff --git a/test/in_memory_catalog_test.cc b/test/in_memory_catalog_test.cc index 36d67cbe..da1804b8 100644 --- a/test/in_memory_catalog_test.cc +++ b/test/in_memory_catalog_test.cc @@ -17,7 +17,7 @@ * under the License. */ -#include "iceberg/catalog/in_memory_catalog.h" +#include "iceberg/catalog/memory/in_memory_catalog.h" #include diff --git a/test/rest_catalog_test.cc b/test/rest_catalog_test.cc new file mode 100644 index 00000000..b10f8c53 --- /dev/null +++ b/test/rest_catalog_test.cc @@ -0,0 +1,111 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "iceberg/catalog/rest/rest_catalog.h" + +#include + +#include +#include + +#include +#include +#include + +namespace iceberg::catalog::rest { + +class RestCatalogIntegrationTest : public ::testing::Test { + protected: + void SetUp() override { + server_ = std::make_unique(); + port_ = server_->bind_to_any_port("127.0.0.1"); + + server_thread_ = std::thread([this]() { server_->listen_after_bind(); }); + } + + void TearDown() override { + server_->stop(); + if (server_thread_.joinable()) { + server_thread_.join(); + } + } + + std::unique_ptr server_; + int port_ = -1; + std::thread server_thread_; +}; + +TEST_F(RestCatalogIntegrationTest, GetConfigSuccessfully) { + server_->Get("/v1/config", [](const httplib::Request&, httplib::Response& res) { + res.status = 200; + res.set_content(R"({"warehouse": "s3://test-bucket"})", "application/json"); + }); + + std::string base_uri = "http://127.0.0.1:" + std::to_string(port_); + RestCatalog catalog(base_uri); + cpr::Response response = catalog.GetConfig(); + + ASSERT_EQ(response.error.code, cpr::ErrorCode::OK); + ASSERT_EQ(response.status_code, 200); + + auto json_body = nlohmann::json::parse(response.text); + EXPECT_EQ(json_body["warehouse"], "s3://test-bucket"); +} + +TEST_F(RestCatalogIntegrationTest, ListNamespacesReturnsMultipleResults) { + server_->Get("/v1/namespaces", [](const httplib::Request&, httplib::Response& res) { + res.status = 200; + res.set_content(R"({ + "namespaces": [ + ["accounting", "db"], + ["production", "db"] + ] + })", + "application/json"); + }); + + std::string base_uri = "http://127.0.0.1:" + std::to_string(port_); + RestCatalog catalog(base_uri); + cpr::Response response = catalog.ListNamespaces(); + + ASSERT_EQ(response.error.code, cpr::ErrorCode::OK); + ASSERT_EQ(response.status_code, 200); + + auto json_body = nlohmann::json::parse(response.text); + ASSERT_TRUE(json_body.contains("namespaces")); + EXPECT_EQ(json_body["namespaces"].size(), 2); + EXPECT_THAT(json_body["namespaces"][0][0], "accounting"); +} + +TEST_F(RestCatalogIntegrationTest, HandlesServerError) { + server_->Get("/v1/config", [](const httplib::Request&, httplib::Response& res) { + res.status = 500; + res.set_content("Internal Server Error", "text/plain"); + }); + + std::string base_uri = "http://127.0.0.1:" + std::to_string(port_); + RestCatalog catalog(base_uri); + cpr::Response response = catalog.GetConfig(); + + ASSERT_EQ(response.error.code, cpr::ErrorCode::OK); + ASSERT_EQ(response.status_code, 500); + ASSERT_EQ(response.text, "Internal Server Error"); +} + +} // namespace iceberg::catalog::rest From e7db68e3ebd21f3ac8502964ffcdbc06076d6d33 Mon Sep 17 00:00:00 2001 From: Li Feiyang Date: Sat, 27 Sep 2025 18:40:15 +0800 Subject: [PATCH 2/4] add json dep for rest --- src/iceberg/catalog/rest/CMakeLists.txt | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/src/iceberg/catalog/rest/CMakeLists.txt b/src/iceberg/catalog/rest/CMakeLists.txt index f18859cf..460abc7e 100644 --- a/src/iceberg/catalog/rest/CMakeLists.txt +++ b/src/iceberg/catalog/rest/CMakeLists.txt @@ -22,19 +22,28 @@ set(ICEBERG_REST_SHARED_BUILD_INTERFACE_LIBS) set(ICEBERG_REST_STATIC_INSTALL_INTERFACE_LIBS) set(ICEBERG_REST_SHARED_INSTALL_INTERFACE_LIBS) -list(APPEND ICEBERG_REST_STATIC_BUILD_INTERFACE_LIBS - "$,iceberg_static,iceberg_shared>" cpr::cpr) -list(APPEND ICEBERG_REST_SHARED_BUILD_INTERFACE_LIBS - "$,iceberg_shared,iceberg_static>" cpr::cpr) +list(APPEND + ICEBERG_REST_STATIC_BUILD_INTERFACE_LIBS + "$,iceberg_static,iceberg_shared>" + cpr::cpr + nlohmann_json::nlohmann_json) +list(APPEND + ICEBERG_REST_SHARED_BUILD_INTERFACE_LIBS + "$,iceberg_shared,iceberg_static>" + cpr::cpr + nlohmann_json::nlohmann_json) list(APPEND ICEBERG_REST_STATIC_INSTALL_INTERFACE_LIBS "$,Iceberg::iceberg_static,Iceberg::iceberg_shared>" - "$,Iceberg::cpr,cpr::cpr>") + "$,Iceberg::cpr,cpr::cpr>" + "$,Iceberg::nlohmann_json,nlohmann_json::nlohmann_json>" +) list(APPEND ICEBERG_REST_SHARED_INSTALL_INTERFACE_LIBS "$,Iceberg::iceberg_shared,Iceberg::iceberg_static>" - "$,Iceberg::cpr,cpr::cpr>") - + "$,Iceberg::cpr,cpr::cpr>" + "$,Iceberg::nlohmann_json,nlohmann_json::nlohmann_json>" +) add_iceberg_lib(iceberg_rest SOURCES ${ICEBERG_REST_SOURCES} From b830c93e0384299322790cce2c886d4d114513e4 Mon Sep 17 00:00:00 2001 From: Li Feiyang Date: Sun, 28 Sep 2025 16:44:14 +0800 Subject: [PATCH 3/4] fix --- test/CMakeLists.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index bcd62032..bba070c0 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -27,7 +27,7 @@ fetchcontent_declare(googletest NAMES GTest) -if(ICEBERG_BUILD_REST_CATALOG) +if(ICEBERG_BUILD_REST) fetchcontent_makeavailable(cpp-httplib googletest) else() fetchcontent_makeavailable(googletest) @@ -142,8 +142,8 @@ if(ICEBERG_BUILD_BUNDLE) endif() -if(ICEBERG_BUILD_REST_CATALOG) +if(ICEBERG_BUILD_REST) add_iceberg_test(rest_catalog_test SOURCES rest_catalog_test.cc) - target_link_libraries(rest_catalog_test PRIVATE iceberg_rest_catalog_static) + target_link_libraries(rest_catalog_test PRIVATE iceberg_rest_static) target_include_directories(rest_catalog_test PRIVATE ${cpp-httplib_SOURCE_DIR}) endif() From 7531e29dd918304d3c105cb2b6240bf6ba832263 Mon Sep 17 00:00:00 2001 From: Li Feiyang Date: Mon, 29 Sep 2025 10:57:44 +0800 Subject: [PATCH 4/4] first version --- src/iceberg/catalog/rest/CMakeLists.txt | 2 +- src/iceberg/catalog/rest/json_internal.cc | 300 ++++++++++++++++++ src/iceberg/catalog/rest/json_internal.h | 129 ++++++++ src/iceberg/catalog/rest/types.h | 65 ++++ src/iceberg/json_internal.cc | 180 +---------- src/iceberg/util/json_util_internal.h | 214 +++++++++++++ test/CMakeLists.txt | 3 +- test/rest_json_internal_test.cc | 361 ++++++++++++++++++++++ 8 files changed, 1073 insertions(+), 181 deletions(-) create mode 100644 src/iceberg/catalog/rest/json_internal.cc create mode 100644 src/iceberg/catalog/rest/json_internal.h create mode 100644 src/iceberg/catalog/rest/types.h create mode 100644 src/iceberg/util/json_util_internal.h create mode 100644 test/rest_json_internal_test.cc diff --git a/src/iceberg/catalog/rest/CMakeLists.txt b/src/iceberg/catalog/rest/CMakeLists.txt index 460abc7e..bbe4c187 100644 --- a/src/iceberg/catalog/rest/CMakeLists.txt +++ b/src/iceberg/catalog/rest/CMakeLists.txt @@ -15,7 +15,7 @@ # specific language governing permissions and limitations # under the License. -set(ICEBERG_REST_SOURCES rest_catalog.cc) +set(ICEBERG_REST_SOURCES rest_catalog.cc json_internal.cc) set(ICEBERG_REST_STATIC_BUILD_INTERFACE_LIBS) set(ICEBERG_REST_SHARED_BUILD_INTERFACE_LIBS) diff --git a/src/iceberg/catalog/rest/json_internal.cc b/src/iceberg/catalog/rest/json_internal.cc new file mode 100644 index 00000000..1d0cf4a5 --- /dev/null +++ b/src/iceberg/catalog/rest/json_internal.cc @@ -0,0 +1,300 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "iceberg/catalog/rest/json_internal.h" + +#include +#include +#include + +#include + +#include "iceberg/json_internal.h" +#include "iceberg/partition_spec.h" +#include "iceberg/result.h" +#include "iceberg/schema.h" +#include "iceberg/sort_order.h" +#include "iceberg/table_identifier.h" +#include "iceberg/table_metadata.h" +#include "iceberg/util/json_util_internal.h" +#include "iceberg/util/macros.h" + +namespace iceberg::rest { + +namespace { + +// REST API JSON field constants +constexpr std::string_view kNamespaces = "namespaces"; +constexpr std::string_view kRemovals = "removals"; +constexpr std::string_view kUpdates = "updates"; +constexpr std::string_view kUpdated = "updated"; +constexpr std::string_view kRemoved = "removed"; +constexpr std::string_view kMissing = "missing"; +constexpr std::string_view kIdentifiers = "identifiers"; +constexpr std::string_view kSource = "source"; +constexpr std::string_view kDestination = "destination"; +constexpr std::string_view kMetadataLocation = "metadata-location"; +constexpr std::string_view kMetadata = "metadata"; +constexpr std::string_view kConfig = "config"; +constexpr std::string_view kName = "name"; +constexpr std::string_view kLocation = "location"; +constexpr std::string_view kSchema = "schema"; +constexpr std::string_view kPartitionSpec = "partition-spec"; +constexpr std::string_view kWriteOrder = "write-order"; +constexpr std::string_view kStageCreate = "stage-create"; +constexpr std::string_view kProperties = "properties"; +constexpr std::string_view kOverwrite = "overwrite"; +constexpr std::string_view kNamespace = "namespace"; + +/// Helper function to convert TableIdentifier to JSON +nlohmann::json TableIdentifierToJson(const TableIdentifier& identifier) { + nlohmann::json json; + json[kNamespace] = identifier.ns.levels; + json[kName] = identifier.name; + return json; +} + +/// Helper function to parse TableIdentifier from JSON +Result TableIdentifierFromJson(const nlohmann::json& json) { + TableIdentifier identifier; + + if (json.contains(kNamespace)) { + ICEBERG_ASSIGN_OR_RAISE(identifier.ns.levels, + GetJsonValue>(json, kNamespace)); + } + + ICEBERG_ASSIGN_OR_RAISE(identifier.name, GetJsonValue(json, kName)); + + return identifier; +} + +} // namespace + +nlohmann::json ToJson(const ListNamespaceResponse& response) { + nlohmann::json json; + json[kNamespaces] = response.namespaces; + return json; +} + +Result ListNamespaceResponseFromJson(const nlohmann::json& json) { + ListNamespaceResponse response; + + ICEBERG_ASSIGN_OR_RAISE( + response.namespaces, + GetJsonValue>>(json, kNamespaces)); + return response; +} + +nlohmann::json ToJson(const UpdateNamespacePropsRequest& request) { + nlohmann::json json; + SetOptionalField(json, kRemovals, request.removals); + SetOptionalField(json, kUpdates, request.updates); + return json; +} + +Result UpdateNamespacePropsRequestFromJson( + const nlohmann::json& json) { + UpdateNamespacePropsRequest request; + + ICEBERG_ASSIGN_OR_RAISE( + request.removals, GetJsonValueOptional>(json, kRemovals)); + using MapType = std::unordered_map; + ICEBERG_ASSIGN_OR_RAISE(request.updates, GetJsonValueOptional(json, kUpdates)); + + return request; +} + +nlohmann::json ToJson(const UpdateNamespacePropsResponse& response) { + nlohmann::json json; + json[kUpdated] = response.updated; + json[kRemoved] = response.removed; + SetOptionalField(json, kMissing, response.missing); + return json; +} + +Result UpdateNamespacePropsResponseFromJson( + const nlohmann::json& json) { + UpdateNamespacePropsResponse response; + + ICEBERG_ASSIGN_OR_RAISE(response.updated, + GetJsonValue>(json, kUpdated)); + ICEBERG_ASSIGN_OR_RAISE(response.removed, + GetJsonValue>(json, kRemoved)); + ICEBERG_ASSIGN_OR_RAISE(response.missing, + GetJsonValueOptional>(json, kMissing)); + + return response; +} + +nlohmann::json ToJson(const ListTableResponse& response) { + nlohmann::json json; + + nlohmann::json identifiers_json = nlohmann::json::array(); + for (const auto& identifier : response.identifiers) { + identifiers_json.push_back(TableIdentifierToJson(identifier)); + } + json[kIdentifiers] = identifiers_json; + return json; +} + +Result ListTableResponseFromJson(const nlohmann::json& json) { + ListTableResponse response; + + ICEBERG_ASSIGN_OR_RAISE(auto identifiers_json, + GetJsonValue(json, kIdentifiers)); + + for (const auto& id_json : identifiers_json) { + ICEBERG_ASSIGN_OR_RAISE(auto identifier, TableIdentifierFromJson(id_json)); + response.identifiers.push_back(std::move(identifier)); + } + return response; +} + +nlohmann::json ToJson(const RenameTableRequest& request) { + nlohmann::json json; + json[kSource] = TableIdentifierToJson(request.source); + json[kDestination] = TableIdentifierToJson(request.destination); + return json; +} + +Result RenameTableRequestFromJson(const nlohmann::json& json) { + RenameTableRequest request; + + ICEBERG_ASSIGN_OR_RAISE(auto source_json, GetJsonValue(json, kSource)); + ICEBERG_ASSIGN_OR_RAISE(request.source, TableIdentifierFromJson(source_json)); + + ICEBERG_ASSIGN_OR_RAISE(auto dest_json, + GetJsonValue(json, kDestination)); + ICEBERG_ASSIGN_OR_RAISE(request.destination, TableIdentifierFromJson(dest_json)); + + return request; +} + +nlohmann::json ToJson(const LoadTableResponse& response) { + nlohmann::json json; + + SetOptionalField(json, kMetadataLocation, response.metadata_location); + json[kMetadata] = iceberg::ToJson(response.metadata); + SetOptionalField(json, kConfig, response.config); + + return json; +} + +Result LoadTableResponseFromJson(const nlohmann::json& json) { + LoadTableResponse response; + + ICEBERG_ASSIGN_OR_RAISE(response.metadata_location, + GetJsonValueOptional(json, kMetadataLocation)); + + ICEBERG_ASSIGN_OR_RAISE(auto metadata_json, + GetJsonValue(json, kMetadata)); + ICEBERG_ASSIGN_OR_RAISE(auto metadata_ptr, + iceberg::TableMetadataFromJson(metadata_json)); + response.metadata = std::move(*metadata_ptr); + + using MapType = std::unordered_map; + ICEBERG_ASSIGN_OR_RAISE(response.config, GetJsonValueOptional(json, kConfig)); + + return response; +} + +nlohmann::json ToJson(const CreateTableRequest& request) { + nlohmann::json json; + json[kName] = request.name; + SetOptionalField(json, kLocation, request.location); + json[kSchema] = ToJson(*request.schema); + + if (request.partition_spec) { + json[kPartitionSpec] = ToJson(*request.partition_spec); + } + + if (request.write_order) { + json[kWriteOrder] = ToJson(*request.write_order); + } + + SetOptionalField(json, kStageCreate, request.stage_create); + SetOptionalField(json, kProperties, request.properties); + return json; +} + +Result CreateTableRequestFromJson(const nlohmann::json& json) { + CreateTableRequest request; + + // Parse required fields + ICEBERG_ASSIGN_OR_RAISE(request.name, GetJsonValue(json, kName)); + + // Parse optional location + ICEBERG_ASSIGN_OR_RAISE(request.location, + GetJsonValueOptional(json, kLocation)); + + // Parse required schema + ICEBERG_ASSIGN_OR_RAISE(auto schema_json, GetJsonValue(json, kSchema)); + ICEBERG_ASSIGN_OR_RAISE(auto schema_ptr, iceberg::SchemaFromJson(schema_json)); + request.schema = std::move(schema_ptr); + + if (json.contains(kPartitionSpec)) { + ICEBERG_ASSIGN_OR_RAISE(auto partition_spec_json, + GetJsonValue(json, kPartitionSpec)); + ICEBERG_ASSIGN_OR_RAISE( + request.partition_spec, + iceberg::PartitionSpecFromJson(request.schema, partition_spec_json)); + } else { + request.partition_spec = nullptr; + } + + if (json.contains(kWriteOrder)) { + ICEBERG_ASSIGN_OR_RAISE(auto write_order_json, + GetJsonValue(json, kWriteOrder)); + ICEBERG_ASSIGN_OR_RAISE(request.write_order, + iceberg::SortOrderFromJson(write_order_json)); + } else { + request.write_order = nullptr; + } + + ICEBERG_ASSIGN_OR_RAISE(request.stage_create, + GetJsonValueOptional(json, kStageCreate)); + + using MapType = std::unordered_map; + ICEBERG_ASSIGN_OR_RAISE(request.properties, + GetJsonValueOptional(json, kProperties)); + + return request; +} + +nlohmann::json ToJson(const RegisterTableRequest& request) { + nlohmann::json json; + json[kName] = request.name; + json[kMetadataLocation] = request.metadata_location; + SetOptionalField(json, kOverwrite, request.overwrite); + return json; +} + +Result RegisterTableRequestFromJson(const nlohmann::json& json) { + RegisterTableRequest request; + + ICEBERG_ASSIGN_OR_RAISE(request.name, GetJsonValue(json, kName)); + ICEBERG_ASSIGN_OR_RAISE(request.metadata_location, + GetJsonValue(json, kMetadataLocation)); + ICEBERG_ASSIGN_OR_RAISE(request.overwrite, + GetJsonValueOptional(json, kOverwrite)); + + return request; +} + +} // namespace iceberg::rest diff --git a/src/iceberg/catalog/rest/json_internal.h b/src/iceberg/catalog/rest/json_internal.h new file mode 100644 index 00000000..474c16bb --- /dev/null +++ b/src/iceberg/catalog/rest/json_internal.h @@ -0,0 +1,129 @@ +// json_internal.h + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include + +#include "iceberg/catalog/rest/types.h" +#include "iceberg/result.h" + +namespace iceberg::rest { + +/// \brief Serializes a `ListNamespaceResponse` object to JSON. +/// +/// \param response The `ListNamespaceResponse` object to be serialized. +/// \return A JSON object representing the `ListNamespaceResponse`. +nlohmann::json ToJson(const ListNamespaceResponse& response); + +/// \brief Deserializes a JSON object into a `ListNamespaceResponse` object. +/// +/// \param json The JSON object representing a `ListNamespaceResponse`. +/// \return A `ListNamespaceResponse` object or an error if the conversion fails. +Result ListNamespaceResponseFromJson(const nlohmann::json& json); + +/// \brief Serializes an `UpdateNamespacePropsRequest` object to JSON. +/// +/// \param request The `UpdateNamespacePropsRequest` object to be serialized. +/// \return A JSON object representing the `UpdateNamespacePropsRequest`. +nlohmann::json ToJson(const UpdateNamespacePropsRequest& request); + +/// \brief Deserializes a JSON object into an `UpdateNamespacePropsRequest` object. +/// +/// \param json The JSON object representing an `UpdateNamespacePropsRequest`. +/// \return An `UpdateNamespacePropsRequest` object or an error if the conversion fails. +Result UpdateNamespacePropsRequestFromJson( + const nlohmann::json& json); + +/// \brief Serializes an `UpdateNamespacePropsResponse` object to JSON. +/// +/// \param response The `UpdateNamespacePropsResponse` object to be serialized. +/// \return A JSON object representing the `UpdateNamespacePropsResponse`. +nlohmann::json ToJson(const UpdateNamespacePropsResponse& response); + +/// \brief Deserializes a JSON object into an `UpdateNamespacePropsResponse` object. +/// +/// \param json The JSON object representing an `UpdateNamespacePropsResponse`. +/// \return An `UpdateNamespacePropsResponse` object or an error if the conversion fails. +Result UpdateNamespacePropsResponseFromJson( + const nlohmann::json& json); + +/// \brief Serializes a `ListTableResponse` object to JSON. +/// +/// \param response The `ListTableResponse` object to be serialized. +/// \return A JSON object representing the `ListTableResponse`. +nlohmann::json ToJson(const ListTableResponse& response); + +/// \brief Deserializes a JSON object into a `ListTableResponse` object. +/// +/// \param json The JSON object representing a `ListTableResponse`. +/// \return A `ListTableResponse` object or an error if the conversion fails. +Result ListTableResponseFromJson(const nlohmann::json& json); + +/// \brief Serializes a `RenameTableRequest` object to JSON. +/// +/// \param request The `RenameTableRequest` object to be serialized. +/// \return A JSON object representing the `RenameTableRequest`. +nlohmann::json ToJson(const RenameTableRequest& request); + +/// \brief Deserializes a JSON object into a `RenameTableRequest` object. +/// +/// \param json The JSON object representing a `RenameTableRequest`. +/// \return A `RenameTableRequest` object or an error if the conversion fails. +Result RenameTableRequestFromJson(const nlohmann::json& json); + +/// \brief Serializes a `LoadTableResponse` object to JSON. +/// +/// \param response The `LoadTableResponse` object to be serialized. +/// \return A JSON object representing the `LoadTableResponse`. +nlohmann::json ToJson(const LoadTableResponse& response); + +/// \brief Deserializes a JSON object into a `LoadTableResponse` object. +/// +/// \param json The JSON object representing a `LoadTableResponse`. +/// \return A `LoadTableResponse` object or an error if the conversion fails. +Result LoadTableResponseFromJson(const nlohmann::json& json); + +/// \brief Serializes a `CreateTableRequest` object to JSON. +/// +/// \param request The `CreateTableRequest` object to be serialized. +/// \return A JSON object representing the `CreateTableRequest`. +nlohmann::json ToJson(const CreateTableRequest& request); + +/// \brief Deserializes a JSON object into a `CreateTableRequest` object. +/// +/// \param json The JSON object representing a `CreateTableRequest`. +/// \return A `CreateTableRequest` object or an error if the conversion fails. +Result CreateTableRequestFromJson(const nlohmann::json& json); + +/// \brief Serializes a `RegisterTableRequest` object to JSON. +/// +/// \param request The `RegisterTableRequest` object to be serialized. +/// \return A JSON object representing the `RegisterTableRequest`. +nlohmann::json ToJson(const RegisterTableRequest& request); + +/// \brief Deserializes a JSON object into a `RegisterTableRequest` object. +/// +/// \param json The JSON object representing a `RegisterTableRequest`. +/// \return A `RegisterTableRequest` object or an error if the conversion fails. +Result RegisterTableRequestFromJson(const nlohmann::json& json); + +} // namespace iceberg::rest diff --git a/src/iceberg/catalog/rest/types.h b/src/iceberg/catalog/rest/types.h new file mode 100644 index 00000000..1ae43a4a --- /dev/null +++ b/src/iceberg/catalog/rest/types.h @@ -0,0 +1,65 @@ +// types.h + +#pragma once + +#include +#include +#include + +#include "iceberg/partition_spec.h" +#include "iceberg/schema.h" +#include "iceberg/sort_order.h" +#include "iceberg/table_identifier.h" +#include "iceberg/table_metadata.h" + +namespace iceberg::rest { + +struct ListNamespaceResponse { + std::vector> namespaces; + // TODO(Feiyang Li): Add next_page_token +}; + +struct UpdateNamespacePropsRequest { + std::optional> removals; + std::optional> updates; +}; + +struct UpdateNamespacePropsResponse { + std::vector updated; + std::vector removed; + std::optional> missing; +}; + +struct ListTableResponse { + std::vector identifiers; + // TODO(Feiyang Li): Add next_page_token +}; + +struct RenameTableRequest { + TableIdentifier source; + TableIdentifier destination; +}; + +struct LoadTableResponse { + std::optional metadata_location; + TableMetadata metadata; + std::optional> config; +}; + +struct CreateTableRequest { + std::string name; + std::optional location; + std::shared_ptr schema; + std::shared_ptr partition_spec; // optional + std::shared_ptr write_order; // optional + std::optional stage_create; + std::optional> properties; +}; + +struct RegisterTableRequest { + std::string name; + std::string metadata_location; + std::optional overwrite; +}; + +} // namespace iceberg::rest diff --git a/src/iceberg/json_internal.cc b/src/iceberg/json_internal.cc index ad470a22..2be95c1e 100644 --- a/src/iceberg/json_internal.cc +++ b/src/iceberg/json_internal.cc @@ -42,6 +42,7 @@ #include "iceberg/transform.h" #include "iceberg/type.h" #include "iceberg/util/formatter.h" // IWYU pragma: keep +#include "iceberg/util/json_util_internal.h" #include "iceberg/util/macros.h" #include "iceberg/util/timepoint.h" @@ -166,185 +167,6 @@ constexpr std::string_view kFileSizeInBytes = "file-size-in-bytes"; constexpr std::string_view kFileFooterSizeInBytes = "file-footer-size-in-bytes"; constexpr std::string_view kBlobMetadata = "blob-metadata"; -template -void SetOptionalField(nlohmann::json& json, std::string_view key, - const std::optional& value) { - if (value.has_value()) { - json[key] = *value; - } -} - -std::string SafeDumpJson(const nlohmann::json& json) { - return json.dump(/*indent=*/-1, /*indent_char=*/' ', /*ensure_ascii=*/false, - nlohmann::detail::error_handler_t::ignore); -} - -template -Result GetJsonValueImpl(const nlohmann::json& json, std::string_view key) { - try { - return json.at(key).get(); - } catch (const std::exception& ex) { - return JsonParseError("Failed to parse '{}' from {}: {}", key, SafeDumpJson(json), - ex.what()); - } -} - -template -Result> GetJsonValueOptional(const nlohmann::json& json, - std::string_view key) { - if (!json.contains(key)) { - return std::nullopt; - } - return GetJsonValueImpl(json, key); -} - -template -Result GetJsonValue(const nlohmann::json& json, std::string_view key) { - if (!json.contains(key)) { - return JsonParseError("Missing '{}' in {}", key, SafeDumpJson(json)); - } - return GetJsonValueImpl(json, key); -} - -template -Result GetJsonValueOrDefault(const nlohmann::json& json, std::string_view key, - T default_value = T{}) { - if (!json.contains(key)) { - return default_value; - } - return GetJsonValueImpl(json, key); -} - -/// \brief Convert a list of items to a json array. -/// -/// Note that ToJson(const T&) is required for this function to work. -template -nlohmann::json::array_t ToJsonList(const std::vector& list) { - return std::accumulate(list.cbegin(), list.cend(), nlohmann::json::array(), - [](nlohmann::json::array_t arr, const T& item) { - arr.push_back(ToJson(item)); - return arr; - }); -} - -/// \brief Overload of the above function for a list of shared pointers. -template -nlohmann::json::array_t ToJsonList(const std::vector>& list) { - return std::accumulate(list.cbegin(), list.cend(), nlohmann::json::array(), - [](nlohmann::json::array_t arr, const std::shared_ptr& item) { - arr.push_back(ToJson(*item)); - return arr; - }); -} - -/// \brief Parse a list of items from a JSON object. -/// -/// \param[in] json The JSON object to parse. -/// \param[in] key The key to parse. -/// \param[in] from_json The function to parse an item from a JSON object. -/// \return The list of items. -template -Result> FromJsonList( - const nlohmann::json& json, std::string_view key, - const std::function(const nlohmann::json&)>& from_json) { - std::vector list{}; - if (json.contains(key)) { - ICEBERG_ASSIGN_OR_RAISE(auto list_json, GetJsonValue(json, key)); - if (!list_json.is_array()) { - return JsonParseError("Cannot parse '{}' from non-array: {}", key, - SafeDumpJson(list_json)); - } - for (const auto& entry_json : list_json) { - ICEBERG_ASSIGN_OR_RAISE(auto entry, from_json(entry_json)); - list.emplace_back(std::move(entry)); - } - } - return list; -} - -/// \brief Parse a list of items from a JSON object. -/// -/// \param[in] json The JSON object to parse. -/// \param[in] key The key to parse. -/// \param[in] from_json The function to parse an item from a JSON object. -/// \return The list of items. -template -Result>> FromJsonList( - const nlohmann::json& json, std::string_view key, - const std::function>(const nlohmann::json&)>& from_json) { - std::vector> list{}; - if (json.contains(key)) { - ICEBERG_ASSIGN_OR_RAISE(auto list_json, GetJsonValue(json, key)); - if (!list_json.is_array()) { - return JsonParseError("Cannot parse '{}' from non-array: {}", key, - SafeDumpJson(list_json)); - } - for (const auto& entry_json : list_json) { - ICEBERG_ASSIGN_OR_RAISE(auto entry, from_json(entry_json)); - list.emplace_back(std::move(entry)); - } - } - return list; -} - -/// \brief Convert a map of type to a json object. -/// -/// Note that ToJson(const T&) is required for this function to work. -template -nlohmann::json::object_t ToJsonMap(const std::unordered_map& map) { - return std::accumulate(map.cbegin(), map.cend(), nlohmann::json::object(), - [](nlohmann::json::object_t obj, const auto& item) { - obj[item.first] = ToJson(item.second); - return obj; - }); -} - -/// \brief Overload of the above function for a map of type >. -template -nlohmann::json::object_t ToJsonMap( - const std::unordered_map>& map) { - return std::accumulate(map.cbegin(), map.cend(), nlohmann::json::object(), - [](nlohmann::json::object_t obj, const auto& item) { - obj[item.first] = ToJson(*item.second); - return obj; - }); -} - -/// \brief Parse a map of type from a JSON object. -/// -/// \param[in] json The JSON object to parse. -/// \param[in] key The key to parse. -/// \param[in] from_json The function to parse an item from a JSON object. -/// \return The map of items. -template -Result> FromJsonMap( - const nlohmann::json& json, std::string_view key, - const std::function(const nlohmann::json&)>& from_json = - [](const nlohmann::json& json) -> Result { - static_assert(std::is_same_v, "T must be std::string"); - try { - return json.get(); - } catch (const std::exception& ex) { - return JsonParseError("Cannot parse {} to a string value: {}", SafeDumpJson(json), - ex.what()); - } - }) { - std::unordered_map map{}; - if (json.contains(key)) { - ICEBERG_ASSIGN_OR_RAISE(auto map_json, GetJsonValue(json, key)); - if (!map_json.is_object()) { - return JsonParseError("Cannot parse '{}' from non-object: {}", key, - SafeDumpJson(map_json)); - } - for (const auto& [key, value] : map_json.items()) { - ICEBERG_ASSIGN_OR_RAISE(auto entry, from_json(value)); - map[key] = std::move(entry); - } - } - return map; -} - } // namespace nlohmann::json ToJson(const SortField& sort_field) { diff --git a/src/iceberg/util/json_util_internal.h b/src/iceberg/util/json_util_internal.h new file mode 100644 index 00000000..c525ac90 --- /dev/null +++ b/src/iceberg/util/json_util_internal.h @@ -0,0 +1,214 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include + +#include + +#include "iceberg/result.h" +#include "iceberg/util/macros.h" + +/// \file iceberg/util/json_util_internal.h +/// \brief Internal utilities for JSON serialization and deserialization. + +namespace iceberg { + +template +void SetOptionalField(nlohmann::json& json, std::string_view key, + const std::optional& value) { + if (value.has_value()) { + json[key] = *value; + } +} + +inline std::string SafeDumpJson(const nlohmann::json& json) { + return json.dump(/*indent=*/-1, /*indent_char=*/' ', /*ensure_ascii=*/false, + nlohmann::detail::error_handler_t::ignore); +} + +template +Result GetJsonValueImpl(const nlohmann::json& json, std::string_view key) { + try { + return json.at(key).get(); + } catch (const std::exception& ex) { + return JsonParseError("Failed to parse '{}' from {}: {}", key, SafeDumpJson(json), + ex.what()); + } +} + +template +Result> GetJsonValueOptional(const nlohmann::json& json, + std::string_view key) { + if (!json.contains(key)) { + return std::nullopt; + } + ICEBERG_ASSIGN_OR_RAISE(auto value, GetJsonValueImpl(json, key)); + return std::optional(std::move(value)); +} + +template +Result GetJsonValue(const nlohmann::json& json, std::string_view key) { + if (!json.contains(key)) { + return JsonParseError("Missing '{}' in {}", key, SafeDumpJson(json)); + } + return GetJsonValueImpl(json, key); +} + +template +Result GetJsonValueOrDefault(const nlohmann::json& json, std::string_view key, + T default_value = T{}) { + if (!json.contains(key)) { + return default_value; + } + return GetJsonValueImpl(json, key); +} + +/// \brief Convert a list of items to a json array. +/// +/// Note that ToJson(const T&) is required for this function to work. +template +nlohmann::json::array_t ToJsonList(const std::vector& list) { + return std::accumulate(list.cbegin(), list.cend(), nlohmann::json::array(), + [](nlohmann::json::array_t arr, const T& item) { + arr.push_back(ToJson(item)); + return arr; + }); +} + +/// \brief Overload of the above function for a list of shared pointers. +template +nlohmann::json::array_t ToJsonList(const std::vector>& list) { + return std::accumulate(list.cbegin(), list.cend(), nlohmann::json::array(), + [](nlohmann::json::array_t arr, const std::shared_ptr& item) { + arr.push_back(ToJson(*item)); + return arr; + }); +} + +/// \brief Parse a list of items from a JSON object. +/// +/// \param[in] json The JSON object to parse. +/// \param[in] key The key to parse. +/// \param[in] from_json The function to parse an item from a JSON object. +/// \return The list of items. +template +Result> FromJsonList( + const nlohmann::json& json, std::string_view key, + const std::function(const nlohmann::json&)>& from_json) { + std::vector list{}; + if (json.contains(key)) { + ICEBERG_ASSIGN_OR_RAISE(auto list_json, GetJsonValue(json, key)); + if (!list_json.is_array()) { + return JsonParseError("Cannot parse '{}' from non-array: {}", key, + SafeDumpJson(list_json)); + } + for (const auto& entry_json : list_json) { + ICEBERG_ASSIGN_OR_RAISE(auto entry, from_json(entry_json)); + list.emplace_back(std::move(entry)); + } + } + return list; +} + +/// \brief Parse a list of items from a JSON object. +/// +/// \param[in] json The JSON object to parse. +/// \param[in] key The key to parse. +/// \param[in] from_json The function to parse an item from a JSON object. +/// \return The list of items. +template +Result>> FromJsonList( + const nlohmann::json& json, std::string_view key, + const std::function>(const nlohmann::json&)>& from_json) { + std::vector> list{}; + if (json.contains(key)) { + ICEBERG_ASSIGN_OR_RAISE(auto list_json, GetJsonValue(json, key)); + if (!list_json.is_array()) { + return JsonParseError("Cannot parse '{}' from non-array: {}", key, + SafeDumpJson(list_json)); + } + for (const auto& entry_json : list_json) { + ICEBERG_ASSIGN_OR_RAISE(auto entry, from_json(entry_json)); + list.emplace_back(std::move(entry)); + } + } + return list; +} + +/// \brief Convert a map of type to a json object. +/// +/// Note that ToJson(const T&) is required for this function to work. +template +nlohmann::json::object_t ToJsonMap(const std::unordered_map& map) { + return std::accumulate(map.cbegin(), map.cend(), nlohmann::json::object(), + [](nlohmann::json::object_t obj, const auto& item) { + obj[item.first] = ToJson(item.second); + return obj; + }); +} + +/// \brief Overload of the above function for a map of type >. +template +nlohmann::json::object_t ToJsonMap( + const std::unordered_map>& map) { + return std::accumulate(map.cbegin(), map.cend(), nlohmann::json::object(), + [](nlohmann::json::object_t obj, const auto& item) { + obj[item.first] = ToJson(*item.second); + return obj; + }); +} + +/// \brief Parse a map of type from a JSON object. +/// +/// \param[in] json The JSON object to parse. +/// \param[in] key The key to parse. +/// \param[in] from_json The function to parse an item from a JSON object. +/// \return The map of items. +template +Result> FromJsonMap( + const nlohmann::json& json, std::string_view key, + const std::function(const nlohmann::json&)>& from_json = + [](const nlohmann::json& json) -> Result { + static_assert(std::is_same_v, "T must be std::string"); + try { + return json.get(); + } catch (const std::exception& ex) { + return JsonParseError("Cannot parse {} to a string value: {}", SafeDumpJson(json), + ex.what()); + } + }) { + std::unordered_map map{}; + if (json.contains(key)) { + ICEBERG_ASSIGN_OR_RAISE(auto map_json, GetJsonValue(json, key)); + if (!map_json.is_object()) { + return JsonParseError("Cannot parse '{}' from non-object: {}", key, + SafeDumpJson(map_json)); + } + for (const auto& [key, value] : map_json.items()) { + ICEBERG_ASSIGN_OR_RAISE(auto entry, from_json(value)); + map[key] = std::move(entry); + } + } + return map; +} + +} // namespace iceberg diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index bba070c0..2e902959 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -143,7 +143,8 @@ if(ICEBERG_BUILD_BUNDLE) endif() if(ICEBERG_BUILD_REST) - add_iceberg_test(rest_catalog_test SOURCES rest_catalog_test.cc) + add_iceberg_test(rest_catalog_test SOURCES rest_catalog_test.cc + rest_json_internal_test.cc) target_link_libraries(rest_catalog_test PRIVATE iceberg_rest_static) target_include_directories(rest_catalog_test PRIVATE ${cpp-httplib_SOURCE_DIR}) endif() diff --git a/test/rest_json_internal_test.cc b/test/rest_json_internal_test.cc new file mode 100644 index 00000000..aab03c40 --- /dev/null +++ b/test/rest_json_internal_test.cc @@ -0,0 +1,361 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include + +#include +#include +#include + +#include "iceberg/catalog/rest/json_internal.h" +#include "iceberg/json_internal.h" +#include "iceberg/partition_spec.h" +#include "iceberg/schema.h" +#include "iceberg/sort_order.h" +#include "iceberg/table_identifier.h" +#include "iceberg/transform.h" +#include "iceberg/util/formatter.h" +#include "iceberg/util/macros.h" +#include "matchers.h" + +namespace iceberg::rest { + +TEST(RestJsonInternalTest, ListNamespaceResponse) { + ListNamespaceResponse response; + response.namespaces = {{"db1"}, {"db2", "schema1"}}; + + auto json = ToJson(response); + nlohmann::json expected_json = R"({ + "namespaces": [["db1"], ["db2", "schema1"]] + })"_json; + + EXPECT_EQ(json, expected_json); + + auto parsed_result = ListNamespaceResponseFromJson(expected_json); + ASSERT_TRUE(parsed_result.has_value()); + EXPECT_EQ(response.namespaces, parsed_result->namespaces); +} + +TEST(RestJsonInternalTest, UpdateNamespacePropsRequest) { + UpdateNamespacePropsRequest request; + request.removals = std::vector{"key1", "key2"}; + request.updates = std::unordered_map{{"key3", "value3"}}; + + auto json = ToJson(request); + nlohmann::json expected_json = R"({ + "removals": ["key1", "key2"], + "updates": {"key3": "value3"} + })"_json; + + EXPECT_EQ(json, expected_json); + + auto parsed_result = UpdateNamespacePropsRequestFromJson(expected_json); + ASSERT_TRUE(parsed_result.has_value()); + EXPECT_EQ(*request.removals, *parsed_result->removals); + EXPECT_EQ(*request.updates, *parsed_result->updates); +} + +TEST(RestJsonInternalTest, UpdateNamespacePropsResponse) { + UpdateNamespacePropsResponse response; + response.updated = {"key1", "key2"}; + response.removed = {"key3"}; + response.missing = std::vector{"key4"}; + + auto json = ToJson(response); + nlohmann::json expected_json = R"({ + "updated": ["key1", "key2"], + "removed": ["key3"], + "missing": ["key4"] + })"_json; + + EXPECT_EQ(json, expected_json); + + auto parsed_result = UpdateNamespacePropsResponseFromJson(expected_json); + ASSERT_TRUE(parsed_result.has_value()); + EXPECT_EQ(response.updated, parsed_result->updated); + EXPECT_EQ(response.removed, parsed_result->removed); + EXPECT_EQ(*response.missing, *parsed_result->missing); +} + +TEST(RestJsonInternalTest, ListTableResponse) { + ListTableResponse response; + response.identifiers = { + TableIdentifier{.ns = Namespace{{"db1"}}, .name = "table1"}, + TableIdentifier{.ns = Namespace{{"db2", "schema1"}}, .name = "table2"}}; + + auto json = ToJson(response); + nlohmann::json expected_json = R"({ + "identifiers": [ + {"namespace": ["db1"], "name": "table1"}, + {"namespace": ["db2", "schema1"], "name": "table2"} + ] + })"_json; + + EXPECT_EQ(json, expected_json); + + auto parsed_result = ListTableResponseFromJson(expected_json); + ASSERT_TRUE(parsed_result.has_value()); + EXPECT_EQ(response.identifiers.size(), parsed_result->identifiers.size()); + EXPECT_EQ(response.identifiers[0].name, parsed_result->identifiers[0].name); + EXPECT_EQ(response.identifiers[1].name, parsed_result->identifiers[1].name); +} + +TEST(RestJsonInternalTest, RenameTableRequest) { + RenameTableRequest request; + request.source = TableIdentifier{.ns = Namespace{{"db1"}}, .name = "old_table"}; + request.destination = TableIdentifier{.ns = Namespace{{"db2"}}, .name = "new_table"}; + + auto json = ToJson(request); + nlohmann::json expected_json = R"({ + "source": {"namespace": ["db1"], "name": "old_table"}, + "destination": {"namespace": ["db2"], "name": "new_table"} + })"_json; + + EXPECT_EQ(json, expected_json); + + auto parsed_result = RenameTableRequestFromJson(expected_json); + ASSERT_TRUE(parsed_result.has_value()); + EXPECT_EQ(request.source.name, parsed_result->source.name); + EXPECT_EQ(request.destination.name, parsed_result->destination.name); +} + +TEST(RestJsonInternalTest, CreateTableRequestBasic) { + auto schema = std::make_shared( + std::vector{SchemaField(1, "id", iceberg::int64(), false), + SchemaField(2, "data", iceberg::string(), true)}, + 0); + + CreateTableRequest request; + request.name = "test_table"; + request.location = "/tmp/test_location"; + request.schema = schema; + request.partition_spec = nullptr; + request.write_order = nullptr; + request.stage_create = false; + request.properties = std::unordered_map{{"key1", "value1"}}; + + auto json = ToJson(request); + + nlohmann::json expected_json = R"({ + "name": "test_table", + "location": "/tmp/test_location", + "schema": { + "type": "struct", + "schema-id": 0, + "fields": [ + {"id": 1, "name": "id", "required": true, "type": "long"}, + {"id": 2, "name": "data", "required": false, "type": "string"} + ] + }, + "stage-create": false, + "properties": {"key1": "value1"} + })"_json; + + EXPECT_EQ(json, expected_json); + + auto parsed_result = CreateTableRequestFromJson(expected_json); + ASSERT_TRUE(parsed_result.has_value()) << parsed_result.error().message; + + EXPECT_EQ(request.name, parsed_result->name); + EXPECT_EQ(*request.location, *parsed_result->location); + EXPECT_EQ(*request.schema, *parsed_result->schema); + EXPECT_EQ(request.partition_spec == nullptr, parsed_result->partition_spec == nullptr); + EXPECT_EQ(request.write_order == nullptr, parsed_result->write_order == nullptr); + EXPECT_EQ(*request.stage_create, *parsed_result->stage_create); + EXPECT_EQ(*request.properties, *parsed_result->properties); +} + +TEST(RestJsonInternalTest, CreateTableRequestWithPartitionSpec) { + auto schema = std::make_shared( + std::vector{SchemaField(1, "id", iceberg::int64(), false), + SchemaField(2, "ts", iceberg::timestamp(), false), + SchemaField(3, "data", iceberg::string(), true)}, + 0); + + auto identity_transform = Transform::Identity(); + auto partition_spec = + std::make_shared(schema, 1, + std::vector{PartitionField( + 2, 1000, "ts_partition", identity_transform)}); + + CreateTableRequest request; + request.name = "partitioned_table"; + request.schema = schema; + request.partition_spec = partition_spec; + request.write_order = nullptr; + + auto json = ToJson(request); + + nlohmann::json expected_json = R"({ + "name": "partitioned_table", + "schema": { + "type": "struct", + "schema-id": 0, + "fields": [ + {"id": 1, "name": "id", "required": true, "type": "long"}, + {"id": 2, "name": "ts", "required": true, "type": "timestamp"}, + {"id": 3, "name": "data", "required": false, "type": "string"} + ] + }, + "partition-spec": { + "spec-id": 1, + "fields": [ + {"source-id": 2, "field-id": 1000, "transform": "identity", "name": "ts_partition"} + ] + } + })"_json; + + EXPECT_EQ(json, expected_json); + + auto parsed_result = CreateTableRequestFromJson(expected_json); + ASSERT_TRUE(parsed_result.has_value()) << parsed_result.error().message; + + EXPECT_EQ(request.name, parsed_result->name); + EXPECT_EQ(*request.schema, *parsed_result->schema); + ASSERT_NE(parsed_result->partition_spec, nullptr); + EXPECT_EQ(*request.partition_spec, *parsed_result->partition_spec); + EXPECT_EQ(request.write_order == nullptr, parsed_result->write_order == nullptr); +} + +TEST(RestJsonInternalTest, CreateTableRequestWithSortOrder) { + auto schema = std::make_shared( + std::vector{SchemaField(1, "id", iceberg::int64(), false), + SchemaField(2, "ts", iceberg::timestamp(), false)}, + 0); + + auto identity_transform = Transform::Identity(); + SortField sort_field(1, identity_transform, SortDirection::kAscending, + NullOrder::kFirst); + auto write_order = std::make_shared(1, std::vector{sort_field}); + + CreateTableRequest request; + request.name = "sorted_table"; + request.schema = schema; + request.partition_spec = nullptr; + request.write_order = write_order; + + auto json = ToJson(request); + + nlohmann::json expected_json = R"({ + "name": "sorted_table", + "schema": { + "type": "struct", + "schema-id": 0, + "fields": [ + {"id": 1, "name": "id", "required": true, "type": "long"}, + {"id": 2, "name": "ts", "required": true, "type": "timestamp"} + ] + }, + "write-order": { + "order-id": 1, + "fields": [ + {"transform": "identity", "source-id": 1, "direction": "asc", "null-order": "nulls-first"} + ] + } + })"_json; + + EXPECT_EQ(json, expected_json); + + auto parsed_result = CreateTableRequestFromJson(expected_json); + ASSERT_TRUE(parsed_result.has_value()) << parsed_result.error().message; + + EXPECT_EQ(request.name, parsed_result->name); + EXPECT_EQ(*request.schema, *parsed_result->schema); + EXPECT_EQ(request.partition_spec == nullptr, parsed_result->partition_spec == nullptr); + ASSERT_NE(parsed_result->write_order, nullptr); + EXPECT_EQ(*request.write_order, *parsed_result->write_order); +} + +TEST(RestJsonInternalTest, CreateTableRequestComplete) { + auto schema = std::make_shared( + std::vector{SchemaField(1, "id", iceberg::int64(), false), + SchemaField(2, "ts", iceberg::timestamp(), false), + SchemaField(3, "data", iceberg::string(), true)}, + 0); + + auto identity_transform = Transform::Identity(); + auto partition_spec = + std::make_shared(schema, 1, + std::vector{PartitionField( + 2, 1000, "ts_partition", identity_transform)}); + + SortField sort_field(1, identity_transform, SortDirection::kAscending, + NullOrder::kFirst); + auto write_order = std::make_shared(1, std::vector{sort_field}); + + CreateTableRequest request; + request.name = "complete_table"; + request.location = "/tmp/complete"; + request.schema = schema; + request.partition_spec = partition_spec; + request.write_order = write_order; + request.stage_create = true; + request.properties = std::unordered_map{{"key1", "value1"}, + {"key2", "value2"}}; + + auto json = ToJson(request); + + auto parsed_result = CreateTableRequestFromJson(json); + ASSERT_TRUE(parsed_result.has_value()) << parsed_result.error().message; + + EXPECT_EQ(request.name, parsed_result->name); + EXPECT_EQ(*request.location, *parsed_result->location); + EXPECT_EQ(*request.schema, *parsed_result->schema); + ASSERT_NE(parsed_result->partition_spec, nullptr); + EXPECT_EQ(*request.partition_spec, *parsed_result->partition_spec); + ASSERT_NE(parsed_result->write_order, nullptr); + EXPECT_EQ(*request.write_order, *parsed_result->write_order); + EXPECT_EQ(*request.stage_create, *parsed_result->stage_create); + EXPECT_EQ(*request.properties, *parsed_result->properties); +} + +TEST(RestJsonInternalTest, CreateTableRequestMissingRequiredFields) { + nlohmann::json invalid_json = R"({ + "location": "/tmp/test" + })"_json; + + auto result = CreateTableRequestFromJson(invalid_json); + EXPECT_FALSE(result.has_value()); + EXPECT_THAT(result, IsError(ErrorKind::kJsonParseError)); + EXPECT_THAT(result, HasErrorMessage("Missing 'name'")); +} + +TEST(RestJsonInternalTest, RegisterTableRequest) { + RegisterTableRequest request; + request.name = "registered_table"; + request.metadata_location = "/tmp/metadata.json"; + request.overwrite = true; + + auto json = ToJson(request); + nlohmann::json expected_json = R"({ + "name": "registered_table", + "metadata-location": "/tmp/metadata.json", + "overwrite": true + })"_json; + + EXPECT_EQ(json, expected_json); + + auto parsed_result = RegisterTableRequestFromJson(expected_json); + ASSERT_TRUE(parsed_result.has_value()); + EXPECT_EQ(request.name, parsed_result->name); + EXPECT_EQ(request.metadata_location, parsed_result->metadata_location); + EXPECT_EQ(*request.overwrite, *parsed_result->overwrite); +} + +} // namespace iceberg::rest