From e811f0b69cecdfc4719a602d339334db3fc7a778 Mon Sep 17 00:00:00 2001 From: Andrey Zvonov <32552679+zvonand@users.noreply.github.com> Date: Mon, 18 May 2026 19:42:32 +0200 Subject: [PATCH 1/2] Merge pull request #1781 from Altinity/1338_fix_3_decoding_antaly_26_3 Antalya 26.3 Backport - Added test cases for s3 encoding fix --- .../tests/gtest_iceberg_path_resolver.cpp | 41 +++++++++++++++++ .../integration/test_database_iceberg/test.py | 44 +++++++++++++++++++ 2 files changed, 85 insertions(+) create mode 100644 src/Storages/ObjectStorage/DataLakes/Iceberg/tests/gtest_iceberg_path_resolver.cpp diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/tests/gtest_iceberg_path_resolver.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/tests/gtest_iceberg_path_resolver.cpp new file mode 100644 index 000000000000..9c4adc113eab --- /dev/null +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/tests/gtest_iceberg_path_resolver.cpp @@ -0,0 +1,41 @@ +#include + +#include + +using namespace DB::Iceberg; + +TEST(GetProperFilePathFromMetadataInfo, S3SchemePreservesPercentEncodedSlash) +{ + auto result = getProperFilePathFromMetadataInfo( + "s3://bucket/warehouse/data/partition=us%2Fwest/file.parquet", + "warehouse", + "s3://bucket/warehouse"); + ASSERT_EQ(result, "warehouse/data/partition=us%2Fwest/file.parquet"); +} + +TEST(GetProperFilePathFromMetadataInfo, SimpleKeyWithoutEncoding) +{ + auto result = getProperFilePathFromMetadataInfo( + "s3://bucket/warehouse/data/file.parquet", + "warehouse", + "s3://bucket/warehouse"); + ASSERT_EQ(result, "warehouse/data/file.parquet"); +} + +TEST(GetProperFilePathFromMetadataInfo, MultiplePercentEncodedSegments) +{ + auto result = getProperFilePathFromMetadataInfo( + "s3://bucket/warehouse/data/region=us%2Fwest/city=san%20francisco/file.parquet", + "warehouse", + "s3://bucket/warehouse"); + ASSERT_EQ(result, "warehouse/data/region=us%2Fwest/city=san%20francisco/file.parquet"); +} + +TEST(GetProperFilePathFromMetadataInfo, HttpSchemePreservesPercentEncodedSlash) +{ + auto result = getProperFilePathFromMetadataInfo( + "http://minio:9000/bucket/warehouse/data/partition=us%2Fwest/file.parquet", + "warehouse", + "http://minio:9000/bucket/warehouse"); + ASSERT_EQ(result, "warehouse/data/partition=us%2Fwest/file.parquet"); +} diff --git a/tests/integration/test_database_iceberg/test.py b/tests/integration/test_database_iceberg/test.py index 31ec8882a357..d57b8706938f 100644 --- a/tests/integration/test_database_iceberg/test.py +++ b/tests/integration/test_database_iceberg/test.py @@ -801,6 +801,50 @@ def test_table_with_slash(started_cluster): assert node.query(f"SELECT * FROM {CATALOG_NAME}.`{root_namespace}.{table_encoded_name}`") == "\\N\tAAPL\t193.24\t193.31\t('bot')\n" +def test_partition_value_with_slash(started_cluster): + """Partition value containing '/' produces object keys with %2F; reading must preserve encoding.""" + node = started_cluster.instances["node1"] + + test_ref = f"test_partition_slash_{uuid.uuid4()}" + table_name = f"{test_ref}_table" + root_namespace = f"{test_ref}_namespace" + + partition_spec = PartitionSpec( + PartitionField( + source_id=2, field_id=1000, transform=IdentityTransform(), name="symbol" + ) + ) + schema = DEFAULT_SCHEMA + + catalog = load_catalog_impl(started_cluster) + catalog.create_namespace(root_namespace) + + table = create_table( + catalog, + root_namespace, + table_name, + schema, + partition_spec=partition_spec, + sort_order=DEFAULT_SORT_ORDER, + ) + + data = [ + { + "datetime": datetime.now(), + "symbol": "us/west", + "bid": 100.0, + "ask": 101.0, + "details": {"created_by": "test"}, + } + ] + df = pa.Table.from_pylist(data) + table.append(df) + + create_clickhouse_iceberg_database(started_cluster, node, CATALOG_NAME) + assert 1 == int(node.query(f"SELECT count() FROM {CATALOG_NAME}.`{root_namespace}.{table_name}`")) + assert "us/west" in node.query(f"SELECT symbol FROM {CATALOG_NAME}.`{root_namespace}.{table_name}`") + + def test_cluster_select(started_cluster): node1 = started_cluster.instances["node1"] node2 = started_cluster.instances["node2"] From cab296ae0ba275be7d82f1b0d746417108485a7a Mon Sep 17 00:00:00 2001 From: blau-ai Date: Tue, 4 Aug 2026 12:24:04 +0000 Subject: [PATCH 2/2] Fix build: port iceberg path-resolver gtest to IcebergPathResolver API The gtest added by the backport called getProperFilePathFromMetadataInfo, a free function that exists in antalya-26.3 but was refactored into IcebergPathResolver::resolve() in 26.6, so the test failed to compile (use of undeclared identifier) and broke Build (amd_debug)/(arm_debug). Rewrite the four test cases against the current IcebergPathResolver API. The resolver applies the same prefix-stripping string logic, so the percent-encoding-preservation expectations are unchanged. Co-Authored-By: Claude Opus 4.8 --- .../tests/gtest_iceberg_path_resolver.cpp | 23 +++++++++++++++---- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/tests/gtest_iceberg_path_resolver.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/tests/gtest_iceberg_path_resolver.cpp index 9c4adc113eab..3e1eb954e654 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/tests/gtest_iceberg_path_resolver.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/tests/gtest_iceberg_path_resolver.cpp @@ -1,12 +1,25 @@ #include -#include +#include using namespace DB::Iceberg; +/// In antalya-26.3 the path resolution lived in the free function +/// getProperFilePathFromMetadataInfo(data_path, common_path, table_location). +/// In 26.6 the same logic is provided by IcebergPathResolver, where +/// common_path became table_root and the data path is passed as an +/// IcebergPathFromMetadata. The mapping is: +/// getProperFilePathFromMetadataInfo(data_path, common_path, table_location) +/// == IcebergPathResolver(table_location, common_path).resolve(deserialize(data_path)) +static String resolvePath(std::string_view data_path, std::string_view common_path, std::string_view table_location) +{ + IcebergPathResolver resolver(String(table_location), String(common_path)); + return resolver.resolve(IcebergPathFromMetadata::deserialize(String(data_path))); +} + TEST(GetProperFilePathFromMetadataInfo, S3SchemePreservesPercentEncodedSlash) { - auto result = getProperFilePathFromMetadataInfo( + auto result = resolvePath( "s3://bucket/warehouse/data/partition=us%2Fwest/file.parquet", "warehouse", "s3://bucket/warehouse"); @@ -15,7 +28,7 @@ TEST(GetProperFilePathFromMetadataInfo, S3SchemePreservesPercentEncodedSlash) TEST(GetProperFilePathFromMetadataInfo, SimpleKeyWithoutEncoding) { - auto result = getProperFilePathFromMetadataInfo( + auto result = resolvePath( "s3://bucket/warehouse/data/file.parquet", "warehouse", "s3://bucket/warehouse"); @@ -24,7 +37,7 @@ TEST(GetProperFilePathFromMetadataInfo, SimpleKeyWithoutEncoding) TEST(GetProperFilePathFromMetadataInfo, MultiplePercentEncodedSegments) { - auto result = getProperFilePathFromMetadataInfo( + auto result = resolvePath( "s3://bucket/warehouse/data/region=us%2Fwest/city=san%20francisco/file.parquet", "warehouse", "s3://bucket/warehouse"); @@ -33,7 +46,7 @@ TEST(GetProperFilePathFromMetadataInfo, MultiplePercentEncodedSegments) TEST(GetProperFilePathFromMetadataInfo, HttpSchemePreservesPercentEncodedSlash) { - auto result = getProperFilePathFromMetadataInfo( + auto result = resolvePath( "http://minio:9000/bucket/warehouse/data/partition=us%2Fwest/file.parquet", "warehouse", "http://minio:9000/bucket/warehouse");