From 1578ae305cf18e25a1112a51403fc38e919c6bd7 Mon Sep 17 00:00:00 2001 From: "zhangchaoming.zcm" Date: Wed, 12 Aug 2026 15:47:33 +0800 Subject: [PATCH 01/16] feat(parquet): support vector type storage --- docs/source/user_guide/data_types.rst | 14 + include/paimon/defs.h | 2 + include/paimon/format/column_stats.h | 12 +- .../common/predicate/literal_converter.cpp | 1 + src/paimon/common/types/data_type.cpp | 3 + .../common/types/data_type_json_parser.cpp | 59 +++ .../common/types/data_type_json_parser.h | 2 + .../types/data_type_json_parser_test.cpp | 50 +++ src/paimon/common/types/data_type_test.cpp | 14 + src/paimon/common/types/vector_type.h | 75 ++++ src/paimon/common/utils/arrow/arrow_utils.cpp | 31 ++ .../common/utils/arrow/arrow_utils_test.cpp | 25 ++ src/paimon/common/utils/field_type_utils.h | 4 + .../common/utils/field_type_utils_test.cpp | 5 + .../core/schema/arrow_schema_validator.cpp | 22 ++ .../schema/arrow_schema_validator_test.cpp | 23 +- src/paimon/core/schema/schema_validation.cpp | 43 +++ src/paimon/core/schema/schema_validation.h | 2 + .../core/schema/schema_validation_test.cpp | 26 ++ src/paimon/core/schema/table_schema.cpp | 8 + src/paimon/format/parquet/CMakeLists.txt | 3 + .../parquet/parquet_field_id_converter.cpp | 6 + .../parquet_field_id_converter_test.cpp | 11 +- .../parquet/parquet_file_batch_reader.cpp | 39 +- .../parquet/parquet_file_batch_reader.h | 2 +- .../format/parquet/parquet_format_writer.cpp | 26 +- .../format/parquet/parquet_format_writer.h | 4 +- .../parquet/parquet_stats_extractor.cpp | 4 +- .../parquet/parquet_stats_extractor_test.cpp | 16 +- .../parquet/parquet_vector_converter.cpp | 343 ++++++++++++++++++ .../format/parquet/parquet_vector_converter.h | 50 +++ .../parquet/parquet_vector_converter_test.cpp | 150 ++++++++ .../format/parquet/parquet_vector_io_test.cpp | 155 ++++++++ test/inte/write_and_read_inte_test.cpp | 60 +++ 34 files changed, 1261 insertions(+), 29 deletions(-) create mode 100644 src/paimon/common/types/vector_type.h create mode 100644 src/paimon/format/parquet/parquet_vector_converter.cpp create mode 100644 src/paimon/format/parquet/parquet_vector_converter.h create mode 100644 src/paimon/format/parquet/parquet_vector_converter_test.cpp create mode 100644 src/paimon/format/parquet/parquet_vector_io_test.cpp diff --git a/docs/source/user_guide/data_types.rst b/docs/source/user_guide/data_types.rst index 3d529332b..fd14ce8f0 100644 --- a/docs/source/user_guide/data_types.rst +++ b/docs/source/user_guide/data_types.rst @@ -186,6 +186,20 @@ and `Arrow DataTypes `` where t is the data type of the contained elements. + * - ``VECTOR`` + - FixedSizeList + - Data type of a dense vector containing exactly ``n`` elements of type ``t``. + + ``n`` must be positive. ``t`` can be ``BOOLEAN``, ``TINYINT``, + ``SMALLINT``, ``INT``, ``BIGINT``, ``FLOAT``, or ``DOUBLE``. A VECTOR + value may be NULL, but its elements cannot be NULL. + + Paimon C++ currently supports VECTOR columns in Parquet data files. They + use the standard Parquet LIST representation on disk and are restored + as Arrow ``FixedSizeList`` values on read. VECTOR columns cannot be + primary, partition, or bucket keys. Dedicated vector storage and Data + Evolution support are not included yet. + * - ``MAP`` - Map - Data type of an associative array that maps keys (including NULL) to values (including NULL). A map cannot contain duplicate keys; each key can map to at most one value. diff --git a/include/paimon/defs.h b/include/paimon/defs.h index 46b20c316..e65b2f085 100644 --- a/include/paimon/defs.h +++ b/include/paimon/defs.h @@ -50,6 +50,8 @@ enum class FieldType { STRUCT = 15, BLOB = 16, VARIANT = 17, + /// Fixed-length dense vector represented by Arrow FixedSizeList. + VECTOR = 18, UNKNOWN = 128, }; diff --git a/include/paimon/format/column_stats.h b/include/paimon/format/column_stats.h index f16cb2547..a4e3de484 100644 --- a/include/paimon/format/column_stats.h +++ b/include/paimon/format/column_stats.h @@ -33,8 +33,8 @@ namespace paimon { /// ColumnStats is an abstract base class that represents statistical information for data columns /// in Paimon tables. It provides min/max values and null count statistics /// -/// Only primitive data types support min/max statistics. Nested types (arrays, maps, structs) only -/// track null counts through `NestedColumnStats`. +/// Only primitive data types support min/max statistics. Nested types (arrays, vectors, maps, +/// structs) only track null counts through `NestedColumnStats`. /// /// @note This is an abstract base class. Use the static factory methods `CreateXXXColumnStats()` to /// create concrete instances for specific data types. @@ -52,7 +52,7 @@ class PAIMON_EXPORT ColumnStats { /// @name CreateXXXColumnStats() /// %Factory methods `CreateXXXColumnStats()` to create column statistics. /// - min/max/null_count for primitive data types - /// - null_count for nested data types (arrays, maps, structs) + /// - null_count for nested data types (arrays, vectors, maps, structs) /// /// @{ static std::unique_ptr CreateBooleanColumnStats(std::optional min, @@ -88,8 +88,8 @@ class PAIMON_EXPORT ColumnStats { static std::unique_ptr CreateDateColumnStats(std::optional min, std::optional max, std::optional null_count); - /// Creates column statistics for nested data types (arrays, maps, structs), which only track - /// null counts. + /// Creates column statistics for nested data types (arrays, vectors, maps, structs), which only + /// track null counts. static std::unique_ptr CreateNestedColumnStats(const FieldType& nested_type, std::optional null_count); /// @} @@ -180,7 +180,7 @@ class PAIMON_EXPORT NestedColumnStats : public ColumnStats { NestedColumnStats(const FieldType& nested_type, std::optional null_count) : nested_type_(nested_type), null_count_(null_count) { assert(nested_type == FieldType::ARRAY || nested_type == FieldType::MAP || - nested_type == FieldType::STRUCT); + nested_type == FieldType::STRUCT || nested_type == FieldType::VECTOR); } std::optional NullCount() const override { diff --git a/src/paimon/common/predicate/literal_converter.cpp b/src/paimon/common/predicate/literal_converter.cpp index dafedc165..af8c17f48 100644 --- a/src/paimon/common/predicate/literal_converter.cpp +++ b/src/paimon/common/predicate/literal_converter.cpp @@ -166,6 +166,7 @@ Result LiteralConverter::ConvertLiteralsFromRow( case FieldType::DATE: return Literal(FieldType::DATE, row.GetInt(field_idx)); case FieldType::ARRAY: + case FieldType::VECTOR: case FieldType::MAP: case FieldType::STRUCT: default: diff --git a/src/paimon/common/types/data_type.cpp b/src/paimon/common/types/data_type.cpp index 623d4ca20..d8deb17c8 100644 --- a/src/paimon/common/types/data_type.cpp +++ b/src/paimon/common/types/data_type.cpp @@ -30,6 +30,7 @@ #include "paimon/common/types/array_type.h" #include "paimon/common/types/map_type.h" #include "paimon/common/types/row_type.h" +#include "paimon/common/types/vector_type.h" #include "paimon/common/utils/date_time_utils.h" #include "paimon/common/utils/decimal_utils.h" #include "paimon/common/utils/rapidjson_util.h" @@ -52,6 +53,8 @@ std::unique_ptr DataType::Create( return std::make_unique(type, nullable, metadata); case arrow::Type::type::LIST: return std::make_unique(type, nullable, metadata); + case arrow::Type::type::FIXED_SIZE_LIST: + return std::make_unique(type, nullable, metadata); case arrow::Type::type::STRUCT: if (VariantTypeUtils::IsVariantMetadata(metadata)) { // A variant field is physically a struct but is a scalar diff --git a/src/paimon/common/types/data_type_json_parser.cpp b/src/paimon/common/types/data_type_json_parser.cpp index 33308ab69..c002b23c4 100644 --- a/src/paimon/common/types/data_type_json_parser.cpp +++ b/src/paimon/common/types/data_type_json_parser.cpp @@ -21,11 +21,13 @@ #include #include +#include #include #include #include #include #include +#include #include #include @@ -33,6 +35,7 @@ #include "paimon/common/data/blob_utils.h" #include "paimon/common/data/variant/variant_type_utils.h" #include "paimon/common/types/data_field.h" +#include "paimon/common/types/vector_type.h" #include "paimon/common/utils/date_time_utils.h" #include "paimon/common/utils/rapidjson_util.h" #include "paimon/common/utils/string_utils.h" @@ -148,6 +151,7 @@ enum class Keyword : int32_t { ROW, BLOB, VARIANT, + VECTOR, // NULL is keyword in c++ NULL_, RAW, @@ -197,6 +201,7 @@ const std::map& Keywords() { {"ROW", Keyword::ROW}, {"BLOB", Keyword::BLOB}, {"VARIANT", Keyword::VARIANT}, + {"VECTOR", Keyword::VECTOR}, {"NULL", Keyword::NULL_}, {"RAW", Keyword::RAW}, {"LEGACY", Keyword::LEGACY}, @@ -249,6 +254,7 @@ class TokenParser { Result> ParseDoubleType(); Result> ParseTimestampType(); Result> ParseTimestampLtzType(); + Result> ParseVectorType(); Result ParseOptionalPrecision(int32_t default_precision); private: @@ -526,6 +532,8 @@ Result> TokenParser::ParseTypeByKeyword( return ParseTimestampType(); case Keyword::TIMESTAMP_LTZ: return ParseTimestampLtzType(); + case Keyword::VECTOR: + return ParseVectorType(); default: return Status::Invalid(fmt::format("Unsupported type: {}", GetToken().value)); } @@ -607,6 +615,34 @@ Result> TokenParser::ParseTimestampLtzType() { return ts_type; } +Result> TokenParser::ParseVectorType() { + PAIMON_RETURN_NOT_OK(NextToken(TokenType::BEGIN_SUBTYPE)); + bool element_nullable = true; + AtomicTypeAttributes element_attributes; + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr element_type, + ParseTypeWithNullability(&element_nullable, &element_attributes)); + if (element_attributes.is_blob || element_attributes.is_variant || + !VectorType::IsValidElementType(element_type)) { + return Status::Invalid( + fmt::format("Invalid element type for vector: {}", element_type->ToString())); + } + PAIMON_RETURN_NOT_OK(NextToken(TokenType::LIST_SEPARATOR)); + PAIMON_RETURN_NOT_OK(NextToken(TokenType::LITERAL_INT)); + const std::string& length_token = GetToken().value; + int64_t length = 0; + const auto [end, error] = + std::from_chars(length_token.data(), length_token.data() + length_token.size(), length); + if (error != std::errc() || end != length_token.data() + length_token.size() || length < 1 || + length > std::numeric_limits::max()) { + return Status::Invalid( + fmt::format("Vector length must be between 1 and {} (both inclusive), but was {}", + std::numeric_limits::max(), length_token)); + } + PAIMON_RETURN_NOT_OK(NextToken(TokenType::END_SUBTYPE)); + return arrow::fixed_size_list(arrow::field("item", element_type, element_nullable), + static_cast(length)); +} + Result TokenParser::ParseOptionalPrecision(int32_t default_precision) { auto precision = default_precision; if (HasNextToken({TokenType::BEGIN_PARAMETER})) { @@ -659,6 +695,8 @@ Result> DataTypeJsonParser::ParseComplexTypeField( if (StringUtils::StartsWith(type_str, "ARRAY")) { return ParseArrayType(name, type_json_value, nullable); + } else if (StringUtils::StartsWith(type_str, "VECTOR")) { + return ParseVectorType(name, type_json_value, nullable); } else if (StringUtils::StartsWith(type_str, "MAP")) { return ParseMapType(name, type_json_value, nullable); } else if (StringUtils::StartsWith(type_str, "ROW")) { @@ -681,6 +719,27 @@ Result> DataTypeJsonParser::ParseArrayType( return arrow::field(name, arrow::list(element_field), nullable); } +Result> DataTypeJsonParser::ParseVectorType( + const std::string& name, const rapidjson::Value& type_json_value, bool nullable) { + if (!type_json_value.HasMember("element") || !type_json_value.HasMember("length")) { + return Status::Invalid("vector data type must have element and length"); + } + if (!type_json_value["length"].IsInt()) { + return Status::Invalid("vector length must be an integer"); + } + int32_t length = type_json_value["length"].GetInt(); + if (length < 1) { + return Status::Invalid("Vector length must be between 1 and 2147483647 (both inclusive)"); + } + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr element_field, + ParseType("item", type_json_value["element"])); + if (!VectorType::IsValidElementType(element_field->type())) { + return Status::Invalid( + fmt::format("Invalid element type for vector: {}", element_field->type()->ToString())); + } + return arrow::field(name, arrow::fixed_size_list(element_field, length), nullable); +} + Result> DataTypeJsonParser::ParseMapType( const std::string& name, const rapidjson::Value& type_json_value, bool nullable) { if (!type_json_value.HasMember("key") || !type_json_value.HasMember("value")) { diff --git a/src/paimon/common/types/data_type_json_parser.h b/src/paimon/common/types/data_type_json_parser.h index 92cb5d55f..2134236a7 100644 --- a/src/paimon/common/types/data_type_json_parser.h +++ b/src/paimon/common/types/data_type_json_parser.h @@ -50,6 +50,8 @@ class DataTypeJsonParser { static Result> ParseArrayType( const std::string& name, const rapidjson::Value& type_json_value, bool nullable); + static Result> ParseVectorType( + const std::string& name, const rapidjson::Value& type_json_value, bool nullable); static Result> ParseMapType( const std::string& name, const rapidjson::Value& type_json_value, bool nullable); static Result> ParseRowType( diff --git a/src/paimon/common/types/data_type_json_parser_test.cpp b/src/paimon/common/types/data_type_json_parser_test.cpp index bfa69c32a..9e071e909 100644 --- a/src/paimon/common/types/data_type_json_parser_test.cpp +++ b/src/paimon/common/types/data_type_json_parser_test.cpp @@ -47,6 +47,56 @@ TEST(DataTypeJsonParserTest, ParseTypeArrayTypeSuccess) { ASSERT_NE(field, nullptr); } +TEST(DataTypeJsonParserTest, ParseVectorTypeSuccess) { + const char* json = R"({ + "type": "VECTOR NOT NULL", + "element": "FLOAT", + "length": 3 + })"; + rapidjson::Document doc; + doc.Parse(json); + + ASSERT_OK_AND_ASSIGN(std::shared_ptr field, + DataTypeJsonParser::ParseType("embedding", doc)); + ASSERT_FALSE(field->nullable()); + ASSERT_EQ(field->type()->id(), arrow::Type::FIXED_SIZE_LIST); + auto vector_type = std::static_pointer_cast(field->type()); + ASSERT_EQ(vector_type->list_size(), 3); + ASSERT_TRUE(vector_type->value_type()->Equals(arrow::float32())); + + rapidjson::Document sql_doc; + rapidjson::Value sql_value("VECTOR", sql_doc.GetAllocator()); + ASSERT_OK_AND_ASSIGN(field, DataTypeJsonParser::ParseType("embedding", sql_value)); + vector_type = std::static_pointer_cast(field->type()); + ASSERT_TRUE(field->nullable()); + ASSERT_EQ(vector_type->list_size(), 5); + ASSERT_FALSE(vector_type->value_field()->nullable()); + ASSERT_TRUE(vector_type->value_type()->Equals(arrow::int64())); +} + +TEST(DataTypeJsonParserTest, ParseVectorTypeFailure) { + for (const char* json : { + R"({"type":"VECTOR","element":"FLOAT","length":0})", + R"({"type":"VECTOR","element":"STRING","length":3})", + R"({"type":"VECTOR","element":"FLOAT"})", + R"({"type":"VECTOR","element":"FLOAT","length":"3"})", + }) { + rapidjson::Document doc; + doc.Parse(json); + ASSERT_NOK(DataTypeJsonParser::ParseType("embedding", doc)); + } + + rapidjson::Document sql_doc; + rapidjson::Value sql_value("VECTOR", sql_doc.GetAllocator()); + ASSERT_NOK_WITH_MSG(DataTypeJsonParser::ParseType("embedding", sql_value), + "Invalid element type for vector"); + sql_value.SetString("VECTOR", sql_doc.GetAllocator()); + ASSERT_OK(DataTypeJsonParser::ParseType("embedding", sql_value)); + sql_value.SetString("VECTOR", sql_doc.GetAllocator()); + ASSERT_NOK_WITH_MSG(DataTypeJsonParser::ParseType("embedding", sql_value), + "Vector length must be between 1 and 2147483647"); +} + TEST(DataTypeJsonParserTest, ParseTypeMapTypeSuccess) { const std::string name = "map_field"; const char* json = R"({ diff --git a/src/paimon/common/types/data_type_test.cpp b/src/paimon/common/types/data_type_test.cpp index 9568c3ff0..d6eacdc1f 100644 --- a/src/paimon/common/types/data_type_test.cpp +++ b/src/paimon/common/types/data_type_test.cpp @@ -148,4 +148,18 @@ TEST(DataTypeTest, NestedTypeSerializationUsesChildMetadata) { R"({"type":"ARRAY","element":"INT"})"); } +TEST(DataTypeTest, VectorTypeSerialization) { + auto vector_field = arrow::field( + "embedding", arrow::fixed_size_list(arrow::field("item", arrow::float32()), 3), false); + auto data_type = + DataType::Create(vector_field->type(), vector_field->nullable(), vector_field->metadata()); + rapidjson::Document doc; + auto value = data_type->ToJson(&doc.GetAllocator()); + rapidjson::StringBuffer buffer; + rapidjson::Writer writer(buffer); + value.Accept(writer); + ASSERT_EQ(std::string(buffer.GetString()), + R"({"type":"VECTOR NOT NULL","element":"FLOAT","length":3})"); +} + } // namespace paimon::test diff --git a/src/paimon/common/types/vector_type.h b/src/paimon/common/types/vector_type.h new file mode 100644 index 000000000..3dd68ec74 --- /dev/null +++ b/src/paimon/common/types/vector_type.h @@ -0,0 +1,75 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include +#include + +#include "arrow/api.h" +#include "paimon/common/types/data_type.h" +#include "paimon/common/utils/rapidjson_util.h" + +namespace paimon { + +/// Fixed-size VECTOR logical type backed by Arrow FixedSizeList. +class VectorType : public DataType { + public: + static constexpr char TYPE[] = "VECTOR"; + + VectorType(const std::shared_ptr& type, bool nullable, + const std::shared_ptr& metadata) + : DataType(type, nullable, metadata) {} + + static bool IsValidElementType(const std::shared_ptr& type) { + switch (type->id()) { + case arrow::Type::BOOL: + case arrow::Type::INT8: + case arrow::Type::INT16: + case arrow::Type::INT32: + case arrow::Type::INT64: + case arrow::Type::FLOAT: + case arrow::Type::DOUBLE: + return true; + default: + return false; + } + } + + rapidjson::Value ToJson(rapidjson::Document::AllocatorType* allocator) const + noexcept(false) override { + rapidjson::Value obj(rapidjson::kObjectType); + obj.AddMember( + rapidjson::StringRef("type"), + RapidJsonUtil::SerializeValue(WithNullable(std::string(TYPE)), allocator).Move(), + *allocator); + auto* type = static_cast(type_.get()); + auto value_field = type->value_field(); + std::shared_ptr data_type = + DataType::Create(value_field->type(), value_field->nullable(), value_field->metadata()); + obj.AddMember(rapidjson::StringRef("element"), + RapidJsonUtil::SerializeValue(*data_type, allocator).Move(), *allocator); + obj.AddMember(rapidjson::StringRef("length"), + RapidJsonUtil::SerializeValue(type->list_size(), allocator).Move(), + *allocator); + return obj; + } +}; + +} // namespace paimon diff --git a/src/paimon/common/utils/arrow/arrow_utils.cpp b/src/paimon/common/utils/arrow/arrow_utils.cpp index de6aedbbb..575ea335f 100644 --- a/src/paimon/common/utils/arrow/arrow_utils.cpp +++ b/src/paimon/common/utils/arrow/arrow_utils.cpp @@ -112,6 +112,11 @@ void ArrowUtils::TraverseArray(const std::shared_ptr& array) { TraverseArray(list_array->values()); return; } + case arrow::Type::type::FIXED_SIZE_LIST: { + auto* vector_array = static_cast(array.get()); + TraverseArray(vector_array->values()); + return; + } default: return; } @@ -122,6 +127,13 @@ bool ArrowUtils::EqualsIgnoreNullable(const std::shared_ptr& ty if (type->id() != other_type->id() || type->num_fields() != other_type->num_fields()) { return false; } + if (type->id() == arrow::Type::FIXED_SIZE_LIST) { + const auto& vector_type = static_cast(*type); + const auto& other_vector_type = static_cast(*other_type); + if (vector_type.list_size() != other_vector_type.list_size()) { + return false; + } + } for (int32_t i = 0; i < type->num_fields(); ++i) { const auto& field = type->field(i); const auto& other_field = other_type->field(i); @@ -155,6 +167,25 @@ Status ArrowUtils::InnerCheckNullabilityMatch(const std::shared_ptr(data); PAIMON_RETURN_NOT_OK( InnerCheckNullabilityMatch(list_type->value_field(), list_array->values())); + } else if (type->id() == arrow::Type::FIXED_SIZE_LIST) { + auto vector_type = std::static_pointer_cast(field->type()); + auto vector_array = std::static_pointer_cast(data); + const std::shared_ptr& values = vector_array->values(); + if (values->null_count() != 0) { + int32_t vector_length = vector_type->list_size(); + for (int64_t i = 0; i < vector_array->length(); ++i) { + if (vector_array->IsNull(i)) { + continue; + } + int64_t value_offset = (vector_array->offset() + i) * vector_length; + for (int32_t j = 0; j < vector_length; ++j) { + if (values->IsNull(value_offset + j)) { + return Status::Invalid(fmt::format( + "VECTOR field {} cannot contain null elements", field->name())); + } + } + } + } } else if (type->id() == arrow::Type::MAP) { auto map_type = arrow::internal::checked_pointer_cast(field->type()); auto map_array = arrow::internal::checked_pointer_cast(data); diff --git a/src/paimon/common/utils/arrow/arrow_utils_test.cpp b/src/paimon/common/utils/arrow/arrow_utils_test.cpp index a2f8d5bcd..40b6449e1 100644 --- a/src/paimon/common/utils/arrow/arrow_utils_test.cpp +++ b/src/paimon/common/utils/arrow/arrow_utils_test.cpp @@ -214,6 +214,23 @@ TEST(ArrowUtilsTest, TestCheckNullableMatchWithList) { } } +TEST(ArrowUtilsTest, TestCheckNullableMatchRejectsNullVectorElement) { + auto vector_type = arrow::fixed_size_list(arrow::float32(), 3); + auto vector_field = arrow::field("embedding", vector_type); + arrow::FloatBuilder values_builder; + ASSERT_TRUE(values_builder.Append(1.0f).ok()); + ASSERT_TRUE(values_builder.AppendNull().ok()); + ASSERT_TRUE(values_builder.Append(3.0f).ok()); + std::shared_ptr values = values_builder.Finish().ValueOrDie(); + auto vector_data = arrow::ArrayData::Make(vector_type, 1, {nullptr}, {values->data()}, 0); + auto vector_array = arrow::MakeArray(vector_data); + auto struct_array = arrow::StructArray::Make({vector_array}, {vector_field}).ValueOrDie(); + + ASSERT_NOK_WITH_MSG( + ArrowUtils::CheckNullabilityMatch(arrow::schema({vector_field}), struct_array), + "VECTOR field embedding cannot contain null elements"); +} + TEST(ArrowUtilsTest, TestCheckNullableMatchWithMap) { auto key_field = arrow::field("key", arrow::int32(), /*nullable=*/false); auto value_field = arrow::field("value", arrow::int32(), /*nullable=*/true); @@ -449,6 +466,14 @@ TEST(ArrowUtilsTest, TestEqualsIgnoreNullable) { ASSERT_TRUE(ArrowUtils::EqualsIgnoreNullable(struct_type1, struct_type3)); ASSERT_FALSE(ArrowUtils::EqualsIgnoreNullable(struct_type1, struct_type4)); } + { + auto vector3 = arrow::fixed_size_list(arrow::float32(), 3); + auto vector3_non_null = + arrow::fixed_size_list(arrow::field("item", arrow::float32(), false), 3); + auto vector5 = arrow::fixed_size_list(arrow::float32(), 5); + ASSERT_TRUE(ArrowUtils::EqualsIgnoreNullable(vector3, vector3_non_null)); + ASSERT_FALSE(ArrowUtils::EqualsIgnoreNullable(vector3, vector5)); + } { // test complex auto key_field = arrow::field("key", arrow::int32(), /*nullable=*/false); diff --git a/src/paimon/common/utils/field_type_utils.h b/src/paimon/common/utils/field_type_utils.h index 722467699..d2786e26e 100644 --- a/src/paimon/common/utils/field_type_utils.h +++ b/src/paimon/common/utils/field_type_utils.h @@ -93,6 +93,8 @@ class FieldTypeUtils { return FieldType::MAP; case arrow::Type::type::STRUCT: return FieldType::STRUCT; + case arrow::Type::type::FIXED_SIZE_LIST: + return FieldType::VECTOR; default: return Status::Invalid( fmt::format("Not support arrow type {}", static_cast(arrow_type))); @@ -135,6 +137,8 @@ class FieldTypeUtils { return "STRUCT"; case FieldType::VARIANT: return "VARIANT"; + case FieldType::VECTOR: + return "VECTOR"; default: return "UNKNOWN, type id:" + std::to_string(static_cast(type)); } diff --git a/src/paimon/common/utils/field_type_utils_test.cpp b/src/paimon/common/utils/field_type_utils_test.cpp index 50f602375..c70edb092 100644 --- a/src/paimon/common/utils/field_type_utils_test.cpp +++ b/src/paimon/common/utils/field_type_utils_test.cpp @@ -101,6 +101,10 @@ TEST(FieldTypeUtilsTest, ConvertToFieldType) { ASSERT_OK_AND_ASSIGN(result, FieldTypeUtils::ConvertToFieldType(arrow::Type::type::STRUCT)); ASSERT_EQ(result, FieldType::STRUCT); + ASSERT_OK_AND_ASSIGN(result, + FieldTypeUtils::ConvertToFieldType(arrow::Type::type::FIXED_SIZE_LIST)); + ASSERT_EQ(result, FieldType::VECTOR); + // Test unsupported Arrow type ASSERT_NOK(FieldTypeUtils::ConvertToFieldType(arrow::Type::type::UINT8)); } @@ -124,6 +128,7 @@ TEST(FieldTypeUtilsTest, FieldTypeToString) { ASSERT_EQ(FieldTypeUtils::FieldTypeToString(FieldType::ARRAY), "ARRAY"); ASSERT_EQ(FieldTypeUtils::FieldTypeToString(FieldType::MAP), "MAP"); ASSERT_EQ(FieldTypeUtils::FieldTypeToString(FieldType::STRUCT), "STRUCT"); + ASSERT_EQ(FieldTypeUtils::FieldTypeToString(FieldType::VECTOR), "VECTOR"); // Test UNKNOWN type auto unknown_type = static_cast(128); diff --git a/src/paimon/core/schema/arrow_schema_validator.cpp b/src/paimon/core/schema/arrow_schema_validator.cpp index e01afe010..804c7070b 100644 --- a/src/paimon/core/schema/arrow_schema_validator.cpp +++ b/src/paimon/core/schema/arrow_schema_validator.cpp @@ -29,6 +29,7 @@ #include "paimon/common/data/variant/variant_access_utils.h" #include "paimon/common/data/variant/variant_type_utils.h" #include "paimon/common/types/data_field.h" +#include "paimon/common/types/vector_type.h" #include "paimon/common/utils/decimal_utils.h" #include "paimon/common/utils/string_utils.h" #include "paimon/result.h" @@ -41,6 +42,7 @@ namespace paimon { bool ArrowSchemaValidator::IsNestedType(const std::shared_ptr& data_type) { return (data_type->id() == arrow::Type::MAP || data_type->id() == arrow::Type::LIST || + data_type->id() == arrow::Type::FIXED_SIZE_LIST || data_type->id() == arrow::Type::STRUCT); } @@ -129,6 +131,14 @@ Status ArrowSchemaValidator::ValidateDataTypeWithFieldId( value_field->type(), value_field->metadata(), /*allow_blob=*/false, field_id_set)); break; } + case arrow::Type::type::FIXED_SIZE_LIST: { + const auto& vector_type = static_cast(*type); + if (vector_type.list_size() < 1 || + !VectorType::IsValidElementType(vector_type.value_type())) { + return Status::Invalid("Invalid VECTOR type: ", type->ToString()); + } + break; + } case arrow::Type::type::STRUCT: { if (VariantTypeUtils::IsVariantMetadata(key_value_metadata)) { // A variant struct is a leaf type: its value/metadata children carry fixed @@ -208,6 +218,18 @@ Status ArrowSchemaValidator::ValidateField(const std::shared_ptr& PAIMON_RETURN_NOT_OK(ValidateField(value_field, /*allow_blob=*/false)); break; } + case arrow::Type::type::FIXED_SIZE_LIST: { + const auto& vector_type = static_cast(*field->type()); + if (vector_type.list_size() < 1) { + return Status::Invalid("Vector length must be positive, but was ", + vector_type.list_size()); + } + if (!VectorType::IsValidElementType(vector_type.value_type())) { + return Status::Invalid("Invalid element type for vector: ", + vector_type.value_type()->ToString()); + } + break; + } case arrow::Type::type::STRUCT: { if (VariantTypeUtils::IsVariantField(field)) { if (VariantAccessUtils::IsVariantAccessType(field->type())) { diff --git a/src/paimon/core/schema/arrow_schema_validator_test.cpp b/src/paimon/core/schema/arrow_schema_validator_test.cpp index 0363dff6a..ed56b6370 100644 --- a/src/paimon/core/schema/arrow_schema_validator_test.cpp +++ b/src/paimon/core/schema/arrow_schema_validator_test.cpp @@ -53,14 +53,29 @@ TEST(ArrowSchemaValidatorTest, TestSimple) { "col16", arrow::struct_({arrow::field("sub1", arrow::int8()), arrow::field("sub2", arrow::int16()), arrow::field("sub3", arrow::int64())})); + auto col17_field = arrow::field("col17", arrow::fixed_size_list(arrow::float32(), 3)); - auto arrow_schema = arrow::schema( - arrow::FieldVector({col1_field, col2_field, col3_field, col4_field, col5_field, col6_field, - col7_field, col8_field, col9_field, col10_field, col11_field, - col12_field, col13_field, col14_field, col15_field, col16_field})); + auto arrow_schema = arrow::schema(arrow::FieldVector( + {col1_field, col2_field, col3_field, col4_field, col5_field, col6_field, col7_field, + col8_field, col9_field, col10_field, col11_field, col12_field, col13_field, col14_field, + col15_field, col16_field, col17_field})); ASSERT_OK(ArrowSchemaValidator::ValidateSchema(*arrow_schema)); } +TEST(ArrowSchemaValidatorTest, TestVectorElementType) { + for (const auto& element_type : + {arrow::boolean(), arrow::int8(), arrow::int16(), arrow::int32(), arrow::int64(), + arrow::float32(), arrow::float64()}) { + auto vector = arrow::field("embedding", arrow::fixed_size_list(element_type, 3)); + ASSERT_OK(ArrowSchemaValidator::ValidateSchema(*arrow::schema({vector}))); + } + for (const auto& element_type : {arrow::utf8()}) { + auto vector = arrow::field("embedding", arrow::fixed_size_list(element_type, 3)); + ASSERT_NOK_WITH_MSG(ArrowSchemaValidator::ValidateSchema(*arrow::schema({vector})), + "Invalid element type for vector"); + } +} + TEST(ArrowSchemaValidatorTest, TestValidateNoRedundantFields) { auto col1_field = arrow::field("col1", arrow::int64()); auto col2_field = arrow::field("col2", arrow::int32()); diff --git a/src/paimon/core/schema/schema_validation.cpp b/src/paimon/core/schema/schema_validation.cpp index ed79e5661..3c21b8fd8 100644 --- a/src/paimon/core/schema/schema_validation.cpp +++ b/src/paimon/core/schema/schema_validation.cpp @@ -76,6 +76,18 @@ bool ContainsBlobField(const std::shared_ptr& field) { return false; } +bool ContainsVectorField(const std::shared_ptr& field) { + if (field->type()->id() == arrow::Type::FIXED_SIZE_LIST) { + return true; + } + for (const auto& child : field->type()->fields()) { + if (ContainsVectorField(child)) { + return true; + } + } + return false; +} + Status ValidateSharedShreddingCompression(const std::string& option_key, const std::string& compression) { std::string normalized = StringUtils::ToLowerCase(compression); @@ -98,6 +110,15 @@ Status ValidateSharedShreddingFileFormat(const std::string& option_key, return Status::OK(); } +Status ValidateVectorFileFormat(const std::string& option_key, const std::string& file_format) { + if (StringUtils::ToLowerCase(file_format) != "parquet") { + return Status::Invalid( + fmt::format("VECTOR currently only supports parquet data files, but {} is {}.", + option_key, file_format)); + } + return Status::OK(); +} + Status ValidatePerLevelOption( const std::map& options, const std::string& option_key, const std::function& validator) { @@ -188,6 +209,7 @@ Status SchemaValidation::ValidateTableSchema(const TableSchema& schema) { PAIMON_RETURN_NOT_OK(ValidateRowTracking(schema, options)); PAIMON_RETURN_NOT_OK(ValidateBlobFields(schema, options)); PAIMON_RETURN_NOT_OK(ValidateMapStorageLayout(schema, options)); + PAIMON_RETURN_NOT_OK(ValidateVectorFields(schema, options)); return Status::OK(); } @@ -625,6 +647,9 @@ Status SchemaValidation::ValidateMapStorageLayout(const TableSchema& schema, if (ContainsBlobField(map_type->item_field())) { return Status::Invalid("MAP shared-shredding currently cannot contain BLOB fields."); } + if (ContainsVectorField(map_type->item_field())) { + return Status::Invalid("MAP shared-shredding currently cannot contain VECTOR fields."); + } // Validate max-columns config PAIMON_RETURN_NOT_OK(options.GetMapSharedShreddingMaxColumns(field_name)); // Validate placement policy config @@ -651,4 +676,22 @@ Status SchemaValidation::ValidateMapStorageLayout(const TableSchema& schema, return Status::OK(); } +Status SchemaValidation::ValidateVectorFields(const TableSchema& schema, + const CoreOptions& options) { + bool has_vector = false; + for (const auto& field : schema.Fields()) { + if (ContainsVectorField(field.ArrowField())) { + has_vector = true; + break; + } + } + if (!has_vector) { + return Status::OK(); + } + PAIMON_RETURN_NOT_OK( + ValidateVectorFileFormat(Options::FILE_FORMAT, options.GetFileFormat()->Identifier())); + return ValidatePerLevelOption(options.ToMap(), Options::FILE_FORMAT_PER_LEVEL, + ValidateVectorFileFormat); +} + } // namespace paimon diff --git a/src/paimon/core/schema/schema_validation.h b/src/paimon/core/schema/schema_validation.h index 613372ff8..abf4d5b02 100644 --- a/src/paimon/core/schema/schema_validation.h +++ b/src/paimon/core/schema/schema_validation.h @@ -75,6 +75,8 @@ class SchemaValidation { static Status ValidateMapStorageLayout(const TableSchema& schema, const CoreOptions& options); + static Status ValidateVectorFields(const TableSchema& schema, const CoreOptions& options); + static bool IsComplexType(const std::shared_ptr& field); }; diff --git a/src/paimon/core/schema/schema_validation_test.cpp b/src/paimon/core/schema/schema_validation_test.cpp index 3c3054ac5..b81c54074 100644 --- a/src/paimon/core/schema/schema_validation_test.cpp +++ b/src/paimon/core/schema/schema_validation_test.cpp @@ -46,6 +46,32 @@ TEST(SchemaValidationTest, TestSimple) { ASSERT_OK(SchemaValidation::ValidateTableSchema(*table_schema)); } +TEST(SchemaValidationTest, TestVectorType) { + auto vector_field = arrow::field("embedding", arrow::fixed_size_list(arrow::float32(), 3)); + auto schema = arrow::schema({arrow::field("id", arrow::int64()), vector_field}); + std::map parquet_options = {{Options::BUCKET, "-1"}, + {Options::FILE_FORMAT, "parquet"}}; + ASSERT_OK_AND_ASSIGN(std::shared_ptr table_schema, + TableSchema::Create(/*schema_id=*/0, schema, /*partition_keys=*/{}, + /*primary_keys=*/{}, parquet_options)); + ASSERT_OK(SchemaValidation::ValidateTableSchema(*table_schema)); + + std::map orc_options = {{Options::BUCKET, "-1"}, + {Options::FILE_FORMAT, "orc"}}; + ASSERT_OK_AND_ASSIGN(table_schema, + TableSchema::Create(/*schema_id=*/0, schema, /*partition_keys=*/{}, + /*primary_keys=*/{}, orc_options)); + ASSERT_NOK_WITH_MSG(SchemaValidation::ValidateTableSchema(*table_schema), + "VECTOR currently only supports parquet data files"); + + std::map primary_key_options = {{Options::BUCKET, "1"}}; + ASSERT_OK_AND_ASSIGN(table_schema, + TableSchema::Create(/*schema_id=*/0, schema, /*partition_keys=*/{}, + /*primary_keys=*/{"embedding"}, primary_key_options)); + ASSERT_NOK_WITH_MSG(SchemaValidation::ValidateTableSchema(*table_schema), + "in primary key field embedding is unsupported"); +} + TEST(SchemaValidationTest, TestRowTracking) { auto f0 = arrow::field("f0", arrow::utf8()); auto f1 = arrow::field("f1", arrow::int32()); diff --git a/src/paimon/core/schema/table_schema.cpp b/src/paimon/core/schema/table_schema.cpp index e5139478c..ef73aa239 100644 --- a/src/paimon/core/schema/table_schema.cpp +++ b/src/paimon/core/schema/table_schema.cpp @@ -118,6 +118,14 @@ Result> TableSchema::AssignFieldIdsRecursively( /*set_field_id=*/false, field_id)); return arrow::field(field->name(), arrow::list(new_value_field), field->nullable(), metadata); + } else if (type->id() == arrow::Type::FIXED_SIZE_LIST) { + auto vector_type = std::static_pointer_cast(field->type()); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr new_value_field, + AssignFieldIdsRecursively(vector_type->value_field(), + /*set_field_id=*/false, field_id)); + return arrow::field(field->name(), + arrow::fixed_size_list(new_value_field, vector_type->list_size()), + field->nullable(), metadata); } else if (field->type()->id() == arrow::Type::MAP) { auto map_type = arrow::internal::checked_pointer_cast(field->type()); std::shared_ptr key_field = map_type->key_field(); diff --git a/src/paimon/format/parquet/CMakeLists.txt b/src/paimon/format/parquet/CMakeLists.txt index a1a566c08..c31e3cc35 100644 --- a/src/paimon/format/parquet/CMakeLists.txt +++ b/src/paimon/format/parquet/CMakeLists.txt @@ -20,6 +20,7 @@ set(PAIMON_PARQUET_FILE_FORMAT file_reader_wrapper.cpp page_filtered_row_group_reader.cpp parquet_timestamp_converter.cpp + parquet_vector_converter.cpp parquet_file_batch_reader.cpp parquet_file_format_factory.cpp parquet_format_writer.cpp @@ -55,6 +56,8 @@ if(PAIMON_BUILD_TESTS) file_reader_wrapper_test.cpp page_filtered_row_group_reader_test.cpp parquet_timestamp_converter_test.cpp + parquet_vector_converter_test.cpp + parquet_vector_io_test.cpp parquet_field_id_converter_test.cpp parquet_file_batch_reader_test.cpp parquet_format_writer_test.cpp diff --git a/src/paimon/format/parquet/parquet_field_id_converter.cpp b/src/paimon/format/parquet/parquet_field_id_converter.cpp index 010df3a0d..fe91a0673 100644 --- a/src/paimon/format/parquet/parquet_field_id_converter.cpp +++ b/src/paimon/format/parquet/parquet_field_id_converter.cpp @@ -104,6 +104,12 @@ arrow::Result> ParquetFieldIdConverter::ProcessField( ProcessField(list_type->value_field(), convert_type)); auto new_type = arrow::list(new_value_field); return field->WithType(new_type)->WithMergedMetadata(updated_metadata); + } else if (type->id() == arrow::Type::FIXED_SIZE_LIST) { + auto vector_type = std::static_pointer_cast(type); + ARROW_ASSIGN_OR_RAISE(auto new_value_field, + ProcessField(vector_type->value_field(), convert_type)); + auto new_type = arrow::fixed_size_list(new_value_field, vector_type->list_size()); + return field->WithType(new_type)->WithMergedMetadata(updated_metadata); } else if (type->id() == arrow::Type::MAP) { auto map_type = std::static_pointer_cast(type); ARROW_ASSIGN_OR_RAISE(auto new_key_field, diff --git a/src/paimon/format/parquet/parquet_field_id_converter_test.cpp b/src/paimon/format/parquet/parquet_field_id_converter_test.cpp index e85cb4403..36b6ee6c8 100644 --- a/src/paimon/format/parquet/parquet_field_id_converter_test.cpp +++ b/src/paimon/format/parquet/parquet_field_id_converter_test.cpp @@ -185,7 +185,8 @@ TEST_F(ParquetFieldIdConverterTest, TestNestedType) { arrow::field("sub2", arrow::timestamp(arrow::TimeUnit::NANO)), arrow::field("sub3", arrow::decimal128(23, 5)), arrow::field("sub4", arrow::binary()), - arrow::field("sub5", arrow::binary())})))}; + arrow::field("sub5", arrow::binary())}))), + arrow::field("f3", arrow::fixed_size_list(arrow::float32(), 7))}; auto schema = arrow::schema(fields); ASSERT_OK_AND_ASSIGN( auto table_schema, @@ -210,8 +211,11 @@ TEST_F(ParquetFieldIdConverterTest, TestNestedType) { {"sub4", arrow::Type::BINARY, "16"}, {"sub5", arrow::Type::BINARY, "17"}, {"sub1", arrow::Type::DATE32, "18"}, {"sub2", arrow::Type::TIMESTAMP, "19"}, {"sub3", arrow::Type::DECIMAL128, "20"}, {"sub4", arrow::Type::BINARY, "21"}, - {"sub5", arrow::Type::BINARY, "22"}}; + {"sub5", arrow::Type::BINARY, "22"}, {"f3", arrow::Type::FIXED_SIZE_LIST, "23"}}; ASSERT_EQ(expected_field_infos, field_infos); + auto new_vector = std::static_pointer_cast( + new_schema->GetFieldByName("f3")->type()); + ASSERT_EQ(new_vector->list_size(), 7); // convert to paimon.id ASSERT_OK_AND_ASSIGN(auto old_schema, ParquetFieldIdConverter::GetPaimonIdsFromParquetIds(new_schema)); @@ -219,6 +223,9 @@ TEST_F(ParquetFieldIdConverterTest, TestNestedType) { PrintFieldMetadata(old_schema, ParquetFieldIdConverter::IdConvertType::PARQUET_TO_PAIMON_ID, &old_field_infos); ASSERT_EQ(expected_field_infos, old_field_infos); + auto old_vector = std::static_pointer_cast( + old_schema->GetFieldByName("f3")->type()); + ASSERT_EQ(old_vector->list_size(), 7); } } // namespace paimon::parquet::test diff --git a/src/paimon/format/parquet/parquet_file_batch_reader.cpp b/src/paimon/format/parquet/parquet_file_batch_reader.cpp index c0cd40e19..bfbbe6ee4 100644 --- a/src/paimon/format/parquet/parquet_file_batch_reader.cpp +++ b/src/paimon/format/parquet/parquet_file_batch_reader.cpp @@ -49,6 +49,7 @@ #include "paimon/format/parquet/parquet_format_defs.h" #include "paimon/format/parquet/parquet_schema_util.h" #include "paimon/format/parquet/parquet_timestamp_converter.h" +#include "paimon/format/parquet/parquet_vector_converter.h" #include "paimon/format/parquet/predicate_converter.h" #include "paimon/reader/batch_reader.h" #include "paimon/utils/roaring_bitmap32.h" @@ -65,6 +66,13 @@ class Predicate; namespace paimon::parquet { namespace { +std::shared_ptr GetListElementType(const std::shared_ptr& type) { + if (type->id() == arrow::Type::FIXED_SIZE_LIST) { + return static_cast(*type).value_type(); + } + return static_cast(*type).value_type(); +} + // LIST/MAP do not support pruning fields from their nested value types, but physical and // logical leaf types may still differ (for example, Parquet reports LTZ timestamps as UTC // while Paimon exposes them in the local timezone). Compare only the nested projection shape @@ -89,7 +97,9 @@ bool HasSameNestedProjectionShape(const std::shared_ptr& read_t } return read_type->Equals(file_type); } - if (read_type->id() != file_type->id()) { + const bool vector_from_list = + read_type->id() == arrow::Type::FIXED_SIZE_LIST && file_type->id() == arrow::Type::LIST; + if (read_type->id() != file_type->id() && !vector_from_list) { return false; } @@ -109,9 +119,18 @@ bool HasSameNestedProjectionShape(const std::shared_ptr& read_t return true; } case arrow::Type::LIST: { - const auto& read_list = static_cast(*read_type); const auto& file_list = static_cast(*file_type); - return HasSameNestedProjectionShape(read_list.value_type(), file_list.value_type()); + return HasSameNestedProjectionShape(GetListElementType(read_type), + file_list.value_type()); + } + case arrow::Type::FIXED_SIZE_LIST: { + if (file_type->id() != arrow::Type::FIXED_SIZE_LIST) { + return false; + } + const auto& read_vector = static_cast(*read_type); + const auto& file_vector = static_cast(*file_type); + return read_vector.list_size() == file_vector.list_size() && + HasSameNestedProjectionShape(read_vector.value_type(), file_vector.value_type()); } case arrow::Type::MAP: { const auto& read_map = static_cast(*read_type); @@ -597,6 +616,9 @@ Result ParquetFileBatchReader::NextBatch() { PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr array, batch->ToStructArray()); PAIMON_RETURN_NOT_OK_FROM_ARROW(array->Validate()); + PAIMON_ASSIGN_OR_RAISE(array, ParquetVectorConverter::ConvertToReadType( + array, read_data_type_, arrow_pool_.get())); + PAIMON_RETURN_NOT_OK_FROM_ARROW(array->Validate()); PAIMON_ASSIGN_OR_RAISE(bool need_cast, ParquetTimestampConverter::NeedCastArrayForTimestamp( array->type(), read_data_type_)); if (need_cast) { @@ -728,7 +750,8 @@ Status ParquetFileBatchReader::CollectLeafIndices(const std::shared_ptrtype(), leaf_index); } } - } else if (file_type->id() == arrow::Type::LIST) { + } else if (file_type->id() == arrow::Type::LIST || + file_type->id() == arrow::Type::FIXED_SIZE_LIST) { // Keep behavior aligned with ORC path: list/map inner partial projection // is currently unsupported and should fail-fast. if (!HasSameNestedProjectionShape(read_type, file_type)) { @@ -736,10 +759,8 @@ Status ParquetFileBatchReader::CollectLeafIndices(const std::shared_ptrToString(), read_type->ToString())); } - const auto& read_list = static_cast(*read_type); - const auto& file_list = static_cast(*file_type); - PAIMON_RETURN_NOT_OK(CollectLeafIndices(read_list.value_type(), file_list.value_type(), - leaf_index, indices)); + PAIMON_RETURN_NOT_OK(CollectLeafIndices( + GetListElementType(read_type), GetListElementType(file_type), leaf_index, indices)); } else if (file_type->id() == arrow::Type::MAP) { if (!HasSameNestedProjectionShape(read_type, file_type)) { return Status::Invalid(fmt::format( @@ -762,7 +783,7 @@ Status ParquetFileBatchReader::CollectLeafIndices(const std::shared_ptr& file_type, int32_t* leaf_index) { if (file_type->id() == arrow::Type::STRUCT || file_type->id() == arrow::Type::LIST || - file_type->id() == arrow::Type::MAP) { + file_type->id() == arrow::Type::FIXED_SIZE_LIST || file_type->id() == arrow::Type::MAP) { for (int32_t i = 0; i < file_type->num_fields(); i++) { SkipLeafIndices(file_type->field(i)->type(), leaf_index); } diff --git a/src/paimon/format/parquet/parquet_file_batch_reader.h b/src/paimon/format/parquet/parquet_file_batch_reader.h index 7e5e9afab..2c47c2049 100644 --- a/src/paimon/format/parquet/parquet_file_batch_reader.h +++ b/src/paimon/format/parquet/parquet_file_batch_reader.h @@ -167,7 +167,7 @@ class ParquetFileBatchReader : public PrefetchFileBatchReader { static void FlattenSchema(const std::shared_ptr& type, int32_t* index, std::vector* index_vector) { if (type->id() == arrow::Type::STRUCT || type->id() == arrow::Type::LIST || - type->id() == arrow::Type::MAP) { + type->id() == arrow::Type::FIXED_SIZE_LIST || type->id() == arrow::Type::MAP) { for (int32_t i = 0; i < type->num_fields(); i++) { auto field = type->field(i); auto inner_type = field->type(); diff --git a/src/paimon/format/parquet/parquet_format_writer.cpp b/src/paimon/format/parquet/parquet_format_writer.cpp index 0a8e38b43..a9866332b 100644 --- a/src/paimon/format/parquet/parquet_format_writer.cpp +++ b/src/paimon/format/parquet/parquet_format_writer.cpp @@ -23,6 +23,7 @@ #include #include +#include "arrow/array/array_nested.h" #include "arrow/c/bridge.h" #include "arrow/memory_pool.h" #include "arrow/record_batch.h" @@ -32,6 +33,7 @@ #include "paimon/common/utils/arrow/arrow_output_stream_adapter.h" #include "paimon/common/utils/arrow/status_utils.h" #include "paimon/format/parquet/parquet_format_defs.h" +#include "paimon/format/parquet/parquet_vector_converter.h" #include "parquet/arrow/writer.h" #include "parquet/properties.h" @@ -55,17 +57,31 @@ Result> ParquetFormatWriter::Create( ::parquet::ArrowWriterProperties::Builder arrow_properties_builder; auto arrow_writer_properties = arrow_properties_builder.enable_deprecated_int96_timestamps()->build(); + auto logical_type = arrow::struct_(schema->fields()); + auto write_type = std::static_pointer_cast( + ParquetVectorConverter::GetWriteType(logical_type)); + auto write_schema = arrow::schema(write_type->fields(), schema->metadata()); PAIMON_ASSIGN_OR_RAISE_FROM_ARROW( std::unique_ptr<::parquet::arrow::FileWriter> file_writer, - ::parquet::arrow::FileWriter::Open(*schema, pool.get(), out, writer_properties, + ::parquet::arrow::FileWriter::Open(*write_schema, pool.get(), out, writer_properties, arrow_writer_properties)); return std::unique_ptr( - new ParquetFormatWriter(std::move(file_writer), out, schema, max_memory_use, pool)); + new ParquetFormatWriter(std::move(file_writer), out, schema, max_memory_use, pool, + !logical_type->Equals(write_type))); } Status ParquetFormatWriter::AddBatch(ArrowArray* batch) { PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr<::arrow::RecordBatch> record_batch, arrow::ImportRecordBatch(batch, schema_)); + if (needs_vector_conversion_) { + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr struct_array, + record_batch->ToStructArray()); + std::shared_ptr array = struct_array; + PAIMON_ASSIGN_OR_RAISE(array, + ParquetVectorConverter::ConvertToWriteType(array, pool_.get())); + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(record_batch, + arrow::RecordBatch::FromStructArray(array, pool_.get())); + } if (static_cast(pool_->bytes_allocated()) > max_memory_use_) { PAIMON_RETURN_NOT_OK_FROM_ARROW(writer_->NewBufferedRowGroup()); } @@ -114,12 +130,14 @@ ParquetFormatWriter::ParquetFormatWriter(std::unique_ptr<::parquet::arrow::FileW const std::shared_ptr& out, const std::shared_ptr& schema, uint64_t max_memory_use, - const std::shared_ptr& pool) + const std::shared_ptr& pool, + bool needs_vector_conversion) : pool_(pool), out_(out), writer_(std::move(writer)), schema_(schema), metrics_(std::make_shared()), - max_memory_use_(max_memory_use) {} + max_memory_use_(max_memory_use), + needs_vector_conversion_(needs_vector_conversion) {} } // namespace paimon::parquet diff --git a/src/paimon/format/parquet/parquet_format_writer.h b/src/paimon/format/parquet/parquet_format_writer.h index 4ab58d73c..3d5956d66 100644 --- a/src/paimon/format/parquet/parquet_format_writer.h +++ b/src/paimon/format/parquet/parquet_format_writer.h @@ -72,7 +72,8 @@ class ParquetFormatWriter : public FormatWriter { ParquetFormatWriter(std::unique_ptr<::parquet::arrow::FileWriter> writer, const std::shared_ptr& out, const std::shared_ptr& schema, uint64_t max_memory_use, - const std::shared_ptr& pool); + const std::shared_ptr& pool, + bool needs_vector_conversion); Result GetEstimateLength() const; @@ -83,6 +84,7 @@ class ParquetFormatWriter : public FormatWriter { std::shared_ptr metrics_; int64_t total_records_written_ = 0; uint64_t max_memory_use_; + bool needs_vector_conversion_; }; } // namespace paimon::parquet diff --git a/src/paimon/format/parquet/parquet_stats_extractor.cpp b/src/paimon/format/parquet/parquet_stats_extractor.cpp index f1df8b6e0..b976b3ebc 100644 --- a/src/paimon/format/parquet/parquet_stats_extractor.cpp +++ b/src/paimon/format/parquet/parquet_stats_extractor.cpp @@ -312,7 +312,9 @@ ParquetStatsExtractor::ExtractWithFileInfo(const std::shared_ptr& fi // nested type do not have parquet stats const auto& logical_type = node->logical_type(); FieldType nested_type = FieldType::UNKNOWN; - if (logical_type->is_list()) { + if (write_schema_->field(field_idx)->type()->id() == arrow::Type::FIXED_SIZE_LIST) { + nested_type = FieldType::VECTOR; + } else if (logical_type->is_list()) { nested_type = FieldType::ARRAY; } else if (logical_type->is_map()) { nested_type = FieldType::MAP; diff --git a/src/paimon/format/parquet/parquet_stats_extractor_test.cpp b/src/paimon/format/parquet/parquet_stats_extractor_test.cpp index 4dc7fbe58..65998426d 100644 --- a/src/paimon/format/parquet/parquet_stats_extractor_test.cpp +++ b/src/paimon/format/parquet/parquet_stats_extractor_test.cpp @@ -62,7 +62,8 @@ class ParquetStatsExtractorTest : public ::testing::Test { void TearDown() override {} void CheckStats(const arrow::FieldVector& fields, const std::string& input, - const std::vector& expected_stats, int64_t expect_row_count) { + const std::vector& expected_stats, int64_t expect_row_count, + const std::vector& expected_types = {}) { auto arrow_schema = arrow::schema(fields); auto struct_type = arrow::struct_(fields); std::map options; @@ -95,6 +96,12 @@ class ParquetStatsExtractorTest : public ::testing::Test { for (size_t i = 0; i < expected_stats.size(); i++) { ASSERT_EQ(expected_stats[i], col_stats_vec[i]->ToString()); } + if (!expected_types.empty()) { + ASSERT_EQ(col_stats_vec.size(), expected_types.size()); + for (size_t i = 0; i < expected_types.size(); ++i) { + ASSERT_EQ(col_stats_vec[i]->GetFieldType(), expected_types[i]); + } + } auto row_count = result.second.GetRowCount(); ASSERT_EQ(row_count, expect_row_count); } @@ -237,6 +244,13 @@ TEST_F(ParquetStatsExtractorTest, TestExtractStatsComplexType) { CheckStats(fields, data_str, expected_stats_str, /*expect_row_count=*/6); } +TEST_F(ParquetStatsExtractorTest, TestExtractVectorStats) { + arrow::FieldVector fields = { + arrow::field("embedding", arrow::fixed_size_list(arrow::float32(), 3))}; + CheckStats(fields, R"([[[1.0, 2.0, 3.0]], [null]])", {"min null, max null, null count null"}, + /*expect_row_count=*/2, {FieldType::VECTOR}); +} + TEST_F(ParquetStatsExtractorTest, TestNullForAllType) { auto timezone = DateTimeUtils::GetLocalTimezoneName(); arrow::FieldVector fields = { diff --git a/src/paimon/format/parquet/parquet_vector_converter.cpp b/src/paimon/format/parquet/parquet_vector_converter.cpp new file mode 100644 index 000000000..205df51f4 --- /dev/null +++ b/src/paimon/format/parquet/parquet_vector_converter.cpp @@ -0,0 +1,343 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "paimon/format/parquet/parquet_vector_converter.h" + +#include +#include +#include +#include + +#include "arrow/array.h" +#include "arrow/array/array_nested.h" +#include "arrow/array/builder_primitive.h" +#include "arrow/compute/api.h" +#include "arrow/type.h" +#include "fmt/format.h" +#include "paimon/common/utils/arrow/status_utils.h" +#include "paimon/status.h" + +namespace paimon::parquet { +namespace { + +bool ContainsVectorType(const std::shared_ptr& type) { + if (type->id() == arrow::Type::FIXED_SIZE_LIST) { + return true; + } + for (const auto& field : type->fields()) { + if (ContainsVectorType(field->type())) { + return true; + } + } + return false; +} + +Status ValidateVectorElements(const arrow::FixedSizeListArray& array, int32_t vector_length) { + const std::shared_ptr& values = array.values(); + if (values->null_count() == 0) { + return Status::OK(); + } + for (int64_t i = 0; i < array.length(); ++i) { + if (array.IsNull(i)) { + continue; + } + int64_t value_offset = (array.offset() + i) * vector_length; + for (int32_t j = 0; j < vector_length; ++j) { + if (values->IsNull(value_offset + j)) { + return Status::Invalid("VECTOR cannot contain null elements"); + } + } + } + return Status::OK(); +} + +Result GetIndexCapacity(int64_t row_count, int32_t vector_length) { + if (vector_length < 1) { + return Status::Invalid("VECTOR length must be positive"); + } + if (row_count > std::numeric_limits::max() / vector_length) { + return Status::Invalid("VECTOR values exceed the supported Arrow array length"); + } + return row_count * vector_length; +} + +Result> ConvertListToVector( + const std::shared_ptr& array, + const std::shared_ptr& read_type, arrow::MemoryPool* pool) { + int32_t vector_length = read_type->list_size(); + if (array->type()->id() == arrow::Type::FIXED_SIZE_LIST) { + const auto& source_type = static_cast(*array->type()); + if (source_type.list_size() != vector_length || + !source_type.value_type()->Equals(read_type->value_type())) { + return Status::Invalid(fmt::format("VECTOR type mismatch: data {} vs read {}", + array->type()->ToString(), read_type->ToString())); + } + const auto& vector_array = static_cast(*array); + PAIMON_RETURN_NOT_OK(ValidateVectorElements(vector_array, vector_length)); + std::shared_ptr data = array->data()->Copy(); + data->type = read_type; + return arrow::MakeArray(data); + } + if (array->type()->id() != arrow::Type::LIST) { + return Status::Invalid( + fmt::format("Cannot restore VECTOR from parquet type {}", array->type()->ToString())); + } + + const auto& list_array = static_cast(*array); + if (!list_array.value_type()->Equals(read_type->value_type())) { + return Status::Invalid(fmt::format("VECTOR element type mismatch: data {} vs read {}", + list_array.value_type()->ToString(), + read_type->value_type()->ToString())); + } + + arrow::Int64Builder indices_builder(pool); + PAIMON_ASSIGN_OR_RAISE(int64_t index_capacity, + GetIndexCapacity(list_array.length(), vector_length)); + PAIMON_RETURN_NOT_OK_FROM_ARROW(indices_builder.Reserve(index_capacity)); + arrow::BooleanBuilder validity_builder(pool); + PAIMON_RETURN_NOT_OK_FROM_ARROW(validity_builder.Reserve(list_array.length())); + + for (int64_t i = 0; i < list_array.length(); ++i) { + bool valid = !list_array.IsNull(i); + PAIMON_RETURN_NOT_OK_FROM_ARROW(validity_builder.Append(valid)); + if (!valid) { + PAIMON_RETURN_NOT_OK_FROM_ARROW(indices_builder.AppendNulls(vector_length)); + continue; + } + int64_t value_length = list_array.value_length(i); + if (value_length != vector_length) { + return Status::Invalid( + fmt::format("Vector length mismatch at row {}: expected {} but got {}", i, + vector_length, value_length)); + } + int64_t value_offset = list_array.value_offset(i); + for (int32_t j = 0; j < vector_length; ++j) { + int64_t index = value_offset + j; + if (list_array.values()->IsNull(index)) { + return Status::Invalid(fmt::format( + "VECTOR cannot contain null elements, found one at row {} position {}", i, j)); + } + PAIMON_RETURN_NOT_OK_FROM_ARROW(indices_builder.Append(index)); + } + } + + std::shared_ptr indices; + PAIMON_RETURN_NOT_OK_FROM_ARROW(indices_builder.Finish(&indices)); + arrow::compute::ExecContext exec_context(pool); + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW( + arrow::Datum values, + arrow::compute::Take(arrow::Datum(list_array.values()), arrow::Datum(indices), + arrow::compute::TakeOptions::NoBoundsCheck(), &exec_context)); + + std::shared_ptr validity; + PAIMON_RETURN_NOT_OK_FROM_ARROW(validity_builder.Finish(&validity)); + std::shared_ptr null_bitmap; + if (list_array.null_count() != 0) { + null_bitmap = validity->data()->buffers[1]; + } + std::shared_ptr data = + arrow::ArrayData::Make(read_type, list_array.length(), {null_bitmap}, + {values.make_array()->data()}, list_array.null_count()); + std::shared_ptr result = arrow::MakeArray(data); + PAIMON_RETURN_NOT_OK_FROM_ARROW(result->ValidateFull()); + return result; +} + +Result> ConvertVectorToList( + const std::shared_ptr& array, arrow::MemoryPool* pool) { + const auto& vector_array = static_cast(*array); + const auto& vector_type = static_cast(*array->type()); + int32_t vector_length = vector_type.list_size(); + PAIMON_RETURN_NOT_OK(ValidateVectorElements(vector_array, vector_length)); + + arrow::Int32Builder offsets_builder(pool); + arrow::Int64Builder indices_builder(pool); + arrow::BooleanBuilder validity_builder(pool); + PAIMON_RETURN_NOT_OK_FROM_ARROW(offsets_builder.Reserve(vector_array.length() + 1)); + PAIMON_ASSIGN_OR_RAISE(int64_t index_capacity, + GetIndexCapacity(vector_array.length(), vector_length)); + PAIMON_RETURN_NOT_OK_FROM_ARROW(indices_builder.Reserve(index_capacity)); + PAIMON_RETURN_NOT_OK_FROM_ARROW(validity_builder.Reserve(vector_array.length())); + PAIMON_RETURN_NOT_OK_FROM_ARROW(offsets_builder.Append(0)); + + int32_t offset = 0; + for (int64_t i = 0; i < vector_array.length(); ++i) { + bool valid = !vector_array.IsNull(i); + PAIMON_RETURN_NOT_OK_FROM_ARROW(validity_builder.Append(valid)); + if (valid) { + if (vector_length > std::numeric_limits::max() - offset) { + return Status::Invalid("VECTOR values exceed the maximum Parquet LIST offset"); + } + int64_t value_offset = (vector_array.offset() + i) * vector_length; + for (int32_t j = 0; j < vector_length; ++j) { + PAIMON_RETURN_NOT_OK_FROM_ARROW(indices_builder.Append(value_offset + j)); + } + offset += vector_length; + } + PAIMON_RETURN_NOT_OK_FROM_ARROW(offsets_builder.Append(offset)); + } + + std::shared_ptr offsets; + std::shared_ptr indices; + std::shared_ptr validity; + PAIMON_RETURN_NOT_OK_FROM_ARROW(offsets_builder.Finish(&offsets)); + PAIMON_RETURN_NOT_OK_FROM_ARROW(indices_builder.Finish(&indices)); + PAIMON_RETURN_NOT_OK_FROM_ARROW(validity_builder.Finish(&validity)); + + arrow::compute::ExecContext exec_context(pool); + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW( + arrow::Datum values, + arrow::compute::Take(arrow::Datum(vector_array.values()), arrow::Datum(indices), + arrow::compute::TakeOptions::NoBoundsCheck(), &exec_context)); + std::shared_ptr null_bitmap; + if (vector_array.null_count() != 0) { + null_bitmap = validity->data()->buffers[1]; + } + std::shared_ptr write_type = + arrow::list(vector_type.value_field()->WithType(values.type())); + return std::make_shared(write_type, vector_array.length(), + offsets->data()->buffers[1], values.make_array(), + null_bitmap, vector_array.null_count()); +} + +std::shared_ptr RebuildNestedType( + const std::shared_ptr& read_type, + const std::vector>& children) { + if (read_type->id() == arrow::Type::STRUCT) { + arrow::FieldVector fields; + fields.reserve(children.size()); + for (int32_t i = 0; i < static_cast(children.size()); ++i) { + fields.push_back(read_type->field(i)->WithType(children[i]->type)); + } + return arrow::struct_(fields); + } + if (read_type->id() == arrow::Type::LIST) { + return arrow::list(read_type->field(0)->WithType(children[0]->type)); + } + + const auto& entries_type = static_cast(*children[0]->type); + const auto& map_type = static_cast(*read_type); + return std::make_shared(entries_type.field(0), entries_type.field(1), + map_type.keys_sorted()); +} + +} // namespace + +std::shared_ptr ParquetVectorConverter::GetWriteType( + const std::shared_ptr& logical_type) { + switch (logical_type->id()) { + case arrow::Type::FIXED_SIZE_LIST: { + const auto& vector_type = static_cast(*logical_type); + return arrow::list( + vector_type.value_field()->WithType(GetWriteType(vector_type.value_type()))); + } + case arrow::Type::STRUCT: { + arrow::FieldVector fields; + fields.reserve(logical_type->num_fields()); + for (const auto& field : logical_type->fields()) { + fields.push_back(field->WithType(GetWriteType(field->type()))); + } + return arrow::struct_(fields); + } + case arrow::Type::LIST: + return arrow::list( + logical_type->field(0)->WithType(GetWriteType(logical_type->field(0)->type()))); + case arrow::Type::MAP: { + const auto& map_type = static_cast(*logical_type); + return std::make_shared( + map_type.key_field()->WithType(GetWriteType(map_type.key_type())), + map_type.item_field()->WithType(GetWriteType(map_type.item_type())), + map_type.keys_sorted()); + } + default: + return logical_type; + } +} + +Result> ParquetVectorConverter::ConvertToReadType( + const std::shared_ptr& array, const std::shared_ptr& read_type, + arrow::MemoryPool* pool) { + if (!ContainsVectorType(read_type)) { + return array; + } + switch (read_type->id()) { + case arrow::Type::FIXED_SIZE_LIST: + return ConvertListToVector( + array, std::static_pointer_cast(read_type), pool); + case arrow::Type::STRUCT: + case arrow::Type::LIST: + case arrow::Type::MAP: { + if (array->type()->id() != read_type->id()) { + return Status::Invalid(fmt::format("Cannot reconcile parquet type {} with {}", + array->type()->ToString(), + read_type->ToString())); + } + if (array->type()->num_fields() != read_type->num_fields()) { + return Status::Invalid(fmt::format("Nested type field count mismatch: {} vs {}", + array->type()->ToString(), + read_type->ToString())); + } + std::vector> children; + children.reserve(read_type->num_fields()); + for (int32_t i = 0; i < read_type->num_fields(); ++i) { + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr child, + ConvertToReadType(arrow::MakeArray(array->data()->child_data[i]), + read_type->field(i)->type(), pool)); + children.push_back(child->data()); + } + std::shared_ptr data = array->data()->Copy(); + data->child_data = std::move(children); + data->type = RebuildNestedType(read_type, data->child_data); + return arrow::MakeArray(data); + } + default: + return array; + } +} + +Result> ParquetVectorConverter::ConvertToWriteType( + const std::shared_ptr& array, arrow::MemoryPool* pool) { + if (!ContainsVectorType(array->type())) { + return array; + } + if (array->type()->id() == arrow::Type::FIXED_SIZE_LIST) { + return ConvertVectorToList(array, pool); + } + switch (array->type()->id()) { + case arrow::Type::STRUCT: + case arrow::Type::LIST: + case arrow::Type::MAP: { + std::vector> children; + children.reserve(array->type()->num_fields()); + for (const auto& child_data : array->data()->child_data) { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr child, + ConvertToWriteType(arrow::MakeArray(child_data), pool)); + children.push_back(child->data()); + } + std::shared_ptr data = array->data()->Copy(); + data->child_data = std::move(children); + data->type = GetWriteType(array->type()); + return arrow::MakeArray(data); + } + default: + return array; + } +} + +} // namespace paimon::parquet diff --git a/src/paimon/format/parquet/parquet_vector_converter.h b/src/paimon/format/parquet/parquet_vector_converter.h new file mode 100644 index 000000000..3efaf57bd --- /dev/null +++ b/src/paimon/format/parquet/parquet_vector_converter.h @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include + +#include "paimon/result.h" + +namespace arrow { +class Array; +class DataType; +class MemoryPool; +} // namespace arrow + +namespace paimon::parquet { + +/// Restores logical FixedSizeList VECTOR arrays from Parquet LIST arrays. +class ParquetVectorConverter { + public: + ParquetVectorConverter() = delete; + ~ParquetVectorConverter() = delete; + + static Result> ConvertToReadType( + const std::shared_ptr& array, + const std::shared_ptr& read_type, arrow::MemoryPool* pool); + + static Result> ConvertToWriteType( + const std::shared_ptr& array, arrow::MemoryPool* pool); + + static std::shared_ptr GetWriteType( + const std::shared_ptr& logical_type); +}; + +} // namespace paimon::parquet diff --git a/src/paimon/format/parquet/parquet_vector_converter_test.cpp b/src/paimon/format/parquet/parquet_vector_converter_test.cpp new file mode 100644 index 000000000..646a3685c --- /dev/null +++ b/src/paimon/format/parquet/parquet_vector_converter_test.cpp @@ -0,0 +1,150 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "paimon/format/parquet/parquet_vector_converter.h" + +#include + +#include "arrow/api.h" +#include "arrow/ipc/json_simple.h" +#include "gtest/gtest.h" +#include "paimon/testing/utils/testharness.h" + +namespace paimon::parquet::test { + +TEST(ParquetVectorConverterTest, ConvertListToVector) { + auto physical_type = arrow::list(arrow::float32()); + auto physical_array = arrow::ipc::internal::json::ArrayFromJSON( + physical_type, R"([[1.0, 2.0, 3.0], null, [4.0, 5.0, 6.0]])") + .ValueOrDie(); + auto vector_type = arrow::fixed_size_list(arrow::float32(), 3); + + ASSERT_OK_AND_ASSIGN(std::shared_ptr converted, + ParquetVectorConverter::ConvertToReadType(physical_array, vector_type, + arrow::default_memory_pool())); + ASSERT_EQ(converted->type()->id(), arrow::Type::FIXED_SIZE_LIST); + auto vector_array = std::static_pointer_cast(converted); + ASSERT_EQ(vector_array->length(), 3); + ASSERT_FALSE(vector_array->IsNull(0)); + ASSERT_TRUE(vector_array->IsNull(1)); + ASSERT_FALSE(vector_array->IsNull(2)); + auto values = std::static_pointer_cast(vector_array->values()); + ASSERT_FLOAT_EQ(values->Value(0), 1.0f); + ASSERT_FLOAT_EQ(values->Value(2), 3.0f); + ASSERT_FLOAT_EQ(values->Value(6), 4.0f); + ASSERT_FLOAT_EQ(values->Value(8), 6.0f); +} + +TEST(ParquetVectorConverterTest, RejectInvalidVectorValues) { + auto vector_type = arrow::fixed_size_list(arrow::float32(), 3); + for (const char* json : {R"([[1.0, 2.0]])", R"([[1.0, null, 3.0]])"}) { + auto physical_array = + arrow::ipc::internal::json::ArrayFromJSON(arrow::list(arrow::float32()), json) + .ValueOrDie(); + ASSERT_NOK(ParquetVectorConverter::ConvertToReadType(physical_array, vector_type, + arrow::default_memory_pool())); + } +} + +TEST(ParquetVectorConverterTest, ConvertNullableVectorToList) { + auto vector_type = arrow::fixed_size_list(arrow::float32(), 3); + auto vector_array = arrow::ipc::internal::json::ArrayFromJSON( + vector_type, R"([[1.0, 2.0, 3.0], null, [4.0, 5.0, 6.0]])") + .ValueOrDie(); + + ASSERT_OK_AND_ASSIGN( + std::shared_ptr converted, + ParquetVectorConverter::ConvertToWriteType(vector_array, arrow::default_memory_pool())); + ASSERT_EQ(converted->type()->id(), arrow::Type::LIST); + auto list_array = std::static_pointer_cast(converted); + ASSERT_EQ(list_array->value_length(0), 3); + ASSERT_TRUE(list_array->IsNull(1)); + ASSERT_EQ(list_array->value_length(1), 0); + ASSERT_EQ(list_array->value_length(2), 3); + ASSERT_EQ(list_array->values()->length(), 6); +} + +TEST(ParquetVectorConverterTest, PreserveUnconvertedNestedTypes) { + auto physical_type = arrow::struct_({ + arrow::field("ts", arrow::timestamp(arrow::TimeUnit::MILLI)), + arrow::field("embedding", arrow::list(arrow::float32())), + }); + auto physical_array = arrow::ipc::internal::json::ArrayFromJSON( + physical_type, R"([["1970-01-01 00:00:01.000", [1.0, 2.0, 3.0]]])") + .ValueOrDie(); + auto read_type = arrow::struct_({ + arrow::field("ts", arrow::timestamp(arrow::TimeUnit::SECOND)), + arrow::field("embedding", arrow::fixed_size_list(arrow::float32(), 3)), + }); + + ASSERT_OK_AND_ASSIGN(std::shared_ptr converted, + ParquetVectorConverter::ConvertToReadType(physical_array, read_type, + arrow::default_memory_pool())); + auto converted_type = std::static_pointer_cast(converted->type()); + ASSERT_TRUE(converted_type->field(0)->type()->Equals(arrow::timestamp(arrow::TimeUnit::MILLI))); + ASSERT_EQ(converted_type->field(1)->type()->id(), arrow::Type::FIXED_SIZE_LIST); +} + +TEST(ParquetVectorConverterTest, ConvertVectorNestedInListAndMap) { + auto vector_type = + arrow::fixed_size_list(arrow::field("item", arrow::float32(), /*nullable=*/false), 2); + auto nested_type = arrow::struct_({ + arrow::field("vectors", arrow::list(vector_type)), + arrow::field("by_name", arrow::map(arrow::utf8(), vector_type)), + }); + auto nested_array = + arrow::ipc::internal::json::ArrayFromJSON(nested_type, + R"([[[[1.0, 2.0], null], [["a", [3.0, 4.0]]]], + [null, [["b", null]]]])") + .ValueOrDie(); + + ASSERT_OK_AND_ASSIGN( + std::shared_ptr physical_array, + ParquetVectorConverter::ConvertToWriteType(nested_array, arrow::default_memory_pool())); + auto physical_type = std::static_pointer_cast(physical_array->type()); + auto physical_list = std::static_pointer_cast(physical_type->field(0)->type()); + auto physical_map = std::static_pointer_cast(physical_type->field(1)->type()); + ASSERT_EQ(physical_list->value_type()->id(), arrow::Type::LIST); + ASSERT_EQ(physical_map->item_type()->id(), arrow::Type::LIST); + + ASSERT_OK_AND_ASSIGN(std::shared_ptr converted, + ParquetVectorConverter::ConvertToReadType(physical_array, nested_type, + arrow::default_memory_pool())); + ASSERT_TRUE(converted->Equals(nested_array)); +} + +TEST(ParquetVectorConverterTest, ConvertSlicedVectorToList) { + auto vector_type = arrow::fixed_size_list(arrow::float64(), 2); + auto vector_array = + arrow::ipc::internal::json::ArrayFromJSON(vector_type, R"([[1.0, 2.0], [3.0, 4.0], null])") + .ValueOrDie() + ->Slice(1, 2); + + ASSERT_OK_AND_ASSIGN( + std::shared_ptr converted, + ParquetVectorConverter::ConvertToWriteType(vector_array, arrow::default_memory_pool())); + auto list_array = std::static_pointer_cast(converted); + ASSERT_EQ(list_array->length(), 2); + ASSERT_EQ(list_array->value_length(0), 2); + ASSERT_TRUE(list_array->IsNull(1)); + auto values = std::static_pointer_cast(list_array->values()); + ASSERT_DOUBLE_EQ(values->Value(0), 3.0); + ASSERT_DOUBLE_EQ(values->Value(1), 4.0); +} + +} // namespace paimon::parquet::test diff --git a/src/paimon/format/parquet/parquet_vector_io_test.cpp b/src/paimon/format/parquet/parquet_vector_io_test.cpp new file mode 100644 index 000000000..5e224bc80 --- /dev/null +++ b/src/paimon/format/parquet/parquet_vector_io_test.cpp @@ -0,0 +1,155 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include + +#include "arrow/api.h" +#include "arrow/c/abi.h" +#include "arrow/c/bridge.h" +#include "arrow/ipc/json_simple.h" +#include "gtest/gtest.h" +#include "paimon/common/utils/arrow/arrow_input_stream_adapter.h" +#include "paimon/common/utils/arrow/mem_utils.h" +#include "paimon/format/parquet/parquet_file_batch_reader.h" +#include "paimon/format/parquet/parquet_format_defs.h" +#include "paimon/format/parquet/parquet_format_writer.h" +#include "paimon/fs/file_system.h" +#include "paimon/memory/memory_pool.h" +#include "paimon/testing/utils/read_result_collector.h" +#include "paimon/testing/utils/testharness.h" +#include "parquet/properties.h" + +namespace paimon::parquet::test { + +class ParquetVectorIoTest : public ::testing::Test { + public: + void SetUp() override { + pool_ = GetDefaultPool(); + arrow_pool_ = GetArrowPool(pool_); + dir_ = paimon::test::UniqueTestDirectory::Create(); + ASSERT_TRUE(dir_); + fs_ = dir_->GetFileSystem(); + } + + void WriteAndCheck(const std::string& file_name, + const std::shared_ptr& write_type, + const std::shared_ptr& read_type, + const std::string& json) { + arrow::Result> write_array_result = + arrow::ipc::internal::json::ArrayFromJSON(write_type, json); + ASSERT_TRUE(write_array_result.ok()) << write_array_result.status().ToString(); + std::shared_ptr write_array = std::move(write_array_result).ValueOrDie(); + auto c_array = std::make_unique(); + ASSERT_TRUE(arrow::ExportArray(*write_array, c_array.get()).ok()); + + std::string file_path = dir_->Str() + "/" + file_name; + ASSERT_OK_AND_ASSIGN(std::shared_ptr out, + fs_->Create(file_path, /*overwrite=*/false)); + ::parquet::WriterProperties::Builder properties_builder; + ASSERT_OK_AND_ASSIGN( + std::unique_ptr writer, + ParquetFormatWriter::Create(out, arrow::schema(write_type->fields()), + properties_builder.build(), + DEFAULT_PARQUET_WRITER_MAX_MEMORY_USE, arrow_pool_)); + ASSERT_OK(writer->AddBatch(c_array.get())); + ASSERT_OK(writer->Finish()); + ASSERT_OK(out->Close()); + + ASSERT_OK_AND_ASSIGN(std::shared_ptr in, fs_->Open(file_path)); + ASSERT_OK_AND_ASSIGN(int64_t length, in->Length()); + auto in_stream = std::make_shared(in, length, arrow_pool_); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr reader, + ParquetFileBatchReader::Create(std::move(in_stream), /*options=*/{}, + /*batch_size=*/10, /*file_metadata=*/nullptr, + /*storage_read_bytes=*/nullptr, arrow_pool_)); + ASSERT_OK_AND_ASSIGN(std::unique_ptr c_file_schema, reader->GetFileSchema()); + arrow::Result> file_type_result = + arrow::ImportType(c_file_schema.get()); + ASSERT_TRUE(file_type_result.ok()) << file_type_result.status().ToString(); + auto file_type = + std::static_pointer_cast(std::move(file_type_result).ValueOrDie()); + std::shared_ptr physical_value_type = file_type->field(1)->type(); + if (physical_value_type->id() == arrow::Type::STRUCT) { + physical_value_type = physical_value_type->field(0)->type(); + } + ASSERT_EQ(physical_value_type->id(), arrow::Type::LIST); + + auto c_schema = std::make_unique(); + ASSERT_TRUE(arrow::ExportSchema(*arrow::schema(read_type->fields()), c_schema.get()).ok()); + ASSERT_OK(reader->SetReadSchema(c_schema.get(), /*predicate=*/nullptr, + /*selection_bitmap=*/std::nullopt)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr actual, + paimon::test::ReadResultCollector::CollectResult(reader.get())); + + arrow::Result> expected_result = + arrow::ipc::internal::json::ArrayFromJSON(read_type, json); + ASSERT_TRUE(expected_result.ok()) << expected_result.status().ToString(); + std::shared_ptr expected = std::move(expected_result).ValueOrDie(); + ASSERT_TRUE(std::make_shared(expected)->Equals(actual)) + << actual->ToString(); + } + + private: + std::shared_ptr pool_; + std::shared_ptr arrow_pool_; + std::shared_ptr fs_; + std::unique_ptr dir_; +}; + +TEST_F(ParquetVectorIoTest, WriteAndReadVector) { + auto vector_type = + arrow::fixed_size_list(arrow::field("item", arrow::float32(), /*nullable=*/false), 3); + auto struct_type = std::static_pointer_cast(arrow::struct_( + {arrow::field("id", arrow::int32()), arrow::field("embedding", vector_type)})); + WriteAndCheck("vector.parquet", struct_type, struct_type, + R"([[1, [1.0, 2.0, 3.0]], [2, null], [3, [4.0, 5.0, 6.0]]])"); +} + +TEST_F(ParquetVectorIoTest, ReadOrdinaryParquetListAsVector) { + auto physical_type = std::static_pointer_cast( + arrow::struct_({arrow::field("id", arrow::int32()), + arrow::field("embedding", arrow::list(arrow::float32()))})); + auto logical_type = std::static_pointer_cast(arrow::struct_({ + arrow::field("id", arrow::int32()), + arrow::field("embedding", arrow::fixed_size_list(arrow::float32(), 3)), + })); + WriteAndCheck("list.parquet", physical_type, logical_type, + R"([[1, [1.0, 2.0, 3.0]], [2, null], [3, [4.0, 5.0, 6.0]]])"); +} + +TEST_F(ParquetVectorIoTest, WriteAndReadNestedDoubleVector) { + auto vector_type = + arrow::fixed_size_list(arrow::field("item", arrow::float64(), /*nullable=*/false), 2); + auto struct_type = std::static_pointer_cast(arrow::struct_({ + arrow::field("id", arrow::int32()), + arrow::field("payload", arrow::struct_({arrow::field("embedding", vector_type), + arrow::field("history", arrow::list(vector_type)), + arrow::field("by_name", arrow::map(arrow::utf8(), + vector_type))})), + })); + WriteAndCheck("nested-vector.parquet", struct_type, struct_type, + R"([[1, [[1.0, 2.0], [[3.0, 4.0], null], [["a", [5.0, 6.0]]]]], + [2, [null, null, [["b", null]]]]])"); +} + +} // namespace paimon::parquet::test diff --git a/test/inte/write_and_read_inte_test.cpp b/test/inte/write_and_read_inte_test.cpp index dd63ed6bc..514fa5d14 100644 --- a/test/inte/write_and_read_inte_test.cpp +++ b/test/inte/write_and_read_inte_test.cpp @@ -303,6 +303,66 @@ TEST_P(WriteAndReadInteTest, TestAppendSimple) { ASSERT_TRUE(success); } +TEST_P(WriteAndReadInteTest, TestAppendVector) { + auto [file_format, file_system] = GetParam(); + if (file_format != "parquet") { + return; + } + + auto vector_type = + arrow::fixed_size_list(arrow::field("item", arrow::float32(), /*nullable=*/false), 3); + arrow::FieldVector fields = {arrow::field("id", arrow::int32()), + arrow::field("embedding", vector_type)}; + std::map options = { + {Options::MANIFEST_FORMAT, "avro"}, {Options::FILE_FORMAT, file_format}, + {Options::TARGET_FILE_SIZE, "1024"}, {Options::BUCKET, "-1"}, + {Options::FILE_SYSTEM, file_system}, + }; + if (file_system == "jindo") { + options = AddOptionsForJindo(options); + } + ASSERT_OK_AND_ASSIGN(auto helper, + TestHelper::Create(test_dir_, arrow::schema(fields), /*partition_keys=*/{}, + /*primary_keys=*/{}, options, + /*is_streaming_mode=*/false)); + arrow::Int32Builder ids_builder; + ASSERT_TRUE(ids_builder.AppendValues({1, 2, 3}).ok()); + std::shared_ptr ids; + ASSERT_TRUE(ids_builder.Finish(&ids).ok()); + arrow::FloatBuilder values_builder; + ASSERT_TRUE(values_builder.AppendValues({1.0f, 2.0f, 3.0f}).ok()); + ASSERT_TRUE(values_builder.AppendNulls(3).ok()); + ASSERT_TRUE(values_builder.AppendValues({4.0f, 5.0f, 6.0f}).ok()); + std::shared_ptr values; + ASSERT_TRUE(values_builder.Finish(&values).ok()); + std::shared_ptr validity = arrow::Buffer::FromString(std::string("\x05", 1)); + auto vectors = + arrow::MakeArray(arrow::ArrayData::Make(vector_type, 3, {validity}, {values->data()}, 1)); + auto data = arrow::StructArray::Make({ids, vectors}, fields).ValueOrDie(); + auto c_array = std::make_unique(); + ASSERT_TRUE(arrow::ExportArray(*data, c_array.get()).ok()); + RecordBatchBuilder batch_builder(c_array.get()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr batch, batch_builder.SetBucket(0).Finish()); + ASSERT_OK_AND_ASSIGN(auto commit_messages, + helper->WriteAndCommit(std::move(batch), /*commit_identifier=*/0, + /*expected_commit_messages=*/std::nullopt)); + (void)commit_messages; + + arrow::FieldVector result_fields = fields; + result_fields.insert(result_fields.begin(), arrow::field("_VALUE_KIND", arrow::int8())); + ASSERT_OK_AND_ASSIGN(std::vector> data_splits, + helper->NewScan(StartupMode::LatestFull(), /*snapshot_id=*/std::nullopt)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr actual, + helper->ReadResult(data_splits)); + auto row_kinds = + std::make_shared(3, arrow::Buffer::FromString(std::string("\0\0\0", 3))); + arrow::Result> expected_result = + arrow::StructArray::Make({row_kinds, ids, vectors}, result_fields); + ASSERT_TRUE(expected_result.ok()) << expected_result.status().ToString(); + std::shared_ptr expected = std::move(expected_result).ValueOrDie(); + ASSERT_TRUE(std::make_shared(expected)->Equals(actual)); +} + TEST_P(WriteAndReadInteTest, TestPKSimple) { arrow::FieldVector fields = { arrow::field("pk", arrow::utf8()), From 3de7fdce8800ea110ef0402035366f65310a9be0 Mon Sep 17 00:00:00 2001 From: "zhangchaoming.zcm" Date: Thu, 13 Aug 2026 15:50:28 +0800 Subject: [PATCH 02/16] fix(parquet): include Arrow memory pool definition --- src/paimon/format/parquet/parquet_vector_converter.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/paimon/format/parquet/parquet_vector_converter.h b/src/paimon/format/parquet/parquet_vector_converter.h index 3efaf57bd..113eda8fb 100644 --- a/src/paimon/format/parquet/parquet_vector_converter.h +++ b/src/paimon/format/parquet/parquet_vector_converter.h @@ -20,12 +20,12 @@ #include +#include "arrow/memory_pool.h" #include "paimon/result.h" namespace arrow { class Array; class DataType; -class MemoryPool; } // namespace arrow namespace paimon::parquet { From 722e617baf2924bc99c50d5c195419df4c162ecc Mon Sep 17 00:00:00 2001 From: "zhangchaoming.zcm" Date: Sun, 16 Aug 2026 23:31:00 +0800 Subject: [PATCH 03/16] fix(parquet): address vector review feedback --- docs/source/user_guide/data_types.rst | 11 +- src/paimon/CMakeLists.txt | 2 + .../common/types/data_type_json_parser.cpp | 12 +- .../types/data_type_json_parser_test.cpp | 4 +- src/paimon/common/types/vector_type.h | 3 +- src/paimon/common/utils/arrow/arrow_utils.cpp | 10 +- .../core/io/vector_file_batch_reader.cpp | 325 ++++++++++++++++++ src/paimon/core/io/vector_file_batch_reader.h | 84 +++++ .../core/io/vector_file_batch_reader_test.cpp | 145 ++++++++ .../core/operation/abstract_split_read.cpp | 4 + .../operation/data_evolution_split_read.h | 2 +- .../core/operation/raw_file_split_read.h | 4 +- .../core/schema/arrow_schema_validator.cpp | 3 +- src/paimon/core/schema/schema_validation.cpp | 4 + .../core/schema/schema_validation_test.cpp | 18 + src/paimon/core/schema/table_schema.cpp | 2 +- .../parquet/parquet_field_id_converter.cpp | 2 +- .../parquet_field_id_converter_test.cpp | 8 +- .../parquet/parquet_file_batch_reader.cpp | 39 +-- .../parquet/parquet_file_batch_reader.h | 2 +- .../format/parquet/parquet_format_writer.cpp | 7 +- .../parquet/parquet_vector_converter.cpp | 155 +-------- .../format/parquet/parquet_vector_converter.h | 6 +- .../parquet/parquet_vector_converter_test.cpp | 75 +--- .../format/parquet/parquet_vector_io_test.cpp | 20 +- test/inte/write_and_read_inte_test.cpp | 35 +- 26 files changed, 667 insertions(+), 315 deletions(-) create mode 100644 src/paimon/core/io/vector_file_batch_reader.cpp create mode 100644 src/paimon/core/io/vector_file_batch_reader.h create mode 100644 src/paimon/core/io/vector_file_batch_reader_test.cpp diff --git a/docs/source/user_guide/data_types.rst b/docs/source/user_guide/data_types.rst index fd14ce8f0..9b0fb7613 100644 --- a/docs/source/user_guide/data_types.rst +++ b/docs/source/user_guide/data_types.rst @@ -194,11 +194,12 @@ and `Arrow DataTypes `` - Map diff --git a/src/paimon/CMakeLists.txt b/src/paimon/CMakeLists.txt index 514c1354d..636b87098 100644 --- a/src/paimon/CMakeLists.txt +++ b/src/paimon/CMakeLists.txt @@ -270,6 +270,7 @@ set(PAIMON_CORE_SRCS core/io/data_file_writer.cpp core/io/field_mapping_reader.cpp core/io/complete_row_tracking_fields_reader.cpp + core/io/vector_file_batch_reader.cpp core/io/file_index_evaluator.cpp core/io/key_value_data_file_record_reader.cpp core/io/key_value_data_file_writer_factory.cpp @@ -735,6 +736,7 @@ if(PAIMON_BUILD_TESTS) core/io/key_value_in_memory_record_reader_test.cpp core/io/merged_key_value_record_reader_test.cpp core/io/complete_row_tracking_fields_reader_test.cpp + core/io/vector_file_batch_reader_test.cpp core/io/data_file_meta_test.cpp core/io/file_index_evaluator_test.cpp core/io/single_file_writer_test.cpp diff --git a/src/paimon/common/types/data_type_json_parser.cpp b/src/paimon/common/types/data_type_json_parser.cpp index c002b23c4..e95582a17 100644 --- a/src/paimon/common/types/data_type_json_parser.cpp +++ b/src/paimon/common/types/data_type_json_parser.cpp @@ -21,13 +21,12 @@ #include #include -#include #include #include #include #include +#include #include -#include #include #include @@ -629,18 +628,15 @@ Result> TokenParser::ParseVectorType() { PAIMON_RETURN_NOT_OK(NextToken(TokenType::LIST_SEPARATOR)); PAIMON_RETURN_NOT_OK(NextToken(TokenType::LITERAL_INT)); const std::string& length_token = GetToken().value; - int64_t length = 0; - const auto [end, error] = - std::from_chars(length_token.data(), length_token.data() + length_token.size(), length); - if (error != std::errc() || end != length_token.data() + length_token.size() || length < 1 || - length > std::numeric_limits::max()) { + std::optional length = StringUtils::StringToValue(length_token); + if (!length || length.value() < 1) { return Status::Invalid( fmt::format("Vector length must be between 1 and {} (both inclusive), but was {}", std::numeric_limits::max(), length_token)); } PAIMON_RETURN_NOT_OK(NextToken(TokenType::END_SUBTYPE)); return arrow::fixed_size_list(arrow::field("item", element_type, element_nullable), - static_cast(length)); + length.value()); } Result TokenParser::ParseOptionalPrecision(int32_t default_precision) { diff --git a/src/paimon/common/types/data_type_json_parser_test.cpp b/src/paimon/common/types/data_type_json_parser_test.cpp index e63f0bf52..e5dfbc21e 100644 --- a/src/paimon/common/types/data_type_json_parser_test.cpp +++ b/src/paimon/common/types/data_type_json_parser_test.cpp @@ -61,14 +61,14 @@ TEST(DataTypeJsonParserTest, ParseVectorTypeSuccess) { DataTypeJsonParser::ParseType("embedding", doc)); ASSERT_FALSE(field->nullable()); ASSERT_EQ(field->type()->id(), arrow::Type::FIXED_SIZE_LIST); - auto vector_type = std::static_pointer_cast(field->type()); + auto vector_type = checked_pointer_cast(field->type()); ASSERT_EQ(vector_type->list_size(), 3); ASSERT_TRUE(vector_type->value_type()->Equals(arrow::float32())); rapidjson::Document sql_doc; rapidjson::Value sql_value("VECTOR", sql_doc.GetAllocator()); ASSERT_OK_AND_ASSIGN(field, DataTypeJsonParser::ParseType("embedding", sql_value)); - vector_type = std::static_pointer_cast(field->type()); + vector_type = checked_pointer_cast(field->type()); ASSERT_TRUE(field->nullable()); ASSERT_EQ(vector_type->list_size(), 5); ASSERT_FALSE(vector_type->value_field()->nullable()); diff --git a/src/paimon/common/types/vector_type.h b/src/paimon/common/types/vector_type.h index 3dd68ec74..9c55c165b 100644 --- a/src/paimon/common/types/vector_type.h +++ b/src/paimon/common/types/vector_type.h @@ -24,6 +24,7 @@ #include "arrow/api.h" #include "paimon/common/types/data_type.h" +#include "paimon/common/utils/checked_cast.h" #include "paimon/common/utils/rapidjson_util.h" namespace paimon { @@ -59,7 +60,7 @@ class VectorType : public DataType { rapidjson::StringRef("type"), RapidJsonUtil::SerializeValue(WithNullable(std::string(TYPE)), allocator).Move(), *allocator); - auto* type = static_cast(type_.get()); + auto* type = checked_cast(type_.get()); auto value_field = type->value_field(); std::shared_ptr data_type = DataType::Create(value_field->type(), value_field->nullable(), value_field->metadata()); diff --git a/src/paimon/common/utils/arrow/arrow_utils.cpp b/src/paimon/common/utils/arrow/arrow_utils.cpp index 4ac13348a..f8c65af18 100644 --- a/src/paimon/common/utils/arrow/arrow_utils.cpp +++ b/src/paimon/common/utils/arrow/arrow_utils.cpp @@ -321,7 +321,7 @@ void ArrowUtils::TraverseArray(const std::shared_ptr& array) { return; } case arrow::Type::type::FIXED_SIZE_LIST: { - auto* vector_array = static_cast(array.get()); + auto* vector_array = checked_cast(array.get()); TraverseArray(vector_array->values()); return; } @@ -336,8 +336,8 @@ bool ArrowUtils::EqualsIgnoreNullable(const std::shared_ptr& ty return false; } if (type->id() == arrow::Type::FIXED_SIZE_LIST) { - const auto& vector_type = static_cast(*type); - const auto& other_vector_type = static_cast(*other_type); + const auto& vector_type = checked_cast(*type); + const auto& other_vector_type = checked_cast(*other_type); if (vector_type.list_size() != other_vector_type.list_size()) { return false; } @@ -376,8 +376,8 @@ Status ArrowUtils::InnerCheckNullabilityMatch(const std::shared_ptrvalue_field(), list_array->values())); } else if (type->id() == arrow::Type::FIXED_SIZE_LIST) { - auto vector_type = std::static_pointer_cast(field->type()); - auto vector_array = std::static_pointer_cast(data); + auto vector_type = checked_pointer_cast(field->type()); + auto vector_array = checked_pointer_cast(data); const std::shared_ptr& values = vector_array->values(); if (values->null_count() != 0) { int32_t vector_length = vector_type->list_size(); diff --git a/src/paimon/core/io/vector_file_batch_reader.cpp b/src/paimon/core/io/vector_file_batch_reader.cpp new file mode 100644 index 000000000..03664bcfd --- /dev/null +++ b/src/paimon/core/io/vector_file_batch_reader.cpp @@ -0,0 +1,325 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "paimon/core/io/vector_file_batch_reader.h" + +#include +#include +#include +#include +#include + +#include "arrow/array.h" +#include "arrow/array/array_nested.h" +#include "arrow/array/builder_primitive.h" +#include "arrow/c/abi.h" +#include "arrow/c/bridge.h" +#include "arrow/compute/api.h" +#include "arrow/type.h" +#include "fmt/format.h" +#include "paimon/common/utils/arrow/mem_utils.h" +#include "paimon/common/utils/arrow/status_utils.h" +#include "paimon/common/utils/checked_cast.h" +#include "paimon/status.h" + +namespace paimon { +namespace { + +bool ContainsVectorType(const std::shared_ptr& type) { + if (type->id() == arrow::Type::FIXED_SIZE_LIST) { + return true; + } + for (const auto& field : type->fields()) { + if (ContainsVectorType(field->type())) { + return true; + } + } + return false; +} + +std::shared_ptr GetPhysicalReadType( + const std::shared_ptr& logical_type) { + switch (logical_type->id()) { + case arrow::Type::FIXED_SIZE_LIST: { + const auto& vector_type = checked_cast(*logical_type); + return arrow::list( + vector_type.value_field()->WithType(GetPhysicalReadType(vector_type.value_type()))); + } + case arrow::Type::STRUCT: { + arrow::FieldVector fields; + fields.reserve(logical_type->num_fields()); + for (const auto& field : logical_type->fields()) { + fields.push_back(field->WithType(GetPhysicalReadType(field->type()))); + } + return arrow::struct_(fields); + } + case arrow::Type::LIST: + return arrow::list(logical_type->field(0)->WithType( + GetPhysicalReadType(logical_type->field(0)->type()))); + case arrow::Type::MAP: { + const auto& map_type = checked_cast(*logical_type); + return std::make_shared( + map_type.key_field()->WithType(GetPhysicalReadType(map_type.key_type())), + map_type.item_field()->WithType(GetPhysicalReadType(map_type.item_type())), + map_type.keys_sorted()); + } + default: + return logical_type; + } +} + +Status ValidateVectorElements(const arrow::FixedSizeListArray& array, int32_t vector_length) { + const std::shared_ptr& values = array.values(); + if (values->null_count() == 0) { + return Status::OK(); + } + for (int64_t i = 0; i < array.length(); ++i) { + if (array.IsNull(i)) { + continue; + } + int64_t value_offset = (array.offset() + i) * vector_length; + for (int32_t j = 0; j < vector_length; ++j) { + if (values->IsNull(value_offset + j)) { + return Status::Invalid("VECTOR cannot contain null elements"); + } + } + } + return Status::OK(); +} + +Result GetIndexCapacity(int64_t row_count, int32_t vector_length) { + if (vector_length < 1) { + return Status::Invalid("VECTOR length must be positive"); + } + if (row_count > std::numeric_limits::max() / vector_length) { + return Status::Invalid("VECTOR values exceed the supported Arrow array length"); + } + return row_count * vector_length; +} + +Result> ConvertListToVector( + const std::shared_ptr& array, + const std::shared_ptr& read_type, arrow::MemoryPool* pool) { + int32_t vector_length = read_type->list_size(); + if (array->type()->id() == arrow::Type::FIXED_SIZE_LIST) { + const auto& source_type = checked_cast(*array->type()); + if (source_type.list_size() != vector_length || + !source_type.value_type()->Equals(read_type->value_type())) { + return Status::Invalid(fmt::format("VECTOR type mismatch: data {} vs read {}", + array->type()->ToString(), read_type->ToString())); + } + const auto& vector_array = checked_cast(*array); + PAIMON_RETURN_NOT_OK(ValidateVectorElements(vector_array, vector_length)); + std::shared_ptr data = array->data()->Copy(); + data->type = read_type; + return arrow::MakeArray(data); + } + if (array->type()->id() != arrow::Type::LIST) { + return Status::Invalid( + fmt::format("Cannot restore VECTOR from type {}", array->type()->ToString())); + } + + const auto& list_array = checked_cast(*array); + if (!list_array.value_type()->Equals(read_type->value_type())) { + return Status::Invalid(fmt::format("VECTOR element type mismatch: data {} vs read {}", + list_array.value_type()->ToString(), + read_type->value_type()->ToString())); + } + + arrow::Int64Builder indices_builder(pool); + PAIMON_ASSIGN_OR_RAISE(int64_t index_capacity, + GetIndexCapacity(list_array.length(), vector_length)); + PAIMON_RETURN_NOT_OK_FROM_ARROW(indices_builder.Reserve(index_capacity)); + arrow::BooleanBuilder validity_builder(pool); + PAIMON_RETURN_NOT_OK_FROM_ARROW(validity_builder.Reserve(list_array.length())); + + for (int64_t i = 0; i < list_array.length(); ++i) { + bool valid = !list_array.IsNull(i); + PAIMON_RETURN_NOT_OK_FROM_ARROW(validity_builder.Append(valid)); + if (!valid) { + PAIMON_RETURN_NOT_OK_FROM_ARROW(indices_builder.AppendNulls(vector_length)); + continue; + } + int64_t value_length = list_array.value_length(i); + if (value_length != vector_length) { + return Status::Invalid( + fmt::format("Vector length mismatch at row {}: expected {} but got {}", i, + vector_length, value_length)); + } + int64_t value_offset = list_array.value_offset(i); + for (int32_t j = 0; j < vector_length; ++j) { + int64_t index = value_offset + j; + if (list_array.values()->IsNull(index)) { + return Status::Invalid(fmt::format( + "VECTOR cannot contain null elements, found one at row {} position {}", i, j)); + } + PAIMON_RETURN_NOT_OK_FROM_ARROW(indices_builder.Append(index)); + } + } + + std::shared_ptr indices; + PAIMON_RETURN_NOT_OK_FROM_ARROW(indices_builder.Finish(&indices)); + arrow::compute::ExecContext exec_context(pool); + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW( + arrow::Datum values, + arrow::compute::Take(arrow::Datum(list_array.values()), arrow::Datum(indices), + arrow::compute::TakeOptions::NoBoundsCheck(), &exec_context)); + + std::shared_ptr validity; + PAIMON_RETURN_NOT_OK_FROM_ARROW(validity_builder.Finish(&validity)); + std::shared_ptr null_bitmap; + if (list_array.null_count() != 0) { + null_bitmap = validity->data()->buffers[1]; + } + std::shared_ptr data = + arrow::ArrayData::Make(read_type, list_array.length(), {null_bitmap}, + {values.make_array()->data()}, list_array.null_count()); + std::shared_ptr result = arrow::MakeArray(data); + PAIMON_RETURN_NOT_OK_FROM_ARROW(result->ValidateFull()); + return result; +} + +std::shared_ptr RebuildNestedType( + const std::shared_ptr& read_type, + const std::vector>& children) { + if (read_type->id() == arrow::Type::STRUCT) { + arrow::FieldVector fields; + fields.reserve(children.size()); + for (int32_t i = 0; i < static_cast(children.size()); ++i) { + fields.push_back(read_type->field(i)->WithType(children[i]->type)); + } + return arrow::struct_(fields); + } + if (read_type->id() == arrow::Type::LIST) { + return arrow::list(read_type->field(0)->WithType(children[0]->type)); + } + + const auto& entries_type = checked_cast(*children[0]->type); + const auto& map_type = checked_cast(*read_type); + return std::make_shared(entries_type.field(0), entries_type.field(1), + map_type.keys_sorted()); +} + +Result> ConvertToReadType( + const std::shared_ptr& array, const std::shared_ptr& read_type, + arrow::MemoryPool* pool) { + if (!ContainsVectorType(read_type)) { + return array; + } + switch (read_type->id()) { + case arrow::Type::FIXED_SIZE_LIST: + return ConvertListToVector( + array, checked_pointer_cast(read_type), pool); + case arrow::Type::STRUCT: + case arrow::Type::LIST: + case arrow::Type::MAP: { + if (array->type()->id() != read_type->id()) { + return Status::Invalid(fmt::format("Cannot reconcile file type {} with {}", + array->type()->ToString(), + read_type->ToString())); + } + if (array->type()->num_fields() != read_type->num_fields()) { + return Status::Invalid(fmt::format("Nested type field count mismatch: {} vs {}", + array->type()->ToString(), + read_type->ToString())); + } + std::vector> children; + children.reserve(read_type->num_fields()); + for (int32_t i = 0; i < read_type->num_fields(); ++i) { + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr child, + ConvertToReadType(arrow::MakeArray(array->data()->child_data[i]), + read_type->field(i)->type(), pool)); + children.push_back(child->data()); + } + std::shared_ptr data = array->data()->Copy(); + data->child_data = std::move(children); + data->type = RebuildNestedType(read_type, data->child_data); + return arrow::MakeArray(data); + } + default: + return array; + } +} + +} // namespace + +VectorFileBatchReader::VectorFileBatchReader(std::unique_ptr&& reader, + const std::shared_ptr& pool) + : arrow_pool_(GetArrowPool(pool)), reader_(std::move(reader)) {} + +bool VectorFileBatchReader::ContainsVector(const std::shared_ptr& schema) { + for (const auto& field : schema->fields()) { + if (ContainsVectorType(field->type())) { + return true; + } + } + return false; +} + +Status VectorFileBatchReader::SetReadSchema( + ::ArrowSchema* read_schema, const std::shared_ptr& predicate, + const std::optional& selection_bitmap) { + if (!read_schema) { + return Status::Invalid("SetReadSchema failed: read schema cannot be nullptr"); + } + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr logical_schema, + arrow::ImportSchema(read_schema)); + arrow::FieldVector physical_fields; + physical_fields.reserve(logical_schema->num_fields()); + for (const auto& field : logical_schema->fields()) { + physical_fields.push_back(field->WithType(GetPhysicalReadType(field->type()))); + } + std::shared_ptr physical_schema = + arrow::schema(physical_fields, logical_schema->metadata()); + ArrowSchema c_physical_schema; + PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportSchema(*physical_schema, &c_physical_schema)); + PAIMON_RETURN_NOT_OK(reader_->SetReadSchema(&c_physical_schema, predicate, selection_bitmap)); + read_type_ = arrow::struct_(logical_schema->fields()); + return Status::OK(); +} + +Result VectorFileBatchReader::ConvertBatch(ReadBatch&& batch) const { + if (BatchReader::IsEofBatch(batch) || !read_type_) { + return std::move(batch); + } + auto& [c_array, c_schema] = batch; + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr array, + arrow::ImportArray(c_array.get(), c_schema.get())); + PAIMON_ASSIGN_OR_RAISE(array, ConvertToReadType(array, read_type_, arrow_pool_.get())); + PAIMON_RETURN_NOT_OK_FROM_ARROW(array->Validate()); + PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportArray(*array, c_array.get(), c_schema.get())); + return std::move(batch); +} + +Result VectorFileBatchReader::NextBatch() { + PAIMON_ASSIGN_OR_RAISE(ReadBatch batch, reader_->NextBatch()); + return ConvertBatch(std::move(batch)); +} + +Result VectorFileBatchReader::NextBatchWithBitmap() { + PAIMON_ASSIGN_OR_RAISE(ReadBatchWithBitmap batch_with_bitmap, reader_->NextBatchWithBitmap()); + if (BatchReader::IsEofBatch(batch_with_bitmap)) { + return std::move(batch_with_bitmap); + } + PAIMON_ASSIGN_OR_RAISE(ReadBatch batch, ConvertBatch(std::move(batch_with_bitmap.first))); + batch_with_bitmap.first = std::move(batch); + return std::move(batch_with_bitmap); +} + +} // namespace paimon diff --git a/src/paimon/core/io/vector_file_batch_reader.h b/src/paimon/core/io/vector_file_batch_reader.h new file mode 100644 index 000000000..79ad5e066 --- /dev/null +++ b/src/paimon/core/io/vector_file_batch_reader.h @@ -0,0 +1,84 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include +#include + +#include "paimon/reader/file_batch_reader.h" + +namespace arrow { +class DataType; +class MemoryPool; +class Schema; +} // namespace arrow + +namespace paimon { +class MemoryPool; + +/// Reconciles logical VECTOR values with the variable-length LIST representation exposed to file +/// format plugins. +class VectorFileBatchReader : public FileBatchReader { + public: + VectorFileBatchReader(std::unique_ptr&& reader, + const std::shared_ptr& pool); + + static bool ContainsVector(const std::shared_ptr& schema); + + Result> GetFileSchema() const override { + return reader_->GetFileSchema(); + } + + Status SetReadSchema(::ArrowSchema* read_schema, const std::shared_ptr& predicate, + const std::optional& selection_bitmap) override; + + Result NextBatch() override; + + Result NextBatchWithBitmap() override; + + std::shared_ptr GetReaderMetrics() const override { + return reader_->GetReaderMetrics(); + } + + void Close() override { + reader_->Close(); + } + + Result GetPreviousBatchFileRowId(uint64_t batch_row_id) const override { + return reader_->GetPreviousBatchFileRowId(batch_row_id); + } + + Result GetNumberOfRows() const override { + return reader_->GetNumberOfRows(); + } + + bool SupportPreciseBitmapSelection() const override { + return reader_->SupportPreciseBitmapSelection(); + } + + private: + Result ConvertBatch(ReadBatch&& batch) const; + + std::shared_ptr arrow_pool_; + std::shared_ptr read_type_; + std::unique_ptr reader_; +}; + +} // namespace paimon diff --git a/src/paimon/core/io/vector_file_batch_reader_test.cpp b/src/paimon/core/io/vector_file_batch_reader_test.cpp new file mode 100644 index 000000000..b93bb9f4f --- /dev/null +++ b/src/paimon/core/io/vector_file_batch_reader_test.cpp @@ -0,0 +1,145 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "paimon/core/io/vector_file_batch_reader.h" + +#include +#include +#include +#include + +#include "arrow/api.h" +#include "arrow/c/bridge.h" +#include "arrow/ipc/json_simple.h" +#include "gtest/gtest.h" +#include "paimon/common/utils/checked_cast.h" +#include "paimon/memory/memory_pool.h" +#include "paimon/testing/mock/mock_file_batch_reader.h" +#include "paimon/testing/utils/testharness.h" + +namespace paimon::test { +namespace { + +std::shared_ptr AsStructType(const std::shared_ptr& type) { + return checked_pointer_cast(type); +} + +} // namespace + +TEST(VectorFileBatchReaderTest, ConvertSchemaAndNextBatch) { + auto physical_type = AsStructType(arrow::struct_({ + arrow::field("id", arrow::int32()), + arrow::field("embedding", arrow::list(arrow::float32())), + })); + auto logical_type = AsStructType(arrow::struct_({ + arrow::field("id", arrow::int32()), + arrow::field("embedding", arrow::fixed_size_list(arrow::float32(), 3)), + })); + const std::string json = R"([ + [1, [1.0, 2.0, 3.0]], + [2, null], + [3, [4.0, 5.0, 6.0]] + ])"; + auto physical_array = + arrow::ipc::internal::json::ArrayFromJSON(physical_type, json).ValueOrDie(); + auto mock_reader = + std::make_unique(physical_array, physical_type, /*batch_size=*/10); + mock_reader->EnableRandomizeBatchSize(false); + MockFileBatchReader* inner_reader = mock_reader.get(); + VectorFileBatchReader reader(std::move(mock_reader), GetDefaultPool()); + + ASSERT_TRUE(VectorFileBatchReader::ContainsVector(arrow::schema(logical_type->fields()))); + ASSERT_FALSE(VectorFileBatchReader::ContainsVector(arrow::schema(physical_type->fields()))); + ArrowSchema c_read_schema; + ASSERT_TRUE(arrow::ExportSchema(*arrow::schema(logical_type->fields()), &c_read_schema).ok()); + ASSERT_OK(reader.SetReadSchema(&c_read_schema, /*predicate=*/nullptr, + /*selection_bitmap=*/std::nullopt)); + ASSERT_EQ(inner_reader->read_schema_->field(1)->type()->id(), arrow::Type::LIST); + + ASSERT_OK_AND_ASSIGN(BatchReader::ReadBatch batch, reader.NextBatch()); + arrow::Result> actual_result = + arrow::ImportArray(batch.first.get(), batch.second.get()); + ASSERT_TRUE(actual_result.ok()) << actual_result.status().ToString(); + std::shared_ptr actual = std::move(actual_result).ValueOrDie(); + auto expected = arrow::ipc::internal::json::ArrayFromJSON(logical_type, json).ValueOrDie(); + ASSERT_TRUE(expected->Equals(actual)) << actual->ToString(); + ASSERT_OK_AND_ASSIGN(batch, reader.NextBatch()); + ASSERT_TRUE(BatchReader::IsEofBatch(batch)); +} + +TEST(VectorFileBatchReaderTest, ConvertNestedVectorsWithBitmap) { + auto logical_vector = + arrow::fixed_size_list(arrow::field("item", arrow::float64(), /*nullable=*/false), 2); + auto physical_vector = arrow::list(arrow::field("item", arrow::float64(), /*nullable=*/false)); + auto logical_type = AsStructType(arrow::struct_({ + arrow::field("vectors", arrow::list(logical_vector)), + arrow::field("by_name", arrow::map(arrow::utf8(), logical_vector)), + })); + auto physical_type = AsStructType(arrow::struct_({ + arrow::field("vectors", arrow::list(physical_vector)), + arrow::field("by_name", arrow::map(arrow::utf8(), physical_vector)), + })); + const std::string json = R"([[[[1.0, 2.0], null], [["a", [3.0, 4.0]]]], + [null, [["b", null]]]])"; + auto physical_array = + arrow::ipc::internal::json::ArrayFromJSON(physical_type, json).ValueOrDie(); + RoaringBitmap32 bitmap; + bitmap.Add(1); + auto mock_reader = std::make_unique(physical_array, physical_type, bitmap, + /*read_batch_size=*/10); + mock_reader->EnableRandomizeBatchSize(false); + VectorFileBatchReader reader(std::move(mock_reader), GetDefaultPool()); + ArrowSchema c_read_schema; + ASSERT_TRUE(arrow::ExportSchema(*arrow::schema(logical_type->fields()), &c_read_schema).ok()); + ASSERT_OK(reader.SetReadSchema(&c_read_schema, /*predicate=*/nullptr, + /*selection_bitmap=*/std::nullopt)); + + ASSERT_OK_AND_ASSIGN(BatchReader::ReadBatchWithBitmap batch_with_bitmap, + reader.NextBatchWithBitmap()); + ASSERT_FALSE(batch_with_bitmap.second.Contains(0)); + ASSERT_TRUE(batch_with_bitmap.second.Contains(1)); + arrow::Result> actual_result = arrow::ImportArray( + batch_with_bitmap.first.first.get(), batch_with_bitmap.first.second.get()); + ASSERT_TRUE(actual_result.ok()) << actual_result.status().ToString(); + std::shared_ptr actual = std::move(actual_result).ValueOrDie(); + auto expected = arrow::ipc::internal::json::ArrayFromJSON(logical_type, json).ValueOrDie(); + ASSERT_TRUE(expected->Equals(actual)) << actual->ToString(); +} + +TEST(VectorFileBatchReaderTest, RejectInvalidVectorValues) { + auto physical_type = + AsStructType(arrow::struct_({arrow::field("embedding", arrow::list(arrow::float32()))})); + auto logical_type = AsStructType( + arrow::struct_({arrow::field("embedding", arrow::fixed_size_list(arrow::float32(), 3))})); + for (const char* json : {R"([[[1.0, 2.0]]])", R"([[[1.0, null, 3.0]]])"}) { + auto physical_array = + arrow::ipc::internal::json::ArrayFromJSON(physical_type, json).ValueOrDie(); + auto mock_reader = std::make_unique(physical_array, physical_type, + /*read_batch_size=*/10); + mock_reader->EnableRandomizeBatchSize(false); + VectorFileBatchReader reader(std::move(mock_reader), GetDefaultPool()); + ArrowSchema c_read_schema; + ASSERT_TRUE( + arrow::ExportSchema(*arrow::schema(logical_type->fields()), &c_read_schema).ok()); + ASSERT_OK(reader.SetReadSchema(&c_read_schema, /*predicate=*/nullptr, + /*selection_bitmap=*/std::nullopt)); + ASSERT_NOK(reader.NextBatch()); + } +} + +} // namespace paimon::test diff --git a/src/paimon/core/operation/abstract_split_read.cpp b/src/paimon/core/operation/abstract_split_read.cpp index 2a3d9e10d..8052b8af2 100644 --- a/src/paimon/core/operation/abstract_split_read.cpp +++ b/src/paimon/core/operation/abstract_split_read.cpp @@ -43,6 +43,7 @@ #include "paimon/core/io/data_file_meta.h" #include "paimon/core/io/data_file_path_factory.h" #include "paimon/core/io/field_mapping_reader.h" +#include "paimon/core/io/vector_file_batch_reader.h" #include "paimon/core/operation/internal_read_context.h" #include "paimon/core/partition/partition_info.h" #include "paimon/core/schema/table_schema.h" @@ -208,6 +209,9 @@ Result> AbstractSplitRead::CreateFieldMappingRe PAIMON_ASSIGN_OR_RAISE(std::unique_ptr file_reader, CreateFileBatchReader(file_format_identifier, data_file_path, file_meta->file_size, reader_builder)); + if (VectorFileBatchReader::ContainsVector(read_schema)) { + file_reader = std::make_unique(std::move(file_reader), pool_); + } std::set skip_map_selected_keys_filter_field_ids; if (file_format_identifier != "blob") { std::pair, std::set> shared_shredding_result; diff --git a/src/paimon/core/operation/data_evolution_split_read.h b/src/paimon/core/operation/data_evolution_split_read.h index 983ca29d7..9d3fa126f 100644 --- a/src/paimon/core/operation/data_evolution_split_read.h +++ b/src/paimon/core/operation/data_evolution_split_read.h @@ -65,7 +65,7 @@ struct DeletionFile; /// ->FieldMappingReader->(ApplyDeletionVectorBatchReader)->(ApplyBitmapIndexBatchReader) /// ->(CompleteRowTrackingFieldsBatchReader)->(ShreddingFileReader) /// ->(MapSharedShreddingFileReader) -/// ->(DelegatingPrefetchReader)->(PrefetchFileBatchReader)->FormatReader +/// ->(VectorFileBatchReader)->(DelegatingPrefetchReader)->(PrefetchFileBatchReader)->FormatReader /// /// /// A union `SplitRead` to read multiple inner files to merge columns, note that this class diff --git a/src/paimon/core/operation/raw_file_split_read.h b/src/paimon/core/operation/raw_file_split_read.h index 1580cd848..2209992d4 100644 --- a/src/paimon/core/operation/raw_file_split_read.h +++ b/src/paimon/core/operation/raw_file_split_read.h @@ -55,8 +55,8 @@ struct DeletionFile; /// splits)->CompleteRowKindBatchReader->(PredicateBatchReader) /// ->ConcatBatchReader across /// files->FieldMappingReader->(ApplyBitmapIndexBatchReader)->(CompleteRowTrackingFieldsBatchReader) -/// ->(ShreddingFileReader)->(MapSharedShreddingFileReader)->(DelegatingPrefetchReader) -/// ->(PrefetchFileBatchReader)->FormatReader +/// ->(ShreddingFileReader)->(MapSharedShreddingFileReader)->(VectorFileBatchReader) +/// ->(DelegatingPrefetchReader)->(PrefetchFileBatchReader)->FormatReader class RawFileSplitRead : public AbstractSplitRead { public: diff --git a/src/paimon/core/schema/arrow_schema_validator.cpp b/src/paimon/core/schema/arrow_schema_validator.cpp index c94f88083..0db1145f7 100644 --- a/src/paimon/core/schema/arrow_schema_validator.cpp +++ b/src/paimon/core/schema/arrow_schema_validator.cpp @@ -214,8 +214,7 @@ Status ArrowSchemaValidator::ValidateField(const std::shared_ptr& break; } case arrow::Type::type::FIXED_SIZE_LIST: { - const auto& vector_type = - checked_cast(*field->type()); + const auto& vector_type = checked_cast(*field->type()); if (vector_type.list_size() < 1) { return Status::Invalid("Vector length must be positive, but was ", vector_type.list_size()); diff --git a/src/paimon/core/schema/schema_validation.cpp b/src/paimon/core/schema/schema_validation.cpp index 31e72cdb2..795b8d205 100644 --- a/src/paimon/core/schema/schema_validation.cpp +++ b/src/paimon/core/schema/schema_validation.cpp @@ -685,6 +685,10 @@ Status SchemaValidation::ValidateVectorFields(const TableSchema& schema, if (!has_vector) { return Status::OK(); } + if (!schema.PrimaryKeys().empty()) { + return Status::NotImplemented( + "VECTOR fields in primary-key tables are not implemented yet."); + } PAIMON_RETURN_NOT_OK( ValidateVectorFileFormat(Options::FILE_FORMAT, options.GetFileFormat()->Identifier())); return ValidatePerLevelOption(options.ToMap(), Options::FILE_FORMAT_PER_LEVEL, diff --git a/src/paimon/core/schema/schema_validation_test.cpp b/src/paimon/core/schema/schema_validation_test.cpp index 73b2702b6..10d0ffacd 100644 --- a/src/paimon/core/schema/schema_validation_test.cpp +++ b/src/paimon/core/schema/schema_validation_test.cpp @@ -70,6 +70,24 @@ TEST(SchemaValidationTest, TestVectorType) { /*primary_keys=*/{"embedding"}, primary_key_options)); ASSERT_NOK_WITH_MSG(SchemaValidation::ValidateTableSchema(*table_schema), "in primary key field embedding is unsupported"); + + primary_key_options[Options::FILE_FORMAT] = "parquet"; + ASSERT_OK_AND_ASSIGN(table_schema, + TableSchema::Create(/*schema_id=*/0, schema, /*partition_keys=*/{}, + /*primary_keys=*/{"id"}, primary_key_options)); + ASSERT_NOK_WITH_MSG(SchemaValidation::ValidateTableSchema(*table_schema), + "VECTOR fields in primary-key tables are not implemented yet."); + + auto nested_schema = arrow::schema({ + arrow::field("id", arrow::int64()), + arrow::field("payload", arrow::struct_({arrow::field("embedding", vector_field->type())})), + }); + ASSERT_OK_AND_ASSIGN( + table_schema, + TableSchema::Create(/*schema_id=*/0, nested_schema, + /*partition_keys=*/{}, /*primary_keys=*/{"id"}, primary_key_options)); + ASSERT_NOK_WITH_MSG(SchemaValidation::ValidateTableSchema(*table_schema), + "VECTOR fields in primary-key tables are not implemented yet."); } TEST(SchemaValidationTest, TestRowTracking) { diff --git a/src/paimon/core/schema/table_schema.cpp b/src/paimon/core/schema/table_schema.cpp index 6fcf67148..6e2d8747e 100644 --- a/src/paimon/core/schema/table_schema.cpp +++ b/src/paimon/core/schema/table_schema.cpp @@ -119,7 +119,7 @@ Result> TableSchema::AssignFieldIdsRecursively( return arrow::field(field->name(), arrow::list(new_value_field), field->nullable(), metadata); } else if (type->id() == arrow::Type::FIXED_SIZE_LIST) { - auto vector_type = std::static_pointer_cast(field->type()); + auto vector_type = checked_pointer_cast(field->type()); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr new_value_field, AssignFieldIdsRecursively(vector_type->value_field(), /*set_field_id=*/false, field_id)); diff --git a/src/paimon/format/parquet/parquet_field_id_converter.cpp b/src/paimon/format/parquet/parquet_field_id_converter.cpp index 43a4fadb0..adb271734 100644 --- a/src/paimon/format/parquet/parquet_field_id_converter.cpp +++ b/src/paimon/format/parquet/parquet_field_id_converter.cpp @@ -106,7 +106,7 @@ arrow::Result> ParquetFieldIdConverter::ProcessField( auto new_type = arrow::list(new_value_field); return field->WithType(new_type)->WithMergedMetadata(updated_metadata); } else if (type->id() == arrow::Type::FIXED_SIZE_LIST) { - auto vector_type = std::static_pointer_cast(type); + auto vector_type = checked_pointer_cast(type); ARROW_ASSIGN_OR_RAISE(auto new_value_field, ProcessField(vector_type->value_field(), convert_type)); auto new_type = arrow::fixed_size_list(new_value_field, vector_type->list_size()); diff --git a/src/paimon/format/parquet/parquet_field_id_converter_test.cpp b/src/paimon/format/parquet/parquet_field_id_converter_test.cpp index f4e0ecae3..d2053f120 100644 --- a/src/paimon/format/parquet/parquet_field_id_converter_test.cpp +++ b/src/paimon/format/parquet/parquet_field_id_converter_test.cpp @@ -214,8 +214,8 @@ TEST_F(ParquetFieldIdConverterTest, TestNestedType) { {"sub3", arrow::Type::DECIMAL128, "20"}, {"sub4", arrow::Type::BINARY, "21"}, {"sub5", arrow::Type::BINARY, "22"}, {"f3", arrow::Type::FIXED_SIZE_LIST, "23"}}; ASSERT_EQ(expected_field_infos, field_infos); - auto new_vector = std::static_pointer_cast( - new_schema->GetFieldByName("f3")->type()); + auto new_vector = + checked_pointer_cast(new_schema->GetFieldByName("f3")->type()); ASSERT_EQ(new_vector->list_size(), 7); // convert to paimon.id ASSERT_OK_AND_ASSIGN(auto old_schema, @@ -224,8 +224,8 @@ TEST_F(ParquetFieldIdConverterTest, TestNestedType) { PrintFieldMetadata(old_schema, ParquetFieldIdConverter::IdConvertType::PARQUET_TO_PAIMON_ID, &old_field_infos); ASSERT_EQ(expected_field_infos, old_field_infos); - auto old_vector = std::static_pointer_cast( - old_schema->GetFieldByName("f3")->type()); + auto old_vector = + checked_pointer_cast(old_schema->GetFieldByName("f3")->type()); ASSERT_EQ(old_vector->list_size(), 7); } diff --git a/src/paimon/format/parquet/parquet_file_batch_reader.cpp b/src/paimon/format/parquet/parquet_file_batch_reader.cpp index bfbbe6ee4..c0cd40e19 100644 --- a/src/paimon/format/parquet/parquet_file_batch_reader.cpp +++ b/src/paimon/format/parquet/parquet_file_batch_reader.cpp @@ -49,7 +49,6 @@ #include "paimon/format/parquet/parquet_format_defs.h" #include "paimon/format/parquet/parquet_schema_util.h" #include "paimon/format/parquet/parquet_timestamp_converter.h" -#include "paimon/format/parquet/parquet_vector_converter.h" #include "paimon/format/parquet/predicate_converter.h" #include "paimon/reader/batch_reader.h" #include "paimon/utils/roaring_bitmap32.h" @@ -66,13 +65,6 @@ class Predicate; namespace paimon::parquet { namespace { -std::shared_ptr GetListElementType(const std::shared_ptr& type) { - if (type->id() == arrow::Type::FIXED_SIZE_LIST) { - return static_cast(*type).value_type(); - } - return static_cast(*type).value_type(); -} - // LIST/MAP do not support pruning fields from their nested value types, but physical and // logical leaf types may still differ (for example, Parquet reports LTZ timestamps as UTC // while Paimon exposes them in the local timezone). Compare only the nested projection shape @@ -97,9 +89,7 @@ bool HasSameNestedProjectionShape(const std::shared_ptr& read_t } return read_type->Equals(file_type); } - const bool vector_from_list = - read_type->id() == arrow::Type::FIXED_SIZE_LIST && file_type->id() == arrow::Type::LIST; - if (read_type->id() != file_type->id() && !vector_from_list) { + if (read_type->id() != file_type->id()) { return false; } @@ -119,18 +109,9 @@ bool HasSameNestedProjectionShape(const std::shared_ptr& read_t return true; } case arrow::Type::LIST: { + const auto& read_list = static_cast(*read_type); const auto& file_list = static_cast(*file_type); - return HasSameNestedProjectionShape(GetListElementType(read_type), - file_list.value_type()); - } - case arrow::Type::FIXED_SIZE_LIST: { - if (file_type->id() != arrow::Type::FIXED_SIZE_LIST) { - return false; - } - const auto& read_vector = static_cast(*read_type); - const auto& file_vector = static_cast(*file_type); - return read_vector.list_size() == file_vector.list_size() && - HasSameNestedProjectionShape(read_vector.value_type(), file_vector.value_type()); + return HasSameNestedProjectionShape(read_list.value_type(), file_list.value_type()); } case arrow::Type::MAP: { const auto& read_map = static_cast(*read_type); @@ -616,9 +597,6 @@ Result ParquetFileBatchReader::NextBatch() { PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr array, batch->ToStructArray()); PAIMON_RETURN_NOT_OK_FROM_ARROW(array->Validate()); - PAIMON_ASSIGN_OR_RAISE(array, ParquetVectorConverter::ConvertToReadType( - array, read_data_type_, arrow_pool_.get())); - PAIMON_RETURN_NOT_OK_FROM_ARROW(array->Validate()); PAIMON_ASSIGN_OR_RAISE(bool need_cast, ParquetTimestampConverter::NeedCastArrayForTimestamp( array->type(), read_data_type_)); if (need_cast) { @@ -750,8 +728,7 @@ Status ParquetFileBatchReader::CollectLeafIndices(const std::shared_ptrtype(), leaf_index); } } - } else if (file_type->id() == arrow::Type::LIST || - file_type->id() == arrow::Type::FIXED_SIZE_LIST) { + } else if (file_type->id() == arrow::Type::LIST) { // Keep behavior aligned with ORC path: list/map inner partial projection // is currently unsupported and should fail-fast. if (!HasSameNestedProjectionShape(read_type, file_type)) { @@ -759,8 +736,10 @@ Status ParquetFileBatchReader::CollectLeafIndices(const std::shared_ptrToString(), read_type->ToString())); } - PAIMON_RETURN_NOT_OK(CollectLeafIndices( - GetListElementType(read_type), GetListElementType(file_type), leaf_index, indices)); + const auto& read_list = static_cast(*read_type); + const auto& file_list = static_cast(*file_type); + PAIMON_RETURN_NOT_OK(CollectLeafIndices(read_list.value_type(), file_list.value_type(), + leaf_index, indices)); } else if (file_type->id() == arrow::Type::MAP) { if (!HasSameNestedProjectionShape(read_type, file_type)) { return Status::Invalid(fmt::format( @@ -783,7 +762,7 @@ Status ParquetFileBatchReader::CollectLeafIndices(const std::shared_ptr& file_type, int32_t* leaf_index) { if (file_type->id() == arrow::Type::STRUCT || file_type->id() == arrow::Type::LIST || - file_type->id() == arrow::Type::FIXED_SIZE_LIST || file_type->id() == arrow::Type::MAP) { + file_type->id() == arrow::Type::MAP) { for (int32_t i = 0; i < file_type->num_fields(); i++) { SkipLeafIndices(file_type->field(i)->type(), leaf_index); } diff --git a/src/paimon/format/parquet/parquet_file_batch_reader.h b/src/paimon/format/parquet/parquet_file_batch_reader.h index 2c47c2049..7e5e9afab 100644 --- a/src/paimon/format/parquet/parquet_file_batch_reader.h +++ b/src/paimon/format/parquet/parquet_file_batch_reader.h @@ -167,7 +167,7 @@ class ParquetFileBatchReader : public PrefetchFileBatchReader { static void FlattenSchema(const std::shared_ptr& type, int32_t* index, std::vector* index_vector) { if (type->id() == arrow::Type::STRUCT || type->id() == arrow::Type::LIST || - type->id() == arrow::Type::FIXED_SIZE_LIST || type->id() == arrow::Type::MAP) { + type->id() == arrow::Type::MAP) { for (int32_t i = 0; i < type->num_fields(); i++) { auto field = type->field(i); auto inner_type = field->type(); diff --git a/src/paimon/format/parquet/parquet_format_writer.cpp b/src/paimon/format/parquet/parquet_format_writer.cpp index a9866332b..339ed2147 100644 --- a/src/paimon/format/parquet/parquet_format_writer.cpp +++ b/src/paimon/format/parquet/parquet_format_writer.cpp @@ -32,6 +32,7 @@ #include "paimon/common/metrics/metrics_impl.h" #include "paimon/common/utils/arrow/arrow_output_stream_adapter.h" #include "paimon/common/utils/arrow/status_utils.h" +#include "paimon/common/utils/checked_cast.h" #include "paimon/format/parquet/parquet_format_defs.h" #include "paimon/format/parquet/parquet_vector_converter.h" #include "parquet/arrow/writer.h" @@ -58,8 +59,8 @@ Result> ParquetFormatWriter::Create( auto arrow_writer_properties = arrow_properties_builder.enable_deprecated_int96_timestamps()->build(); auto logical_type = arrow::struct_(schema->fields()); - auto write_type = std::static_pointer_cast( - ParquetVectorConverter::GetWriteType(logical_type)); + auto write_type = + checked_pointer_cast(ParquetVectorConverter::GetWriteType(logical_type)); auto write_schema = arrow::schema(write_type->fields(), schema->metadata()); PAIMON_ASSIGN_OR_RAISE_FROM_ARROW( std::unique_ptr<::parquet::arrow::FileWriter> file_writer, @@ -74,6 +75,8 @@ Status ParquetFormatWriter::AddBatch(ArrowArray* batch) { PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr<::arrow::RecordBatch> record_batch, arrow::ImportRecordBatch(batch, schema_)); if (needs_vector_conversion_) { + // TODO(ChaomingZhangCN): Remove this conversion after upgrading Arrow. Arrow 17 + // mishandles nullable FixedSizeList values when writing them as Parquet LIST. PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr struct_array, record_batch->ToStructArray()); std::shared_ptr array = struct_array; diff --git a/src/paimon/format/parquet/parquet_vector_converter.cpp b/src/paimon/format/parquet/parquet_vector_converter.cpp index 205df51f4..00064f52b 100644 --- a/src/paimon/format/parquet/parquet_vector_converter.cpp +++ b/src/paimon/format/parquet/parquet_vector_converter.cpp @@ -28,8 +28,8 @@ #include "arrow/array/builder_primitive.h" #include "arrow/compute/api.h" #include "arrow/type.h" -#include "fmt/format.h" #include "paimon/common/utils/arrow/status_utils.h" +#include "paimon/common/utils/checked_cast.h" #include "paimon/status.h" namespace paimon::parquet { @@ -76,92 +76,10 @@ Result GetIndexCapacity(int64_t row_count, int32_t vector_length) { return row_count * vector_length; } -Result> ConvertListToVector( - const std::shared_ptr& array, - const std::shared_ptr& read_type, arrow::MemoryPool* pool) { - int32_t vector_length = read_type->list_size(); - if (array->type()->id() == arrow::Type::FIXED_SIZE_LIST) { - const auto& source_type = static_cast(*array->type()); - if (source_type.list_size() != vector_length || - !source_type.value_type()->Equals(read_type->value_type())) { - return Status::Invalid(fmt::format("VECTOR type mismatch: data {} vs read {}", - array->type()->ToString(), read_type->ToString())); - } - const auto& vector_array = static_cast(*array); - PAIMON_RETURN_NOT_OK(ValidateVectorElements(vector_array, vector_length)); - std::shared_ptr data = array->data()->Copy(); - data->type = read_type; - return arrow::MakeArray(data); - } - if (array->type()->id() != arrow::Type::LIST) { - return Status::Invalid( - fmt::format("Cannot restore VECTOR from parquet type {}", array->type()->ToString())); - } - - const auto& list_array = static_cast(*array); - if (!list_array.value_type()->Equals(read_type->value_type())) { - return Status::Invalid(fmt::format("VECTOR element type mismatch: data {} vs read {}", - list_array.value_type()->ToString(), - read_type->value_type()->ToString())); - } - - arrow::Int64Builder indices_builder(pool); - PAIMON_ASSIGN_OR_RAISE(int64_t index_capacity, - GetIndexCapacity(list_array.length(), vector_length)); - PAIMON_RETURN_NOT_OK_FROM_ARROW(indices_builder.Reserve(index_capacity)); - arrow::BooleanBuilder validity_builder(pool); - PAIMON_RETURN_NOT_OK_FROM_ARROW(validity_builder.Reserve(list_array.length())); - - for (int64_t i = 0; i < list_array.length(); ++i) { - bool valid = !list_array.IsNull(i); - PAIMON_RETURN_NOT_OK_FROM_ARROW(validity_builder.Append(valid)); - if (!valid) { - PAIMON_RETURN_NOT_OK_FROM_ARROW(indices_builder.AppendNulls(vector_length)); - continue; - } - int64_t value_length = list_array.value_length(i); - if (value_length != vector_length) { - return Status::Invalid( - fmt::format("Vector length mismatch at row {}: expected {} but got {}", i, - vector_length, value_length)); - } - int64_t value_offset = list_array.value_offset(i); - for (int32_t j = 0; j < vector_length; ++j) { - int64_t index = value_offset + j; - if (list_array.values()->IsNull(index)) { - return Status::Invalid(fmt::format( - "VECTOR cannot contain null elements, found one at row {} position {}", i, j)); - } - PAIMON_RETURN_NOT_OK_FROM_ARROW(indices_builder.Append(index)); - } - } - - std::shared_ptr indices; - PAIMON_RETURN_NOT_OK_FROM_ARROW(indices_builder.Finish(&indices)); - arrow::compute::ExecContext exec_context(pool); - PAIMON_ASSIGN_OR_RAISE_FROM_ARROW( - arrow::Datum values, - arrow::compute::Take(arrow::Datum(list_array.values()), arrow::Datum(indices), - arrow::compute::TakeOptions::NoBoundsCheck(), &exec_context)); - - std::shared_ptr validity; - PAIMON_RETURN_NOT_OK_FROM_ARROW(validity_builder.Finish(&validity)); - std::shared_ptr null_bitmap; - if (list_array.null_count() != 0) { - null_bitmap = validity->data()->buffers[1]; - } - std::shared_ptr data = - arrow::ArrayData::Make(read_type, list_array.length(), {null_bitmap}, - {values.make_array()->data()}, list_array.null_count()); - std::shared_ptr result = arrow::MakeArray(data); - PAIMON_RETURN_NOT_OK_FROM_ARROW(result->ValidateFull()); - return result; -} - Result> ConvertVectorToList( const std::shared_ptr& array, arrow::MemoryPool* pool) { - const auto& vector_array = static_cast(*array); - const auto& vector_type = static_cast(*array->type()); + const auto& vector_array = checked_cast(*array); + const auto& vector_type = checked_cast(*array->type()); int32_t vector_length = vector_type.list_size(); PAIMON_RETURN_NOT_OK(ValidateVectorElements(vector_array, vector_length)); @@ -215,34 +133,13 @@ Result> ConvertVectorToList( null_bitmap, vector_array.null_count()); } -std::shared_ptr RebuildNestedType( - const std::shared_ptr& read_type, - const std::vector>& children) { - if (read_type->id() == arrow::Type::STRUCT) { - arrow::FieldVector fields; - fields.reserve(children.size()); - for (int32_t i = 0; i < static_cast(children.size()); ++i) { - fields.push_back(read_type->field(i)->WithType(children[i]->type)); - } - return arrow::struct_(fields); - } - if (read_type->id() == arrow::Type::LIST) { - return arrow::list(read_type->field(0)->WithType(children[0]->type)); - } - - const auto& entries_type = static_cast(*children[0]->type); - const auto& map_type = static_cast(*read_type); - return std::make_shared(entries_type.field(0), entries_type.field(1), - map_type.keys_sorted()); -} - } // namespace std::shared_ptr ParquetVectorConverter::GetWriteType( const std::shared_ptr& logical_type) { switch (logical_type->id()) { case arrow::Type::FIXED_SIZE_LIST: { - const auto& vector_type = static_cast(*logical_type); + const auto& vector_type = checked_cast(*logical_type); return arrow::list( vector_type.value_field()->WithType(GetWriteType(vector_type.value_type()))); } @@ -258,7 +155,7 @@ std::shared_ptr ParquetVectorConverter::GetWriteType( return arrow::list( logical_type->field(0)->WithType(GetWriteType(logical_type->field(0)->type()))); case arrow::Type::MAP: { - const auto& map_type = static_cast(*logical_type); + const auto& map_type = checked_cast(*logical_type); return std::make_shared( map_type.key_field()->WithType(GetWriteType(map_type.key_type())), map_type.item_field()->WithType(GetWriteType(map_type.item_type())), @@ -269,48 +166,6 @@ std::shared_ptr ParquetVectorConverter::GetWriteType( } } -Result> ParquetVectorConverter::ConvertToReadType( - const std::shared_ptr& array, const std::shared_ptr& read_type, - arrow::MemoryPool* pool) { - if (!ContainsVectorType(read_type)) { - return array; - } - switch (read_type->id()) { - case arrow::Type::FIXED_SIZE_LIST: - return ConvertListToVector( - array, std::static_pointer_cast(read_type), pool); - case arrow::Type::STRUCT: - case arrow::Type::LIST: - case arrow::Type::MAP: { - if (array->type()->id() != read_type->id()) { - return Status::Invalid(fmt::format("Cannot reconcile parquet type {} with {}", - array->type()->ToString(), - read_type->ToString())); - } - if (array->type()->num_fields() != read_type->num_fields()) { - return Status::Invalid(fmt::format("Nested type field count mismatch: {} vs {}", - array->type()->ToString(), - read_type->ToString())); - } - std::vector> children; - children.reserve(read_type->num_fields()); - for (int32_t i = 0; i < read_type->num_fields(); ++i) { - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr child, - ConvertToReadType(arrow::MakeArray(array->data()->child_data[i]), - read_type->field(i)->type(), pool)); - children.push_back(child->data()); - } - std::shared_ptr data = array->data()->Copy(); - data->child_data = std::move(children); - data->type = RebuildNestedType(read_type, data->child_data); - return arrow::MakeArray(data); - } - default: - return array; - } -} - Result> ParquetVectorConverter::ConvertToWriteType( const std::shared_ptr& array, arrow::MemoryPool* pool) { if (!ContainsVectorType(array->type())) { diff --git a/src/paimon/format/parquet/parquet_vector_converter.h b/src/paimon/format/parquet/parquet_vector_converter.h index 113eda8fb..a265e2d12 100644 --- a/src/paimon/format/parquet/parquet_vector_converter.h +++ b/src/paimon/format/parquet/parquet_vector_converter.h @@ -30,16 +30,12 @@ class DataType; namespace paimon::parquet { -/// Restores logical FixedSizeList VECTOR arrays from Parquet LIST arrays. +/// Converts logical FixedSizeList VECTOR arrays to Parquet LIST arrays. class ParquetVectorConverter { public: ParquetVectorConverter() = delete; ~ParquetVectorConverter() = delete; - static Result> ConvertToReadType( - const std::shared_ptr& array, - const std::shared_ptr& read_type, arrow::MemoryPool* pool); - static Result> ConvertToWriteType( const std::shared_ptr& array, arrow::MemoryPool* pool); diff --git a/src/paimon/format/parquet/parquet_vector_converter_test.cpp b/src/paimon/format/parquet/parquet_vector_converter_test.cpp index 646a3685c..3f2598c64 100644 --- a/src/paimon/format/parquet/parquet_vector_converter_test.cpp +++ b/src/paimon/format/parquet/parquet_vector_converter_test.cpp @@ -23,44 +23,11 @@ #include "arrow/api.h" #include "arrow/ipc/json_simple.h" #include "gtest/gtest.h" +#include "paimon/common/utils/checked_cast.h" #include "paimon/testing/utils/testharness.h" namespace paimon::parquet::test { -TEST(ParquetVectorConverterTest, ConvertListToVector) { - auto physical_type = arrow::list(arrow::float32()); - auto physical_array = arrow::ipc::internal::json::ArrayFromJSON( - physical_type, R"([[1.0, 2.0, 3.0], null, [4.0, 5.0, 6.0]])") - .ValueOrDie(); - auto vector_type = arrow::fixed_size_list(arrow::float32(), 3); - - ASSERT_OK_AND_ASSIGN(std::shared_ptr converted, - ParquetVectorConverter::ConvertToReadType(physical_array, vector_type, - arrow::default_memory_pool())); - ASSERT_EQ(converted->type()->id(), arrow::Type::FIXED_SIZE_LIST); - auto vector_array = std::static_pointer_cast(converted); - ASSERT_EQ(vector_array->length(), 3); - ASSERT_FALSE(vector_array->IsNull(0)); - ASSERT_TRUE(vector_array->IsNull(1)); - ASSERT_FALSE(vector_array->IsNull(2)); - auto values = std::static_pointer_cast(vector_array->values()); - ASSERT_FLOAT_EQ(values->Value(0), 1.0f); - ASSERT_FLOAT_EQ(values->Value(2), 3.0f); - ASSERT_FLOAT_EQ(values->Value(6), 4.0f); - ASSERT_FLOAT_EQ(values->Value(8), 6.0f); -} - -TEST(ParquetVectorConverterTest, RejectInvalidVectorValues) { - auto vector_type = arrow::fixed_size_list(arrow::float32(), 3); - for (const char* json : {R"([[1.0, 2.0]])", R"([[1.0, null, 3.0]])"}) { - auto physical_array = - arrow::ipc::internal::json::ArrayFromJSON(arrow::list(arrow::float32()), json) - .ValueOrDie(); - ASSERT_NOK(ParquetVectorConverter::ConvertToReadType(physical_array, vector_type, - arrow::default_memory_pool())); - } -} - TEST(ParquetVectorConverterTest, ConvertNullableVectorToList) { auto vector_type = arrow::fixed_size_list(arrow::float32(), 3); auto vector_array = arrow::ipc::internal::json::ArrayFromJSON( @@ -71,7 +38,7 @@ TEST(ParquetVectorConverterTest, ConvertNullableVectorToList) { std::shared_ptr converted, ParquetVectorConverter::ConvertToWriteType(vector_array, arrow::default_memory_pool())); ASSERT_EQ(converted->type()->id(), arrow::Type::LIST); - auto list_array = std::static_pointer_cast(converted); + auto list_array = checked_pointer_cast(converted); ASSERT_EQ(list_array->value_length(0), 3); ASSERT_TRUE(list_array->IsNull(1)); ASSERT_EQ(list_array->value_length(1), 0); @@ -79,28 +46,7 @@ TEST(ParquetVectorConverterTest, ConvertNullableVectorToList) { ASSERT_EQ(list_array->values()->length(), 6); } -TEST(ParquetVectorConverterTest, PreserveUnconvertedNestedTypes) { - auto physical_type = arrow::struct_({ - arrow::field("ts", arrow::timestamp(arrow::TimeUnit::MILLI)), - arrow::field("embedding", arrow::list(arrow::float32())), - }); - auto physical_array = arrow::ipc::internal::json::ArrayFromJSON( - physical_type, R"([["1970-01-01 00:00:01.000", [1.0, 2.0, 3.0]]])") - .ValueOrDie(); - auto read_type = arrow::struct_({ - arrow::field("ts", arrow::timestamp(arrow::TimeUnit::SECOND)), - arrow::field("embedding", arrow::fixed_size_list(arrow::float32(), 3)), - }); - - ASSERT_OK_AND_ASSIGN(std::shared_ptr converted, - ParquetVectorConverter::ConvertToReadType(physical_array, read_type, - arrow::default_memory_pool())); - auto converted_type = std::static_pointer_cast(converted->type()); - ASSERT_TRUE(converted_type->field(0)->type()->Equals(arrow::timestamp(arrow::TimeUnit::MILLI))); - ASSERT_EQ(converted_type->field(1)->type()->id(), arrow::Type::FIXED_SIZE_LIST); -} - -TEST(ParquetVectorConverterTest, ConvertVectorNestedInListAndMap) { +TEST(ParquetVectorConverterTest, ConvertNestedVectorsToList) { auto vector_type = arrow::fixed_size_list(arrow::field("item", arrow::float32(), /*nullable=*/false), 2); auto nested_type = arrow::struct_({ @@ -116,16 +62,11 @@ TEST(ParquetVectorConverterTest, ConvertVectorNestedInListAndMap) { ASSERT_OK_AND_ASSIGN( std::shared_ptr physical_array, ParquetVectorConverter::ConvertToWriteType(nested_array, arrow::default_memory_pool())); - auto physical_type = std::static_pointer_cast(physical_array->type()); - auto physical_list = std::static_pointer_cast(physical_type->field(0)->type()); - auto physical_map = std::static_pointer_cast(physical_type->field(1)->type()); + auto physical_type = checked_pointer_cast(physical_array->type()); + auto physical_list = checked_pointer_cast(physical_type->field(0)->type()); + auto physical_map = checked_pointer_cast(physical_type->field(1)->type()); ASSERT_EQ(physical_list->value_type()->id(), arrow::Type::LIST); ASSERT_EQ(physical_map->item_type()->id(), arrow::Type::LIST); - - ASSERT_OK_AND_ASSIGN(std::shared_ptr converted, - ParquetVectorConverter::ConvertToReadType(physical_array, nested_type, - arrow::default_memory_pool())); - ASSERT_TRUE(converted->Equals(nested_array)); } TEST(ParquetVectorConverterTest, ConvertSlicedVectorToList) { @@ -138,11 +79,11 @@ TEST(ParquetVectorConverterTest, ConvertSlicedVectorToList) { ASSERT_OK_AND_ASSIGN( std::shared_ptr converted, ParquetVectorConverter::ConvertToWriteType(vector_array, arrow::default_memory_pool())); - auto list_array = std::static_pointer_cast(converted); + auto list_array = checked_pointer_cast(converted); ASSERT_EQ(list_array->length(), 2); ASSERT_EQ(list_array->value_length(0), 2); ASSERT_TRUE(list_array->IsNull(1)); - auto values = std::static_pointer_cast(list_array->values()); + auto values = checked_pointer_cast(list_array->values()); ASSERT_DOUBLE_EQ(values->Value(0), 3.0); ASSERT_DOUBLE_EQ(values->Value(1), 4.0); } diff --git a/src/paimon/format/parquet/parquet_vector_io_test.cpp b/src/paimon/format/parquet/parquet_vector_io_test.cpp index 5e224bc80..e871e0517 100644 --- a/src/paimon/format/parquet/parquet_vector_io_test.cpp +++ b/src/paimon/format/parquet/parquet_vector_io_test.cpp @@ -29,6 +29,8 @@ #include "gtest/gtest.h" #include "paimon/common/utils/arrow/arrow_input_stream_adapter.h" #include "paimon/common/utils/arrow/mem_utils.h" +#include "paimon/common/utils/checked_cast.h" +#include "paimon/core/io/vector_file_batch_reader.h" #include "paimon/format/parquet/parquet_file_batch_reader.h" #include "paimon/format/parquet/parquet_format_defs.h" #include "paimon/format/parquet/parquet_format_writer.h" @@ -87,19 +89,21 @@ class ParquetVectorIoTest : public ::testing::Test { arrow::ImportType(c_file_schema.get()); ASSERT_TRUE(file_type_result.ok()) << file_type_result.status().ToString(); auto file_type = - std::static_pointer_cast(std::move(file_type_result).ValueOrDie()); + checked_pointer_cast(std::move(file_type_result).ValueOrDie()); std::shared_ptr physical_value_type = file_type->field(1)->type(); if (physical_value_type->id() == arrow::Type::STRUCT) { physical_value_type = physical_value_type->field(0)->type(); } ASSERT_EQ(physical_value_type->id(), arrow::Type::LIST); + std::unique_ptr vector_reader = + std::make_unique(std::move(reader), pool_); auto c_schema = std::make_unique(); ASSERT_TRUE(arrow::ExportSchema(*arrow::schema(read_type->fields()), c_schema.get()).ok()); - ASSERT_OK(reader->SetReadSchema(c_schema.get(), /*predicate=*/nullptr, - /*selection_bitmap=*/std::nullopt)); + ASSERT_OK(vector_reader->SetReadSchema(c_schema.get(), /*predicate=*/nullptr, + /*selection_bitmap=*/std::nullopt)); ASSERT_OK_AND_ASSIGN(std::shared_ptr actual, - paimon::test::ReadResultCollector::CollectResult(reader.get())); + paimon::test::ReadResultCollector::CollectResult(vector_reader.get())); arrow::Result> expected_result = arrow::ipc::internal::json::ArrayFromJSON(read_type, json); @@ -119,17 +123,17 @@ class ParquetVectorIoTest : public ::testing::Test { TEST_F(ParquetVectorIoTest, WriteAndReadVector) { auto vector_type = arrow::fixed_size_list(arrow::field("item", arrow::float32(), /*nullable=*/false), 3); - auto struct_type = std::static_pointer_cast(arrow::struct_( + auto struct_type = checked_pointer_cast(arrow::struct_( {arrow::field("id", arrow::int32()), arrow::field("embedding", vector_type)})); WriteAndCheck("vector.parquet", struct_type, struct_type, R"([[1, [1.0, 2.0, 3.0]], [2, null], [3, [4.0, 5.0, 6.0]]])"); } TEST_F(ParquetVectorIoTest, ReadOrdinaryParquetListAsVector) { - auto physical_type = std::static_pointer_cast( + auto physical_type = checked_pointer_cast( arrow::struct_({arrow::field("id", arrow::int32()), arrow::field("embedding", arrow::list(arrow::float32()))})); - auto logical_type = std::static_pointer_cast(arrow::struct_({ + auto logical_type = checked_pointer_cast(arrow::struct_({ arrow::field("id", arrow::int32()), arrow::field("embedding", arrow::fixed_size_list(arrow::float32(), 3)), })); @@ -140,7 +144,7 @@ TEST_F(ParquetVectorIoTest, ReadOrdinaryParquetListAsVector) { TEST_F(ParquetVectorIoTest, WriteAndReadNestedDoubleVector) { auto vector_type = arrow::fixed_size_list(arrow::field("item", arrow::float64(), /*nullable=*/false), 2); - auto struct_type = std::static_pointer_cast(arrow::struct_({ + auto struct_type = checked_pointer_cast(arrow::struct_({ arrow::field("id", arrow::int32()), arrow::field("payload", arrow::struct_({arrow::field("embedding", vector_type), arrow::field("history", arrow::list(vector_type)), diff --git a/test/inte/write_and_read_inte_test.cpp b/test/inte/write_and_read_inte_test.cpp index 165d56af4..57da416cd 100644 --- a/test/inte/write_and_read_inte_test.cpp +++ b/test/inte/write_and_read_inte_test.cpp @@ -331,20 +331,13 @@ TEST_P(WriteAndReadInteTest, TestAppendVector) { TestHelper::Create(test_dir_, arrow::schema(fields), /*partition_keys=*/{}, /*primary_keys=*/{}, options, /*is_streaming_mode=*/false)); - arrow::Int32Builder ids_builder; - ASSERT_TRUE(ids_builder.AppendValues({1, 2, 3}).ok()); - std::shared_ptr ids; - ASSERT_TRUE(ids_builder.Finish(&ids).ok()); - arrow::FloatBuilder values_builder; - ASSERT_TRUE(values_builder.AppendValues({1.0f, 2.0f, 3.0f}).ok()); - ASSERT_TRUE(values_builder.AppendNulls(3).ok()); - ASSERT_TRUE(values_builder.AppendValues({4.0f, 5.0f, 6.0f}).ok()); - std::shared_ptr values; - ASSERT_TRUE(values_builder.Finish(&values).ok()); - std::shared_ptr validity = arrow::Buffer::FromString(std::string("\x05", 1)); - auto vectors = - arrow::MakeArray(arrow::ArrayData::Make(vector_type, 3, {validity}, {values->data()}, 1)); - auto data = arrow::StructArray::Make({ids, vectors}, fields).ValueOrDie(); + const std::string data_json = R"([ + [1, [1.0, 2.0, 3.0]], + [2, null], + [3, [4.0, 5.0, 6.0]] + ])"; + auto data = + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), data_json).ValueOrDie(); auto c_array = std::make_unique(); ASSERT_TRUE(arrow::ExportArray(*data, c_array.get()).ok()); RecordBatchBuilder batch_builder(c_array.get()); @@ -360,12 +353,14 @@ TEST_P(WriteAndReadInteTest, TestAppendVector) { helper->NewScan(StartupMode::LatestFull(), /*snapshot_id=*/std::nullopt)); ASSERT_OK_AND_ASSIGN(std::shared_ptr actual, helper->ReadResult(data_splits)); - auto row_kinds = - std::make_shared(3, arrow::Buffer::FromString(std::string("\0\0\0", 3))); - arrow::Result> expected_result = - arrow::StructArray::Make({row_kinds, ids, vectors}, result_fields); - ASSERT_TRUE(expected_result.ok()) << expected_result.status().ToString(); - std::shared_ptr expected = std::move(expected_result).ValueOrDie(); + const std::string expected_json = R"([ + [0, 1, [1.0, 2.0, 3.0]], + [0, 2, null], + [0, 3, [4.0, 5.0, 6.0]] + ])"; + auto expected = + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(result_fields), expected_json) + .ValueOrDie(); ASSERT_TRUE(std::make_shared(expected)->Equals(actual)); } From 906a286c46b2b2947aae4366d4035730d87186ae Mon Sep 17 00:00:00 2001 From: "zhangchaoming.zcm" Date: Mon, 17 Aug 2026 14:07:07 +0800 Subject: [PATCH 04/16] fix(parquet): handle cross-language vector schemas --- .../core/io/vector_file_batch_reader.cpp | 202 +++++++++--------- .../core/io/vector_file_batch_reader_test.cpp | 55 +++++ .../parquet/parquet_vector_converter.cpp | 130 +++-------- .../parquet/parquet_vector_converter_test.cpp | 17 +- .../format/parquet/parquet_vector_io_test.cpp | 85 ++++++++ .../parquet/vector_compatibility/README.md | 41 ++++ .../vector_compatibility/java_vector.parquet | Bin 0 -> 1303 bytes .../vector_compatibility/rust_vector.parquet | Bin 0 -> 949 bytes 8 files changed, 317 insertions(+), 213 deletions(-) create mode 100644 test/test_data/parquet/vector_compatibility/README.md create mode 100644 test/test_data/parquet/vector_compatibility/java_vector.parquet create mode 100644 test/test_data/parquet/vector_compatibility/rust_vector.parquet diff --git a/src/paimon/core/io/vector_file_batch_reader.cpp b/src/paimon/core/io/vector_file_batch_reader.cpp index 03664bcfd..e0cf33f9d 100644 --- a/src/paimon/core/io/vector_file_batch_reader.cpp +++ b/src/paimon/core/io/vector_file_batch_reader.cpp @@ -19,14 +19,13 @@ #include "paimon/core/io/vector_file_batch_reader.h" #include -#include #include +#include #include #include #include "arrow/array.h" #include "arrow/array/array_nested.h" -#include "arrow/array/builder_primitive.h" #include "arrow/c/abi.h" #include "arrow/c/bridge.h" #include "arrow/compute/api.h" @@ -52,30 +51,60 @@ bool ContainsVectorType(const std::shared_ptr& type) { return false; } +std::shared_ptr FindField(const std::shared_ptr& type, + const std::string& name) { + for (const auto& field : type->fields()) { + if (field->name() == name) { + return field; + } + } + return nullptr; +} + std::shared_ptr GetPhysicalReadType( - const std::shared_ptr& logical_type) { + const std::shared_ptr& logical_type, + const std::shared_ptr& file_type) { switch (logical_type->id()) { case arrow::Type::FIXED_SIZE_LIST: { + if (!file_type || file_type->id() != arrow::Type::LIST) { + return logical_type; + } const auto& vector_type = checked_cast(*logical_type); - return arrow::list( - vector_type.value_field()->WithType(GetPhysicalReadType(vector_type.value_type()))); + const auto& list_type = checked_cast(*file_type); + return arrow::list(vector_type.value_field()->WithType( + GetPhysicalReadType(vector_type.value_type(), list_type.value_type()))); } case arrow::Type::STRUCT: { + if (!file_type || file_type->id() != arrow::Type::STRUCT) { + return logical_type; + } arrow::FieldVector fields; fields.reserve(logical_type->num_fields()); for (const auto& field : logical_type->fields()) { - fields.push_back(field->WithType(GetPhysicalReadType(field->type()))); + std::shared_ptr file_field = FindField(file_type, field->name()); + fields.push_back(field->WithType( + GetPhysicalReadType(field->type(), file_field ? file_field->type() : nullptr))); } return arrow::struct_(fields); } - case arrow::Type::LIST: + case arrow::Type::LIST: { + if (!file_type || file_type->id() != arrow::Type::LIST) { + return logical_type; + } return arrow::list(logical_type->field(0)->WithType( - GetPhysicalReadType(logical_type->field(0)->type()))); + GetPhysicalReadType(logical_type->field(0)->type(), file_type->field(0)->type()))); + } case arrow::Type::MAP: { + if (!file_type || file_type->id() != arrow::Type::MAP) { + return logical_type; + } const auto& map_type = checked_cast(*logical_type); + const auto& file_map_type = checked_cast(*file_type); return std::make_shared( - map_type.key_field()->WithType(GetPhysicalReadType(map_type.key_type())), - map_type.item_field()->WithType(GetPhysicalReadType(map_type.item_type())), + map_type.key_field()->WithType( + GetPhysicalReadType(map_type.key_type(), file_map_type.key_type())), + map_type.item_field()->WithType( + GetPhysicalReadType(map_type.item_type(), file_map_type.item_type())), map_type.keys_sorted()); } default: @@ -83,114 +112,54 @@ std::shared_ptr GetPhysicalReadType( } } -Status ValidateVectorElements(const arrow::FixedSizeListArray& array, int32_t vector_length) { - const std::shared_ptr& values = array.values(); - if (values->null_count() == 0) { - return Status::OK(); - } +Status ValidateVectorElements(const arrow::ListArray& array) { for (int64_t i = 0; i < array.length(); ++i) { if (array.IsNull(i)) { continue; } - int64_t value_offset = (array.offset() + i) * vector_length; - for (int32_t j = 0; j < vector_length; ++j) { - if (values->IsNull(value_offset + j)) { - return Status::Invalid("VECTOR cannot contain null elements"); + int64_t value_offset = array.value_offset(i); + int64_t value_length = array.value_length(i); + for (int64_t j = 0; j < value_length; ++j) { + if (array.values()->IsNull(value_offset + j)) { + return Status::Invalid(fmt::format( + "VECTOR cannot contain null elements, found one at row {} position {}", i, j)); } } } return Status::OK(); } -Result GetIndexCapacity(int64_t row_count, int32_t vector_length) { - if (vector_length < 1) { - return Status::Invalid("VECTOR length must be positive"); - } - if (row_count > std::numeric_limits::max() / vector_length) { - return Status::Invalid("VECTOR values exceed the supported Arrow array length"); - } - return row_count * vector_length; -} - -Result> ConvertListToVector( - const std::shared_ptr& array, - const std::shared_ptr& read_type, arrow::MemoryPool* pool) { - int32_t vector_length = read_type->list_size(); - if (array->type()->id() == arrow::Type::FIXED_SIZE_LIST) { - const auto& source_type = checked_cast(*array->type()); - if (source_type.list_size() != vector_length || - !source_type.value_type()->Equals(read_type->value_type())) { - return Status::Invalid(fmt::format("VECTOR type mismatch: data {} vs read {}", - array->type()->ToString(), read_type->ToString())); - } - const auto& vector_array = checked_cast(*array); - PAIMON_RETURN_NOT_OK(ValidateVectorElements(vector_array, vector_length)); - std::shared_ptr data = array->data()->Copy(); - data->type = read_type; - return arrow::MakeArray(data); - } - if (array->type()->id() != arrow::Type::LIST) { - return Status::Invalid( - fmt::format("Cannot restore VECTOR from type {}", array->type()->ToString())); - } - - const auto& list_array = checked_cast(*array); - if (!list_array.value_type()->Equals(read_type->value_type())) { - return Status::Invalid(fmt::format("VECTOR element type mismatch: data {} vs read {}", - list_array.value_type()->ToString(), - read_type->value_type()->ToString())); - } - - arrow::Int64Builder indices_builder(pool); - PAIMON_ASSIGN_OR_RAISE(int64_t index_capacity, - GetIndexCapacity(list_array.length(), vector_length)); - PAIMON_RETURN_NOT_OK_FROM_ARROW(indices_builder.Reserve(index_capacity)); - arrow::BooleanBuilder validity_builder(pool); - PAIMON_RETURN_NOT_OK_FROM_ARROW(validity_builder.Reserve(list_array.length())); - - for (int64_t i = 0; i < list_array.length(); ++i) { - bool valid = !list_array.IsNull(i); - PAIMON_RETURN_NOT_OK_FROM_ARROW(validity_builder.Append(valid)); - if (!valid) { - PAIMON_RETURN_NOT_OK_FROM_ARROW(indices_builder.AppendNulls(vector_length)); +Status ValidateVectorElements(const arrow::FixedSizeListArray& array) { + const auto& vector_type = checked_cast(*array.type()); + for (int64_t i = 0; i < array.length(); ++i) { + if (array.IsNull(i)) { continue; } - int64_t value_length = list_array.value_length(i); - if (value_length != vector_length) { - return Status::Invalid( - fmt::format("Vector length mismatch at row {}: expected {} but got {}", i, - vector_length, value_length)); - } - int64_t value_offset = list_array.value_offset(i); - for (int32_t j = 0; j < vector_length; ++j) { - int64_t index = value_offset + j; - if (list_array.values()->IsNull(index)) { + int64_t value_offset = (array.offset() + i) * vector_type.list_size(); + for (int32_t j = 0; j < vector_type.list_size(); ++j) { + if (array.values()->IsNull(value_offset + j)) { return Status::Invalid(fmt::format( "VECTOR cannot contain null elements, found one at row {} position {}", i, j)); } - PAIMON_RETURN_NOT_OK_FROM_ARROW(indices_builder.Append(index)); } } + return Status::OK(); +} - std::shared_ptr indices; - PAIMON_RETURN_NOT_OK_FROM_ARROW(indices_builder.Finish(&indices)); +Result> CastListToVector( + const std::shared_ptr& array, + const std::shared_ptr& read_type, arrow::MemoryPool* pool) { + if (array->type_id() != arrow::Type::LIST) { + return Status::Invalid( + fmt::format("Cannot restore VECTOR from type {}", array->type()->ToString())); + } + PAIMON_RETURN_NOT_OK(ValidateVectorElements(checked_cast(*array))); arrow::compute::ExecContext exec_context(pool); + arrow::TypeHolder type_holder(read_type.get()); + arrow::compute::CastOptions options = arrow::compute::CastOptions::Safe(); PAIMON_ASSIGN_OR_RAISE_FROM_ARROW( - arrow::Datum values, - arrow::compute::Take(arrow::Datum(list_array.values()), arrow::Datum(indices), - arrow::compute::TakeOptions::NoBoundsCheck(), &exec_context)); - - std::shared_ptr validity; - PAIMON_RETURN_NOT_OK_FROM_ARROW(validity_builder.Finish(&validity)); - std::shared_ptr null_bitmap; - if (list_array.null_count() != 0) { - null_bitmap = validity->data()->buffers[1]; - } - std::shared_ptr data = - arrow::ArrayData::Make(read_type, list_array.length(), {null_bitmap}, - {values.make_array()->data()}, list_array.null_count()); - std::shared_ptr result = arrow::MakeArray(data); - PAIMON_RETURN_NOT_OK_FROM_ARROW(result->ValidateFull()); + std::shared_ptr result, + arrow::compute::Cast(*array, type_holder, options, &exec_context)); return result; } @@ -222,21 +191,37 @@ Result> ConvertToReadType( return array; } switch (read_type->id()) { - case arrow::Type::FIXED_SIZE_LIST: - return ConvertListToVector( + case arrow::Type::FIXED_SIZE_LIST: { + if (array->type_id() == arrow::Type::FIXED_SIZE_LIST) { + const auto& source_type = + checked_cast(*array->type()); + const auto& vector_type = checked_cast(*read_type); + if (source_type.list_size() != vector_type.list_size() || + !source_type.value_type()->Equals(vector_type.value_type())) { + return Status::Invalid(fmt::format("VECTOR type mismatch: data {} vs read {}", + source_type.ToString(), + vector_type.ToString())); + } + PAIMON_RETURN_NOT_OK( + ValidateVectorElements(checked_cast(*array))); + return array; + } + return CastListToVector( array, checked_pointer_cast(read_type), pool); + } case arrow::Type::STRUCT: case arrow::Type::LIST: case arrow::Type::MAP: { - if (array->type()->id() != read_type->id()) { + if (array->type_id() != read_type->id()) { return Status::Invalid(fmt::format("Cannot reconcile file type {} with {}", array->type()->ToString(), read_type->ToString())); } - if (array->type()->num_fields() != read_type->num_fields()) { - return Status::Invalid(fmt::format("Nested type field count mismatch: {} vs {}", - array->type()->ToString(), - read_type->ToString())); + if (array->type()->num_fields() != read_type->num_fields() || + array->data()->child_data.size() != static_cast(read_type->num_fields())) { + return Status::Invalid( + fmt::format("Cannot reconcile file type {} with {}: nested field count differs", + array->type()->ToString(), read_type->ToString())); } std::vector> children; children.reserve(read_type->num_fields()); @@ -280,10 +265,15 @@ Status VectorFileBatchReader::SetReadSchema( } PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr logical_schema, arrow::ImportSchema(read_schema)); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<::ArrowSchema> c_file_schema, reader_->GetFileSchema()); + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr file_schema, + arrow::ImportSchema(c_file_schema.get())); arrow::FieldVector physical_fields; physical_fields.reserve(logical_schema->num_fields()); for (const auto& field : logical_schema->fields()) { - physical_fields.push_back(field->WithType(GetPhysicalReadType(field->type()))); + std::shared_ptr file_field = file_schema->GetFieldByName(field->name()); + physical_fields.push_back(field->WithType( + GetPhysicalReadType(field->type(), file_field ? file_field->type() : nullptr))); } std::shared_ptr physical_schema = arrow::schema(physical_fields, logical_schema->metadata()); diff --git a/src/paimon/core/io/vector_file_batch_reader_test.cpp b/src/paimon/core/io/vector_file_batch_reader_test.cpp index b93bb9f4f..ed7ef38da 100644 --- a/src/paimon/core/io/vector_file_batch_reader_test.cpp +++ b/src/paimon/core/io/vector_file_batch_reader_test.cpp @@ -82,6 +82,36 @@ TEST(VectorFileBatchReaderTest, ConvertSchemaAndNextBatch) { ASSERT_TRUE(BatchReader::IsEofBatch(batch)); } +TEST(VectorFileBatchReaderTest, KeepFixedSizeListFileSchema) { + auto logical_type = AsStructType(arrow::struct_({ + arrow::field("id", arrow::int32()), + arrow::field("embedding", arrow::fixed_size_list(arrow::float32(), 3)), + })); + const std::string json = R"([ + [1, [1.0, 2.0, 3.0]], + [2, null], + [3, [4.0, 5.0, 6.0]] + ])"; + auto logical_array = arrow::ipc::internal::json::ArrayFromJSON(logical_type, json).ValueOrDie(); + auto mock_reader = + std::make_unique(logical_array, logical_type, /*batch_size=*/10); + mock_reader->EnableRandomizeBatchSize(false); + MockFileBatchReader* inner_reader = mock_reader.get(); + VectorFileBatchReader reader(std::move(mock_reader), GetDefaultPool()); + + ArrowSchema c_read_schema; + ASSERT_TRUE(arrow::ExportSchema(*arrow::schema(logical_type->fields()), &c_read_schema).ok()); + ASSERT_OK(reader.SetReadSchema(&c_read_schema, /*predicate=*/nullptr, + /*selection_bitmap=*/std::nullopt)); + ASSERT_EQ(inner_reader->read_schema_->field(1)->type()->id(), arrow::Type::FIXED_SIZE_LIST); + + ASSERT_OK_AND_ASSIGN(BatchReader::ReadBatch batch, reader.NextBatch()); + arrow::Result> actual_result = + arrow::ImportArray(batch.first.get(), batch.second.get()); + ASSERT_TRUE(actual_result.ok()) << actual_result.status().ToString(); + ASSERT_TRUE(logical_array->Equals(std::move(actual_result).ValueOrDie())); +} + TEST(VectorFileBatchReaderTest, ConvertNestedVectorsWithBitmap) { auto logical_vector = arrow::fixed_size_list(arrow::field("item", arrow::float64(), /*nullable=*/false), 2); @@ -142,4 +172,29 @@ TEST(VectorFileBatchReaderTest, RejectInvalidVectorValues) { } } +TEST(VectorFileBatchReaderTest, RejectInvalidFixedSizeListVectorValues) { + auto values_builder = std::make_shared(); + arrow::FixedSizeListBuilder vector_builder(arrow::default_memory_pool(), values_builder, 3); + ASSERT_TRUE(values_builder->Append(1.0f).ok()); + ASSERT_TRUE(values_builder->AppendNull().ok()); + ASSERT_TRUE(values_builder->Append(3.0f).ok()); + ASSERT_TRUE(vector_builder.Append().ok()); + std::shared_ptr vector_array; + ASSERT_TRUE(vector_builder.Finish(&vector_array).ok()); + arrow::Result> struct_result = + arrow::StructArray::Make({vector_array}, {"embedding"}); + ASSERT_TRUE(struct_result.ok()) << struct_result.status().ToString(); + std::shared_ptr physical_array = std::move(struct_result).ValueOrDie(); + auto physical_type = AsStructType(physical_array->type()); + auto mock_reader = std::make_unique(physical_array, physical_type, + /*read_batch_size=*/10); + mock_reader->EnableRandomizeBatchSize(false); + VectorFileBatchReader reader(std::move(mock_reader), GetDefaultPool()); + ArrowSchema c_read_schema; + ASSERT_TRUE(arrow::ExportSchema(*arrow::schema(physical_type->fields()), &c_read_schema).ok()); + ASSERT_OK(reader.SetReadSchema(&c_read_schema, /*predicate=*/nullptr, + /*selection_bitmap=*/std::nullopt)); + ASSERT_NOK(reader.NextBatch()); +} + } // namespace paimon::test diff --git a/src/paimon/format/parquet/parquet_vector_converter.cpp b/src/paimon/format/parquet/parquet_vector_converter.cpp index 00064f52b..df6a10f3d 100644 --- a/src/paimon/format/parquet/parquet_vector_converter.cpp +++ b/src/paimon/format/parquet/parquet_vector_converter.cpp @@ -19,13 +19,10 @@ #include "paimon/format/parquet/parquet_vector_converter.h" #include -#include #include -#include #include "arrow/array.h" #include "arrow/array/array_nested.h" -#include "arrow/array/builder_primitive.h" #include "arrow/compute/api.h" #include "arrow/type.h" #include "paimon/common/utils/arrow/status_utils.h" @@ -47,90 +44,34 @@ bool ContainsVectorType(const std::shared_ptr& type) { return false; } -Status ValidateVectorElements(const arrow::FixedSizeListArray& array, int32_t vector_length) { - const std::shared_ptr& values = array.values(); - if (values->null_count() == 0) { +Status ValidateVectorElements(const std::shared_ptr& array) { + if (!ContainsVectorType(array->type())) { return Status::OK(); } - for (int64_t i = 0; i < array.length(); ++i) { - if (array.IsNull(i)) { - continue; + if (array->type_id() == arrow::Type::FIXED_SIZE_LIST) { + const auto& vector_array = checked_cast(*array); + const auto& vector_type = checked_cast(*array->type()); + const std::shared_ptr& values = vector_array.values(); + if (values->null_count() == 0) { + return Status::OK(); } - int64_t value_offset = (array.offset() + i) * vector_length; - for (int32_t j = 0; j < vector_length; ++j) { - if (values->IsNull(value_offset + j)) { - return Status::Invalid("VECTOR cannot contain null elements"); + for (int64_t i = 0; i < vector_array.length(); ++i) { + if (vector_array.IsNull(i)) { + continue; } - } - } - return Status::OK(); -} - -Result GetIndexCapacity(int64_t row_count, int32_t vector_length) { - if (vector_length < 1) { - return Status::Invalid("VECTOR length must be positive"); - } - if (row_count > std::numeric_limits::max() / vector_length) { - return Status::Invalid("VECTOR values exceed the supported Arrow array length"); - } - return row_count * vector_length; -} - -Result> ConvertVectorToList( - const std::shared_ptr& array, arrow::MemoryPool* pool) { - const auto& vector_array = checked_cast(*array); - const auto& vector_type = checked_cast(*array->type()); - int32_t vector_length = vector_type.list_size(); - PAIMON_RETURN_NOT_OK(ValidateVectorElements(vector_array, vector_length)); - - arrow::Int32Builder offsets_builder(pool); - arrow::Int64Builder indices_builder(pool); - arrow::BooleanBuilder validity_builder(pool); - PAIMON_RETURN_NOT_OK_FROM_ARROW(offsets_builder.Reserve(vector_array.length() + 1)); - PAIMON_ASSIGN_OR_RAISE(int64_t index_capacity, - GetIndexCapacity(vector_array.length(), vector_length)); - PAIMON_RETURN_NOT_OK_FROM_ARROW(indices_builder.Reserve(index_capacity)); - PAIMON_RETURN_NOT_OK_FROM_ARROW(validity_builder.Reserve(vector_array.length())); - PAIMON_RETURN_NOT_OK_FROM_ARROW(offsets_builder.Append(0)); - - int32_t offset = 0; - for (int64_t i = 0; i < vector_array.length(); ++i) { - bool valid = !vector_array.IsNull(i); - PAIMON_RETURN_NOT_OK_FROM_ARROW(validity_builder.Append(valid)); - if (valid) { - if (vector_length > std::numeric_limits::max() - offset) { - return Status::Invalid("VECTOR values exceed the maximum Parquet LIST offset"); - } - int64_t value_offset = (vector_array.offset() + i) * vector_length; - for (int32_t j = 0; j < vector_length; ++j) { - PAIMON_RETURN_NOT_OK_FROM_ARROW(indices_builder.Append(value_offset + j)); + int64_t value_offset = (vector_array.offset() + i) * vector_type.list_size(); + for (int32_t j = 0; j < vector_type.list_size(); ++j) { + if (values->IsNull(value_offset + j)) { + return Status::Invalid("VECTOR cannot contain null elements"); + } } - offset += vector_length; } - PAIMON_RETURN_NOT_OK_FROM_ARROW(offsets_builder.Append(offset)); + return Status::OK(); } - - std::shared_ptr offsets; - std::shared_ptr indices; - std::shared_ptr validity; - PAIMON_RETURN_NOT_OK_FROM_ARROW(offsets_builder.Finish(&offsets)); - PAIMON_RETURN_NOT_OK_FROM_ARROW(indices_builder.Finish(&indices)); - PAIMON_RETURN_NOT_OK_FROM_ARROW(validity_builder.Finish(&validity)); - - arrow::compute::ExecContext exec_context(pool); - PAIMON_ASSIGN_OR_RAISE_FROM_ARROW( - arrow::Datum values, - arrow::compute::Take(arrow::Datum(vector_array.values()), arrow::Datum(indices), - arrow::compute::TakeOptions::NoBoundsCheck(), &exec_context)); - std::shared_ptr null_bitmap; - if (vector_array.null_count() != 0) { - null_bitmap = validity->data()->buffers[1]; + for (const auto& child_data : array->data()->child_data) { + PAIMON_RETURN_NOT_OK(ValidateVectorElements(arrow::MakeArray(child_data))); } - std::shared_ptr write_type = - arrow::list(vector_type.value_field()->WithType(values.type())); - return std::make_shared(write_type, vector_array.length(), - offsets->data()->buffers[1], values.make_array(), - null_bitmap, vector_array.null_count()); + return Status::OK(); } } // namespace @@ -171,28 +112,15 @@ Result> ParquetVectorConverter::ConvertToWriteType if (!ContainsVectorType(array->type())) { return array; } - if (array->type()->id() == arrow::Type::FIXED_SIZE_LIST) { - return ConvertVectorToList(array, pool); - } - switch (array->type()->id()) { - case arrow::Type::STRUCT: - case arrow::Type::LIST: - case arrow::Type::MAP: { - std::vector> children; - children.reserve(array->type()->num_fields()); - for (const auto& child_data : array->data()->child_data) { - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr child, - ConvertToWriteType(arrow::MakeArray(child_data), pool)); - children.push_back(child->data()); - } - std::shared_ptr data = array->data()->Copy(); - data->child_data = std::move(children); - data->type = GetWriteType(array->type()); - return arrow::MakeArray(data); - } - default: - return array; - } + PAIMON_RETURN_NOT_OK(ValidateVectorElements(array)); + std::shared_ptr write_type = GetWriteType(array->type()); + arrow::compute::ExecContext exec_context(pool); + arrow::TypeHolder type_holder(write_type.get()); + arrow::compute::CastOptions options = arrow::compute::CastOptions::Safe(); + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW( + std::shared_ptr result, + arrow::compute::Cast(*array, type_holder, options, &exec_context)); + return result; } } // namespace paimon::parquet diff --git a/src/paimon/format/parquet/parquet_vector_converter_test.cpp b/src/paimon/format/parquet/parquet_vector_converter_test.cpp index 3f2598c64..602fab08a 100644 --- a/src/paimon/format/parquet/parquet_vector_converter_test.cpp +++ b/src/paimon/format/parquet/parquet_vector_converter_test.cpp @@ -29,10 +29,15 @@ namespace paimon::parquet::test { TEST(ParquetVectorConverterTest, ConvertNullableVectorToList) { - auto vector_type = arrow::fixed_size_list(arrow::float32(), 3); - auto vector_array = arrow::ipc::internal::json::ArrayFromJSON( - vector_type, R"([[1.0, 2.0, 3.0], null, [4.0, 5.0, 6.0]])") - .ValueOrDie(); + auto values_builder = std::make_shared(); + arrow::FixedSizeListBuilder vector_builder(arrow::default_memory_pool(), values_builder, 3); + ASSERT_TRUE(values_builder->AppendValues({1.0f, 2.0f, 3.0f}).ok()); + ASSERT_TRUE(vector_builder.Append().ok()); + ASSERT_TRUE(vector_builder.AppendNull().ok()); + ASSERT_TRUE(values_builder->AppendValues({4.0f, 5.0f, 6.0f}).ok()); + ASSERT_TRUE(vector_builder.Append().ok()); + std::shared_ptr vector_array; + ASSERT_TRUE(vector_builder.Finish(&vector_array).ok()); ASSERT_OK_AND_ASSIGN( std::shared_ptr converted, @@ -41,9 +46,9 @@ TEST(ParquetVectorConverterTest, ConvertNullableVectorToList) { auto list_array = checked_pointer_cast(converted); ASSERT_EQ(list_array->value_length(0), 3); ASSERT_TRUE(list_array->IsNull(1)); - ASSERT_EQ(list_array->value_length(1), 0); + ASSERT_EQ(list_array->value_length(1), 3); ASSERT_EQ(list_array->value_length(2), 3); - ASSERT_EQ(list_array->values()->length(), 6); + ASSERT_EQ(list_array->values()->length(), 9); } TEST(ParquetVectorConverterTest, ConvertNestedVectorsToList) { diff --git a/src/paimon/format/parquet/parquet_vector_io_test.cpp b/src/paimon/format/parquet/parquet_vector_io_test.cpp index e871e0517..8da61e47c 100644 --- a/src/paimon/format/parquet/parquet_vector_io_test.cpp +++ b/src/paimon/format/parquet/parquet_vector_io_test.cpp @@ -21,6 +21,7 @@ #include #include #include +#include #include "arrow/api.h" #include "arrow/c/abi.h" @@ -113,6 +114,75 @@ class ParquetVectorIoTest : public ::testing::Test { << actual->ToString(); } + void ReadFixtureAndCheck( + const std::string& file_name, arrow::Type::type expected_file_vector_type, + int32_t vector_length, const std::vector& expected_ids, + const std::vector>>& expected_vectors) { + std::string file_path = + paimon::test::GetDataDir() + "/parquet/vector_compatibility/" + file_name; + ASSERT_OK_AND_ASSIGN(std::shared_ptr in, fs_->Open(file_path)); + ASSERT_OK_AND_ASSIGN(int64_t length, in->Length()); + auto in_stream = std::make_shared(in, length, arrow_pool_); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr reader, + ParquetFileBatchReader::Create(std::move(in_stream), /*options=*/{}, + /*batch_size=*/10, /*file_metadata=*/nullptr, + /*storage_read_bytes=*/nullptr, arrow_pool_)); + ASSERT_OK_AND_ASSIGN(std::unique_ptr c_file_schema, reader->GetFileSchema()); + arrow::Result> file_type_result = + arrow::ImportType(c_file_schema.get()); + ASSERT_TRUE(file_type_result.ok()) << file_type_result.status().ToString(); + auto file_type = + checked_pointer_cast(std::move(file_type_result).ValueOrDie()); + std::shared_ptr file_vector_field = file_type->GetFieldByName("embedding"); + ASSERT_TRUE(file_vector_field); + ASSERT_EQ(file_vector_field->type()->id(), expected_file_vector_type); + std::shared_ptr file_id_field = file_type->GetFieldByName("id"); + ASSERT_TRUE(file_id_field); + + auto vector_type = arrow::fixed_size_list( + arrow::field("element", arrow::float32(), /*nullable=*/false), vector_length); + auto logical_schema = + arrow::schema({file_id_field, file_vector_field->WithType(vector_type)}); + std::unique_ptr vector_reader = + std::make_unique(std::move(reader), pool_); + auto c_read_schema = std::make_unique(); + ASSERT_TRUE(arrow::ExportSchema(*logical_schema, c_read_schema.get()).ok()); + ASSERT_OK(vector_reader->SetReadSchema(c_read_schema.get(), /*predicate=*/nullptr, + /*selection_bitmap=*/std::nullopt)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr actual, + paimon::test::ReadResultCollector::CollectResult(vector_reader.get())); + ASSERT_EQ(actual->num_chunks(), 1); + ASSERT_EQ(actual->type()->id(), arrow::Type::STRUCT); + auto struct_array = checked_pointer_cast(actual->chunk(0)); + std::shared_ptr id_field = struct_array->GetFieldByName("id"); + std::shared_ptr vector_field = struct_array->GetFieldByName("embedding"); + ASSERT_TRUE(id_field); + ASSERT_TRUE(vector_field); + ASSERT_EQ(id_field->type_id(), arrow::Type::INT32); + ASSERT_EQ(vector_field->type_id(), arrow::Type::FIXED_SIZE_LIST); + auto vector_array = checked_pointer_cast(vector_field); + ASSERT_EQ(id_field->length(), static_cast(expected_ids.size())); + ASSERT_EQ(vector_array->length(), static_cast(expected_vectors.size())); + const int32_t* id_values = id_field->data()->GetValues(1); + for (int64_t i = 0; i < id_field->length(); ++i) { + ASSERT_FALSE(id_field->IsNull(i)); + ASSERT_EQ(id_values[id_field->offset() + i], expected_ids[i]); + if (!expected_vectors[i]) { + ASSERT_TRUE(vector_array->IsNull(i)); + continue; + } + ASSERT_FALSE(vector_array->IsNull(i)); + ASSERT_EQ(vector_array->value_length(i), + static_cast(expected_vectors[i]->size())); + std::shared_ptr values = vector_array->value_slice(i); + const float* vector_values = values->data()->GetValues(1); + for (int64_t j = 0; j < vector_array->value_length(i); ++j) { + ASSERT_FLOAT_EQ(vector_values[j], expected_vectors[i].value()[j]); + } + } + } + private: std::shared_ptr pool_; std::shared_ptr arrow_pool_; @@ -156,4 +226,19 @@ TEST_F(ParquetVectorIoTest, WriteAndReadNestedDoubleVector) { [2, [null, null, [["b", null]]]]])"); } +TEST_F(ParquetVectorIoTest, ReadJavaFixture) { + ReadFixtureAndCheck( + "java_vector.parquet", arrow::Type::LIST, /*vector_length=*/2, + /*expected_ids=*/{0, 1, 2, 3, 4}, + /*expected_vectors=*/ + {{{0.0f, 0.0f}}, {{1.0f, 0.0f}}, {{2.0f, 0.0f}}, {{3.0f, 0.0f}}, {{4.0f, 0.0f}}}); +} + +TEST_F(ParquetVectorIoTest, ReadRustFixture) { + ReadFixtureAndCheck("rust_vector.parquet", arrow::Type::FIXED_SIZE_LIST, + /*vector_length=*/3, /*expected_ids=*/{1, 2, 3}, + /*expected_vectors=*/ + {{{1.0f, 2.0f, 3.0f}}, {{7.0f, 8.0f, 9.0f}}, {{4.0f, 5.0f, 6.0f}}}); +} + } // namespace paimon::parquet::test diff --git a/test/test_data/parquet/vector_compatibility/README.md b/test/test_data/parquet/vector_compatibility/README.md new file mode 100644 index 000000000..8aad4adfe --- /dev/null +++ b/test/test_data/parquet/vector_compatibility/README.md @@ -0,0 +1,41 @@ + + +# VECTOR Parquet compatibility fixtures + +These files pin the two physical Arrow schemas produced by Java and Rust writers for Paimon +VECTOR columns. + +- `java_vector.parquet` was copied from Apache Paimon Rust commit + `403a2b2e9bfc4ea66cd7e633619f1460efd18bc8`, path + `crates/paimon/testdata/pkvector/pk_vector_ivf_flat/bucket-0/data-932a1249-f7e0-4a03-8e1f-ab8c85cbb76f-0.parquet`. + The fixture documentation records Apache Paimon Java commit `7234e4c34` and + `PkVectorFixtureGenerator` as its source. Its VECTOR column is exposed as Arrow `list`. +- `rust_vector.parquet` was generated with Apache Arrow Rust 58.4.0 using + `FixedSizeListBuilder` and `parquet::arrow::ArrowWriter`, the same Arrow and + Parquet representation used by Apache Paimon Rust. Its VECTOR column is exposed as Arrow + `fixed_size_list[3]`. The rows are `(1, [1, 2, 3])`, `(2, [7, 8, 9])`, and + `(3, [4, 5, 6])`. + +SHA-256 checksums: + +```text +2b2325cc2266301beaa2c78ec666cb5e0ee62283049de2a7231e3c9ae07bf3ca java_vector.parquet +b5ba47e766ad72fca9c8485aa718ad27709c1fb4d34fb3670aa35e2001cbdbb0 rust_vector.parquet +``` diff --git a/test/test_data/parquet/vector_compatibility/java_vector.parquet b/test/test_data/parquet/vector_compatibility/java_vector.parquet new file mode 100644 index 0000000000000000000000000000000000000000..5184c7a9f29400d93dec34ea86ee08b9a96b67f4 GIT binary patch literal 1303 zcmbVMO=uHA6rM~b$#!kA#k^q`7Re<`Dq6FxX(|Lwt0}eBQmZKlg_xhM2AZ_CX^*l{ z=&3?4UPPe>5iK5a(UW-5gC6Wr6zZV|QADUG4@IqScC$&^Uc_Z5JM-qf@B7|+Ga2h2 zH-JC{lJIeJWvfjC8J7}BghZa5{7r_|i2Zo*m*Vi^U^sA%eq%5Q)d%bMHw7P9?n;0!eQymV)&MSzQlx~6t?^EwLtOE|Oh z{qpYhD>)&sUHUbL9;KhfSrT6yR?=ZQ1_N;S;itC&4T31g4}Lf7u~Qb`Iu#WPdcD46 zs1-pmPVJzoV&{(^@dc{#niawg3qBTfV zjZ08QS}|LMN^Mr9FkGAN(E9!cgIQrkkd>jf*=(^iQ_jriZ1{C6F0OUCR^; z=|8)>y8-Mb+S)T$QkT=I(Q~QHX!_DXYCQc5HxD}jhcXlW7t^WC$c54K;iIt~v7!Q6pG~=cr5%eswL1R#uPSPoUZ`IlNqK;z??QqX1S8KRxOJW zE~}b4sS^z)hl!-t^+79_`2;WPQo<>|(+N%@b}O6})&vhS>Lf_T-vl7YU;3bn2t4NC z9mvfFzRfH8VxrMHrQS~@s5i7>-AlA7h`#&aj7@~XC8jFPb+S3G^Hq;trTRCI=BRCC zfOR2m2FLL{GEp6Ivz$bJ1c38(e=U((VX(~gn(WYjF3PfmSRY(0Ne%m%+Sllm-5a+j zh#%ySJp8>!-(dPzudUlkHtTj*MBT5uvvFn#zb{yq128~%m2p{@_;X(nuiWr#cJ9`K zSvgUf3oV*+^TnBR$mlY9yP}~;ZlO3`2%U;{=X?5$Xuc=jUFbgD8|^WTJ|iACvUxK% bW#;p7BWF(a8lCE-Mo9BZe&P)t!@tD8O9jc# literal 0 HcmV?d00001 diff --git a/test/test_data/parquet/vector_compatibility/rust_vector.parquet b/test/test_data/parquet/vector_compatibility/rust_vector.parquet new file mode 100644 index 0000000000000000000000000000000000000000..761ee734172e3e0204c7f85cf76d19ca91d0d636 GIT binary patch literal 949 zcmZuw&2AD=6h2(0itriRKmlJZj-TDB}* zyEHC*2;DmYI5qt!{b7!!L>B&8F&i(E=-}wo&b$3Tv(v@yY+LCfaDIyZkG*psw zq*K647HZ=meTxv@l+BjRnQ)@jJ|ZhP(BDwfL|Nzx{#WRa3TTS5SU^TX!`vI+QvxO@ ztc~oWR~HU^FDE?L;s#($ec`D!%!L3mw`AD!?Tq&r1mxvaS>8h+dzUHMW*0H9o0-9U zK3l$t-o6he18D)gO|BRXsU+N+!Q>$SGN_H6BF^DKIx8Zv^9D5=e9D8hM0VC=5(!uxt7E_HJNdZZxOqi)O!x6^bCbLRc821-&nyh7k7>E3J_}hS zCUs)cy{nc?8|(5YlvCeQIR0hIe}`JU1pcn}xz^R~np)+%|Mo=cOIj-RQ^*RbPeR$g z1GB|zb2GZzeR=pK-jw@CJYZQat6J+0l;wriOI-`~bB18G=2w;u?pi-%7tq?`#%^uK z_*4Gi5f*&Ay1vv03?a|(nm~hOTMxq|QfnF~)OF2sMm(n( zHhD+qEIf7ftdB3b-q~RO;XGK}pGVsl!8{txrrV{x;%@PwSI86L&@cRS+W5o%1v4(U AuK)l5 literal 0 HcmV?d00001 From 50a972d97981881f6d5b2aae92f70a2e83d76046 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Mon, 17 Aug 2026 12:10:07 +0000 Subject: [PATCH 05/16] refactor(common): centralize VECTOR helper functions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `ContainsVectorType`, `ContainsVectorField` and `ValidateVectorElements` were duplicated across schema validation, the Parquet write path and the VECTOR read wrapper. Move them into `VectorUtils` and let every caller share one implementation. While validating a FixedSizeList, also reject a VECTOR whose child does not hold `length * list_size` values. Arrow does not verify this when importing an array over the C data interface, so the element scan reading the child validity bitmap could run past its end on the write path. Keep the name and metadata of a MAP entries field when rebuilding a nested type for reads and writes, instead of falling back to Arrow's defaults. Co-authored-by: 小明同学 --- src/paimon/CMakeLists.txt | 2 + src/paimon/common/utils/arrow/arrow_utils.cpp | 22 +-- .../common/utils/arrow/arrow_utils_test.cpp | 21 ++- .../common/utils/arrow/vector_utils.cpp | 141 ++++++++++++++++++ src/paimon/common/utils/arrow/vector_utils.h | 58 +++++++ .../common/utils/arrow/vector_utils_test.cpp | 124 +++++++++++++++ .../core/io/vector_file_batch_reader.cpp | 91 ++++------- src/paimon/core/schema/schema_validation.cpp | 17 +-- .../parquet/parquet_vector_converter.cpp | 59 +------- 9 files changed, 386 insertions(+), 149 deletions(-) create mode 100644 src/paimon/common/utils/arrow/vector_utils.cpp create mode 100644 src/paimon/common/utils/arrow/vector_utils.h create mode 100644 src/paimon/common/utils/arrow/vector_utils_test.cpp diff --git a/src/paimon/CMakeLists.txt b/src/paimon/CMakeLists.txt index 636b87098..3a20b67ea 100644 --- a/src/paimon/CMakeLists.txt +++ b/src/paimon/CMakeLists.txt @@ -157,6 +157,7 @@ set(PAIMON_COMMON_SRCS common/utils/arrow/arrow_output_stream_adapter.cpp common/utils/arrow/arrow_utils.cpp common/utils/arrow/mem_utils.cpp + common/utils/arrow/vector_utils.cpp common/utils/binary_row_partition_computer.cpp common/utils/bit_set.cpp common/utils/bloom_filter.cpp @@ -603,6 +604,7 @@ if(PAIMON_BUILD_TESTS) common/utils/row_range_index_test.cpp common/utils/var_length_int_utils_test.cpp common/utils/arrow/arrow_utils_test.cpp + common/utils/arrow/vector_utils_test.cpp common/utils/arrow/arrow_stream_adapter_test.cpp common/utils/arrow/mem_utils_test.cpp common/utils/arrow/status_utils_test.cpp diff --git a/src/paimon/common/utils/arrow/arrow_utils.cpp b/src/paimon/common/utils/arrow/arrow_utils.cpp index f8c65af18..c301fd5bb 100644 --- a/src/paimon/common/utils/arrow/arrow_utils.cpp +++ b/src/paimon/common/utils/arrow/arrow_utils.cpp @@ -30,6 +30,7 @@ #include "arrow/util/compression.h" #include "fmt/format.h" #include "paimon/common/utils/arrow/status_utils.h" +#include "paimon/common/utils/arrow/vector_utils.h" #include "paimon/common/utils/checked_cast.h" #include "paimon/common/utils/string_utils.h" @@ -376,23 +377,10 @@ Status ArrowUtils::InnerCheckNullabilityMatch(const std::shared_ptrvalue_field(), list_array->values())); } else if (type->id() == arrow::Type::FIXED_SIZE_LIST) { - auto vector_type = checked_pointer_cast(field->type()); - auto vector_array = checked_pointer_cast(data); - const std::shared_ptr& values = vector_array->values(); - if (values->null_count() != 0) { - int32_t vector_length = vector_type->list_size(); - for (int64_t i = 0; i < vector_array->length(); ++i) { - if (vector_array->IsNull(i)) { - continue; - } - int64_t value_offset = (vector_array->offset() + i) * vector_length; - for (int32_t j = 0; j < vector_length; ++j) { - if (values->IsNull(value_offset + j)) { - return Status::Invalid(fmt::format( - "VECTOR field {} cannot contain null elements", field->name())); - } - } - } + Status status = VectorUtils::ValidateVectorElements(*data); + if (!status.ok()) { + return Status::Invalid( + fmt::format("VECTOR field {} is invalid: {}", field->name(), status.message())); } } else if (type->id() == arrow::Type::MAP) { auto map_type = checked_pointer_cast(field->type()); diff --git a/src/paimon/common/utils/arrow/arrow_utils_test.cpp b/src/paimon/common/utils/arrow/arrow_utils_test.cpp index f1f383fe7..571326989 100644 --- a/src/paimon/common/utils/arrow/arrow_utils_test.cpp +++ b/src/paimon/common/utils/arrow/arrow_utils_test.cpp @@ -263,7 +263,26 @@ TEST(ArrowUtilsTest, TestCheckNullableMatchRejectsNullVectorElement) { ASSERT_NOK_WITH_MSG( ArrowUtils::CheckNullabilityMatch(arrow::schema({vector_field}), struct_array), - "VECTOR field embedding cannot contain null elements"); + "VECTOR field embedding is invalid: VECTOR cannot contain null elements"); +} + +// Arrow accepts a FixedSizeList whose child is shorter than `length * list_size` when importing +// it over the C data interface, so the nullability check must reject it rather than scan past the +// end of the child. +TEST(ArrowUtilsTest, TestCheckNullableMatchRejectsTruncatedVector) { + auto vector_type = arrow::fixed_size_list(arrow::float32(), 3); + auto vector_field = arrow::field("embedding", vector_type); + arrow::FloatBuilder values_builder; + ASSERT_TRUE(values_builder.AppendValues({1.0f, 2.0f, 3.0f}).ok()); + std::shared_ptr values = values_builder.Finish().ValueOrDie(); + auto vector_data = arrow::ArrayData::Make(vector_type, /*length=*/2, {nullptr}, + {values->data()}, /*null_count=*/0); + auto vector_array = arrow::MakeArray(vector_data); + auto struct_array = arrow::StructArray::Make({vector_array}, {vector_field}).ValueOrDie(); + + ASSERT_NOK_WITH_MSG( + ArrowUtils::CheckNullabilityMatch(arrow::schema({vector_field}), struct_array), + "VECTOR field embedding is invalid: VECTOR holds 3 elements while 2 rows of dimension 3"); } TEST(ArrowUtilsTest, TestCheckNullableMatchWithMap) { diff --git a/src/paimon/common/utils/arrow/vector_utils.cpp b/src/paimon/common/utils/arrow/vector_utils.cpp new file mode 100644 index 000000000..620373603 --- /dev/null +++ b/src/paimon/common/utils/arrow/vector_utils.cpp @@ -0,0 +1,141 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/common/utils/arrow/vector_utils.h" + +#include + +#include "arrow/array.h" +#include "arrow/array/array_nested.h" +#include "arrow/type.h" +#include "fmt/format.h" +#include "paimon/common/utils/checked_cast.h" + +namespace paimon { +namespace { + +Status ValidateListVector(const arrow::ListArray& array) { + if (array.values()->null_count() == 0) { + return Status::OK(); + } + for (int64_t i = 0; i < array.length(); ++i) { + if (array.IsNull(i)) { + continue; + } + int64_t value_offset = array.value_offset(i); + int64_t value_length = array.value_length(i); + for (int64_t j = 0; j < value_length; ++j) { + if (array.values()->IsNull(value_offset + j)) { + return Status::Invalid(fmt::format( + "VECTOR cannot contain null elements, found one at row {} position {}", i, j)); + } + } + } + return Status::OK(); +} + +Status ValidateFixedSizeListVector(const arrow::FixedSizeListArray& array) { + const auto& vector_type = checked_cast(*array.type()); + int32_t vector_length = vector_type.list_size(); + const std::shared_ptr& values = array.values(); + // Arrow does not check this when importing an array over the C data interface, so the + // element scan below would otherwise read past the end of the values array. + if (values->length() < (array.offset() + array.length()) * vector_length) { + return Status::Invalid(fmt::format( + "VECTOR holds {} elements while {} rows of dimension {} require {}", values->length(), + array.length(), vector_length, (array.offset() + array.length()) * vector_length)); + } + if (values->null_count() == 0) { + return Status::OK(); + } + for (int64_t i = 0; i < array.length(); ++i) { + if (array.IsNull(i)) { + continue; + } + int64_t value_offset = (array.offset() + i) * vector_length; + for (int32_t j = 0; j < vector_length; ++j) { + if (values->IsNull(value_offset + j)) { + return Status::Invalid(fmt::format( + "VECTOR cannot contain null elements, found one at row {} position {}", i, j)); + } + } + } + return Status::OK(); +} + +} // namespace + +bool VectorUtils::ContainsVectorType(const std::shared_ptr& type) { + if (!type) { + return false; + } + if (type->id() == arrow::Type::FIXED_SIZE_LIST) { + return true; + } + for (const auto& field : type->fields()) { + if (ContainsVectorType(field->type())) { + return true; + } + } + return false; +} + +bool VectorUtils::ContainsVectorField(const std::shared_ptr& field) { + return field != nullptr && ContainsVectorType(field->type()); +} + +bool VectorUtils::ContainsVector(const std::shared_ptr& schema) { + if (!schema) { + return false; + } + for (const auto& field : schema->fields()) { + if (ContainsVectorField(field)) { + return true; + } + } + return false; +} + +Status VectorUtils::ValidateVectorElements(const arrow::Array& array) { + switch (array.type_id()) { + case arrow::Type::LIST: + return ValidateListVector(checked_cast(array)); + case arrow::Type::FIXED_SIZE_LIST: + return ValidateFixedSizeListVector( + checked_cast(array)); + default: + return Status::Invalid(fmt::format("Cannot validate VECTOR values of type {}", + array.type()->ToString())); + } +} + +Status VectorUtils::ValidateNestedVectorElements(const std::shared_ptr& array) { + if (!array || !ContainsVectorType(array->type())) { + return Status::OK(); + } + if (array->type_id() == arrow::Type::FIXED_SIZE_LIST) { + return ValidateVectorElements(*array); + } + for (const auto& child_data : array->data()->child_data) { + PAIMON_RETURN_NOT_OK(ValidateNestedVectorElements(arrow::MakeArray(child_data))); + } + return Status::OK(); +} + +} // namespace paimon diff --git a/src/paimon/common/utils/arrow/vector_utils.h b/src/paimon/common/utils/arrow/vector_utils.h new file mode 100644 index 000000000..034923887 --- /dev/null +++ b/src/paimon/common/utils/arrow/vector_utils.h @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include + +#include "paimon/result.h" +#include "paimon/status.h" +#include "paimon/visibility.h" + +namespace arrow { +class Array; +class DataType; +class Field; +class Schema; +} // namespace arrow + +namespace paimon { + +/// Helpers shared by the schema, read and write paths handling VECTOR values, which are +/// represented as Arrow FixedSizeList. +class PAIMON_EXPORT VectorUtils { + public: + VectorUtils() = delete; + ~VectorUtils() = delete; + + static bool ContainsVectorType(const std::shared_ptr& type); + + static bool ContainsVectorField(const std::shared_ptr& field); + + static bool ContainsVector(const std::shared_ptr& schema); + + /// Rejects VECTOR values whose elements are not fully materialized or contain nulls. + /// `array` must be the List or FixedSizeList array holding the VECTOR values. + static Status ValidateVectorElements(const arrow::Array& array); + + /// Validates every VECTOR value reachable from `array`, including nested ones. + static Status ValidateNestedVectorElements(const std::shared_ptr& array); +}; + +} // namespace paimon diff --git a/src/paimon/common/utils/arrow/vector_utils_test.cpp b/src/paimon/common/utils/arrow/vector_utils_test.cpp new file mode 100644 index 000000000..131f81638 --- /dev/null +++ b/src/paimon/common/utils/arrow/vector_utils_test.cpp @@ -0,0 +1,124 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/common/utils/arrow/vector_utils.h" + +#include + +#include "arrow/api.h" +#include "arrow/ipc/json_simple.h" +#include "gtest/gtest.h" +#include "paimon/testing/utils/testharness.h" + +namespace paimon::test { +namespace { + +std::shared_ptr ArrayFromJSON(const std::shared_ptr& type, + const std::string& json) { + arrow::Result> result = + arrow::ipc::internal::json::ArrayFromJSON(type, json); + EXPECT_TRUE(result.ok()) << result.status().ToString(); + return std::move(result).ValueOrDie(); +} + +} // namespace + +TEST(VectorUtilsTest, TestContainsVector) { + auto vector_type = arrow::fixed_size_list(arrow::float32(), 3); + ASSERT_TRUE(VectorUtils::ContainsVectorType(vector_type)); + ASSERT_TRUE(VectorUtils::ContainsVectorType(arrow::list(vector_type))); + ASSERT_TRUE(VectorUtils::ContainsVectorType(arrow::map(arrow::utf8(), vector_type))); + ASSERT_TRUE( + VectorUtils::ContainsVectorType(arrow::struct_({arrow::field("v", vector_type)}))); + ASSERT_FALSE(VectorUtils::ContainsVectorType(arrow::list(arrow::float32()))); + ASSERT_FALSE(VectorUtils::ContainsVectorType(nullptr)); + + ASSERT_TRUE(VectorUtils::ContainsVectorField(arrow::field("v", arrow::list(vector_type)))); + ASSERT_FALSE(VectorUtils::ContainsVectorField(arrow::field("v", arrow::int32()))); + ASSERT_FALSE(VectorUtils::ContainsVectorField(nullptr)); + + ASSERT_TRUE(VectorUtils::ContainsVector( + arrow::schema({arrow::field("id", arrow::int32()), arrow::field("v", vector_type)}))); + ASSERT_FALSE(VectorUtils::ContainsVector(arrow::schema({arrow::field("id", arrow::int32())}))); + ASSERT_FALSE(VectorUtils::ContainsVector(nullptr)); +} + +TEST(VectorUtilsTest, TestValidateVectorElements) { + auto vector_type = arrow::fixed_size_list(arrow::float32(), 3); + ASSERT_OK(VectorUtils::ValidateVectorElements( + *ArrayFromJSON(vector_type, R"([[1.0, 2.0, 3.0], null, [4.0, 5.0, 6.0]])"))); + ASSERT_NOK_WITH_MSG( + VectorUtils::ValidateVectorElements( + *ArrayFromJSON(vector_type, R"([[1.0, 2.0, 3.0], [4.0, null, 6.0]])")), + "VECTOR cannot contain null elements, found one at row 1 position 1"); + + // A sliced array must be validated against its own rows only. + std::shared_ptr sliced = + ArrayFromJSON(vector_type, R"([[1.0, null, 3.0], [4.0, 5.0, 6.0]])")->Slice(1, 1); + ASSERT_OK(VectorUtils::ValidateVectorElements(*sliced)); + + auto list_type = arrow::list(arrow::float32()); + ASSERT_OK(VectorUtils::ValidateVectorElements( + *ArrayFromJSON(list_type, R"([[1.0, 2.0, 3.0], null])"))); + ASSERT_NOK_WITH_MSG( + VectorUtils::ValidateVectorElements(*ArrayFromJSON(list_type, R"([[1.0, null, 3.0]])")), + "VECTOR cannot contain null elements, found one at row 0 position 1"); + + ASSERT_NOK_WITH_MSG( + VectorUtils::ValidateVectorElements(*ArrayFromJSON(arrow::int32(), "[1, 2]")), + "Cannot validate VECTOR values of type int32"); +} + +// Arrow does not check that a FixedSizeList child holds `length * list_size` values when +// importing an array over the C data interface, so the element scan must reject it instead of +// reading past the end of the child. +TEST(VectorUtilsTest, TestValidateVectorElementsRejectsTruncatedValues) { + auto vector_type = arrow::fixed_size_list(arrow::float32(), 3); + std::shared_ptr values = ArrayFromJSON(arrow::float32(), "[1.0, null, 3.0]"); + auto truncated = arrow::MakeArray( + arrow::ArrayData::Make(vector_type, /*length=*/2, {nullptr}, {values->data()}, + /*null_count=*/0)); + + ASSERT_NOK_WITH_MSG(VectorUtils::ValidateVectorElements(*truncated), + "VECTOR holds 3 elements while 2 rows of dimension 3 require 6"); +} + +TEST(VectorUtilsTest, TestValidateNestedVectorElements) { + auto vector_type = arrow::fixed_size_list(arrow::float32(), 2); + auto nested_type = arrow::struct_({ + arrow::field("id", arrow::int32()), + arrow::field("vectors", arrow::list(vector_type)), + arrow::field("by_name", arrow::map(arrow::utf8(), vector_type)), + }); + ASSERT_OK(VectorUtils::ValidateNestedVectorElements( + ArrayFromJSON(nested_type, R"([[1, [[1.0, 2.0], null], [["a", [3.0, 4.0]]]]])"))); + ASSERT_NOK_WITH_MSG( + VectorUtils::ValidateNestedVectorElements( + ArrayFromJSON(nested_type, R"([[1, [[1.0, null]], [["a", [3.0, 4.0]]]]])")), + "VECTOR cannot contain null elements"); + ASSERT_NOK_WITH_MSG( + VectorUtils::ValidateNestedVectorElements( + ArrayFromJSON(nested_type, R"([[1, [[1.0, 2.0]], [["a", [null, 4.0]]]]])")), + "VECTOR cannot contain null elements"); + ASSERT_OK(VectorUtils::ValidateNestedVectorElements( + ArrayFromJSON(arrow::list(arrow::float32()), R"([[1.0, null]])"))); + ASSERT_OK(VectorUtils::ValidateNestedVectorElements(nullptr)); +} + +} // namespace paimon::test diff --git a/src/paimon/core/io/vector_file_batch_reader.cpp b/src/paimon/core/io/vector_file_batch_reader.cpp index e0cf33f9d..a9cf99a04 100644 --- a/src/paimon/core/io/vector_file_batch_reader.cpp +++ b/src/paimon/core/io/vector_file_batch_reader.cpp @@ -18,14 +18,12 @@ #include "paimon/core/io/vector_file_batch_reader.h" -#include #include #include #include #include #include "arrow/array.h" -#include "arrow/array/array_nested.h" #include "arrow/c/abi.h" #include "arrow/c/bridge.h" #include "arrow/compute/api.h" @@ -33,24 +31,13 @@ #include "fmt/format.h" #include "paimon/common/utils/arrow/mem_utils.h" #include "paimon/common/utils/arrow/status_utils.h" +#include "paimon/common/utils/arrow/vector_utils.h" #include "paimon/common/utils/checked_cast.h" #include "paimon/status.h" namespace paimon { namespace { -bool ContainsVectorType(const std::shared_ptr& type) { - if (type->id() == arrow::Type::FIXED_SIZE_LIST) { - return true; - } - for (const auto& field : type->fields()) { - if (ContainsVectorType(field->type())) { - return true; - } - } - return false; -} - std::shared_ptr FindField(const std::shared_ptr& type, const std::string& name) { for (const auto& field : type->fields()) { @@ -61,6 +48,19 @@ std::shared_ptr FindField(const std::shared_ptr& return nullptr; } +/// Rebuilds `map_type` with new key and item types, keeping the name and metadata of its +/// entries field. +std::shared_ptr MakeMapType(const arrow::MapType& map_type, + const std::shared_ptr& key_field, + const std::shared_ptr& item_field) { + return std::make_shared( + map_type.value_field()->WithType(arrow::struct_({key_field, item_field})), + map_type.keys_sorted()); +} + +/// Returns the type to request from the file format plugin. A VECTOR is only read back as a +/// LIST when the file itself stores it as one: writers such as Paimon Java expose VECTOR +/// columns as Arrow LIST, while Paimon Rust and Python expose them as FixedSizeList. std::shared_ptr GetPhysicalReadType( const std::shared_ptr& logical_type, const std::shared_ptr& file_type) { @@ -100,52 +100,17 @@ std::shared_ptr GetPhysicalReadType( } const auto& map_type = checked_cast(*logical_type); const auto& file_map_type = checked_cast(*file_type); - return std::make_shared( - map_type.key_field()->WithType( - GetPhysicalReadType(map_type.key_type(), file_map_type.key_type())), - map_type.item_field()->WithType( - GetPhysicalReadType(map_type.item_type(), file_map_type.item_type())), - map_type.keys_sorted()); + return MakeMapType(map_type, + map_type.key_field()->WithType(GetPhysicalReadType( + map_type.key_type(), file_map_type.key_type())), + map_type.item_field()->WithType(GetPhysicalReadType( + map_type.item_type(), file_map_type.item_type()))); } default: return logical_type; } } -Status ValidateVectorElements(const arrow::ListArray& array) { - for (int64_t i = 0; i < array.length(); ++i) { - if (array.IsNull(i)) { - continue; - } - int64_t value_offset = array.value_offset(i); - int64_t value_length = array.value_length(i); - for (int64_t j = 0; j < value_length; ++j) { - if (array.values()->IsNull(value_offset + j)) { - return Status::Invalid(fmt::format( - "VECTOR cannot contain null elements, found one at row {} position {}", i, j)); - } - } - } - return Status::OK(); -} - -Status ValidateVectorElements(const arrow::FixedSizeListArray& array) { - const auto& vector_type = checked_cast(*array.type()); - for (int64_t i = 0; i < array.length(); ++i) { - if (array.IsNull(i)) { - continue; - } - int64_t value_offset = (array.offset() + i) * vector_type.list_size(); - for (int32_t j = 0; j < vector_type.list_size(); ++j) { - if (array.values()->IsNull(value_offset + j)) { - return Status::Invalid(fmt::format( - "VECTOR cannot contain null elements, found one at row {} position {}", i, j)); - } - } - } - return Status::OK(); -} - Result> CastListToVector( const std::shared_ptr& array, const std::shared_ptr& read_type, arrow::MemoryPool* pool) { @@ -153,7 +118,7 @@ Result> CastListToVector( return Status::Invalid( fmt::format("Cannot restore VECTOR from type {}", array->type()->ToString())); } - PAIMON_RETURN_NOT_OK(ValidateVectorElements(checked_cast(*array))); + PAIMON_RETURN_NOT_OK(VectorUtils::ValidateVectorElements(*array)); arrow::compute::ExecContext exec_context(pool); arrow::TypeHolder type_holder(read_type.get()); arrow::compute::CastOptions options = arrow::compute::CastOptions::Safe(); @@ -180,14 +145,14 @@ std::shared_ptr RebuildNestedType( const auto& entries_type = checked_cast(*children[0]->type); const auto& map_type = checked_cast(*read_type); - return std::make_shared(entries_type.field(0), entries_type.field(1), - map_type.keys_sorted()); + return MakeMapType(map_type, map_type.key_field()->WithType(entries_type.field(0)->type()), + map_type.item_field()->WithType(entries_type.field(1)->type())); } Result> ConvertToReadType( const std::shared_ptr& array, const std::shared_ptr& read_type, arrow::MemoryPool* pool) { - if (!ContainsVectorType(read_type)) { + if (!VectorUtils::ContainsVectorType(read_type)) { return array; } switch (read_type->id()) { @@ -202,8 +167,7 @@ Result> ConvertToReadType( source_type.ToString(), vector_type.ToString())); } - PAIMON_RETURN_NOT_OK( - ValidateVectorElements(checked_cast(*array))); + PAIMON_RETURN_NOT_OK(VectorUtils::ValidateVectorElements(*array)); return array; } return CastListToVector( @@ -249,12 +213,7 @@ VectorFileBatchReader::VectorFileBatchReader(std::unique_ptr&& : arrow_pool_(GetArrowPool(pool)), reader_(std::move(reader)) {} bool VectorFileBatchReader::ContainsVector(const std::shared_ptr& schema) { - for (const auto& field : schema->fields()) { - if (ContainsVectorType(field->type())) { - return true; - } - } - return false; + return VectorUtils::ContainsVector(schema); } Status VectorFileBatchReader::SetReadSchema( diff --git a/src/paimon/core/schema/schema_validation.cpp b/src/paimon/core/schema/schema_validation.cpp index 795b8d205..d4874f08e 100644 --- a/src/paimon/core/schema/schema_validation.cpp +++ b/src/paimon/core/schema/schema_validation.cpp @@ -38,6 +38,7 @@ #include "paimon/common/data/variant/variant_type_utils.h" #include "paimon/common/table/special_fields.h" #include "paimon/common/types/data_field.h" +#include "paimon/common/utils/arrow/vector_utils.h" #include "paimon/common/utils/checked_cast.h" #include "paimon/common/utils/object_utils.h" #include "paimon/common/utils/preconditions.h" @@ -76,18 +77,6 @@ bool ContainsBlobField(const std::shared_ptr& field) { return false; } -bool ContainsVectorField(const std::shared_ptr& field) { - if (field->type()->id() == arrow::Type::FIXED_SIZE_LIST) { - return true; - } - for (const auto& child : field->type()->fields()) { - if (ContainsVectorField(child)) { - return true; - } - } - return false; -} - Status ValidateSharedShreddingCompression(const std::string& option_key, const std::string& compression) { std::string normalized = StringUtils::ToLowerCase(compression); @@ -644,7 +633,7 @@ Status SchemaValidation::ValidateMapStorageLayout(const TableSchema& schema, if (ContainsBlobField(map_type->item_field())) { return Status::Invalid("MAP shared-shredding currently cannot contain BLOB fields."); } - if (ContainsVectorField(map_type->item_field())) { + if (VectorUtils::ContainsVectorField(map_type->item_field())) { return Status::Invalid("MAP shared-shredding currently cannot contain VECTOR fields."); } // Validate max-columns config @@ -677,7 +666,7 @@ Status SchemaValidation::ValidateVectorFields(const TableSchema& schema, const CoreOptions& options) { bool has_vector = false; for (const auto& field : schema.Fields()) { - if (ContainsVectorField(field.ArrowField())) { + if (VectorUtils::ContainsVectorField(field.ArrowField())) { has_vector = true; break; } diff --git a/src/paimon/format/parquet/parquet_vector_converter.cpp b/src/paimon/format/parquet/parquet_vector_converter.cpp index df6a10f3d..b5e4acf1e 100644 --- a/src/paimon/format/parquet/parquet_vector_converter.cpp +++ b/src/paimon/format/parquet/parquet_vector_converter.cpp @@ -18,63 +18,17 @@ #include "paimon/format/parquet/parquet_vector_converter.h" -#include #include #include "arrow/array.h" -#include "arrow/array/array_nested.h" #include "arrow/compute/api.h" #include "arrow/type.h" #include "paimon/common/utils/arrow/status_utils.h" +#include "paimon/common/utils/arrow/vector_utils.h" #include "paimon/common/utils/checked_cast.h" #include "paimon/status.h" namespace paimon::parquet { -namespace { - -bool ContainsVectorType(const std::shared_ptr& type) { - if (type->id() == arrow::Type::FIXED_SIZE_LIST) { - return true; - } - for (const auto& field : type->fields()) { - if (ContainsVectorType(field->type())) { - return true; - } - } - return false; -} - -Status ValidateVectorElements(const std::shared_ptr& array) { - if (!ContainsVectorType(array->type())) { - return Status::OK(); - } - if (array->type_id() == arrow::Type::FIXED_SIZE_LIST) { - const auto& vector_array = checked_cast(*array); - const auto& vector_type = checked_cast(*array->type()); - const std::shared_ptr& values = vector_array.values(); - if (values->null_count() == 0) { - return Status::OK(); - } - for (int64_t i = 0; i < vector_array.length(); ++i) { - if (vector_array.IsNull(i)) { - continue; - } - int64_t value_offset = (vector_array.offset() + i) * vector_type.list_size(); - for (int32_t j = 0; j < vector_type.list_size(); ++j) { - if (values->IsNull(value_offset + j)) { - return Status::Invalid("VECTOR cannot contain null elements"); - } - } - } - return Status::OK(); - } - for (const auto& child_data : array->data()->child_data) { - PAIMON_RETURN_NOT_OK(ValidateVectorElements(arrow::MakeArray(child_data))); - } - return Status::OK(); -} - -} // namespace std::shared_ptr ParquetVectorConverter::GetWriteType( const std::shared_ptr& logical_type) { @@ -98,8 +52,11 @@ std::shared_ptr ParquetVectorConverter::GetWriteType( case arrow::Type::MAP: { const auto& map_type = checked_cast(*logical_type); return std::make_shared( - map_type.key_field()->WithType(GetWriteType(map_type.key_type())), - map_type.item_field()->WithType(GetWriteType(map_type.item_type())), + map_type.value_field()->WithType( + arrow::struct_({map_type.key_field()->WithType( + GetWriteType(map_type.key_type())), + map_type.item_field()->WithType( + GetWriteType(map_type.item_type()))})), map_type.keys_sorted()); } default: @@ -109,10 +66,10 @@ std::shared_ptr ParquetVectorConverter::GetWriteType( Result> ParquetVectorConverter::ConvertToWriteType( const std::shared_ptr& array, arrow::MemoryPool* pool) { - if (!ContainsVectorType(array->type())) { + if (!VectorUtils::ContainsVectorType(array->type())) { return array; } - PAIMON_RETURN_NOT_OK(ValidateVectorElements(array)); + PAIMON_RETURN_NOT_OK(VectorUtils::ValidateNestedVectorElements(array)); std::shared_ptr write_type = GetWriteType(array->type()); arrow::compute::ExecContext exec_context(pool); arrow::TypeHolder type_holder(write_type.get()); From f70c46c1293f45eb2e2d33d597a52b4fc8db65d3 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Mon, 17 Aug 2026 12:10:33 +0000 Subject: [PATCH 06/16] perf(common): rebase VECTOR offsets by slicing buffers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `NormalizeRecordBatchOffsets` had no FixedSizeList branch, so every sliced VECTOR column fell back to a full copy of its values. Slice the child instead, the way the list and struct layouts already do, since a contiguous slice of the parent always spans a contiguous range of the child. Co-authored-by: 小明同学 --- src/paimon/common/utils/arrow/arrow_utils.cpp | 24 +++++++++++++++++++ .../common/utils/arrow/arrow_utils_test.cpp | 3 +++ 2 files changed, 27 insertions(+) diff --git a/src/paimon/common/utils/arrow/arrow_utils.cpp b/src/paimon/common/utils/arrow/arrow_utils.cpp index c301fd5bb..f29e1d11e 100644 --- a/src/paimon/common/utils/arrow/arrow_utils.cpp +++ b/src/paimon/common/utils/arrow/arrow_utils.cpp @@ -161,6 +161,28 @@ Result> RebaseListLike( return rebased; } +/// Rebases a fixed size list array, whose child holds `list_size` values per row. +Result> RebaseFixedSizeList( + const std::shared_ptr& data, arrow::MemoryPool* pool) { + if (data->child_data.size() != 1) { + return CopyToZeroOffset(data, pool); + } + const int64_t list_size = + checked_cast(*data->type).list_size(); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr validity, + RebaseValidityBitmap(*data, pool)); + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW( + std::shared_ptr child_slice, + data->child_data[0]->SliceSafe(data->offset * list_size, data->length * list_size)); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr child, + RebaseToZeroOffset(child_slice, pool)); + std::shared_ptr rebased = + arrow::ArrayData::Make(data->type, data->length, data->null_count.load(), /*offset=*/0); + rebased->buffers = {std::move(validity)}; + rebased->child_data = {std::move(child)}; + return rebased; +} + /// Rebases a struct array, whose slices keep full length children. Result> RebaseStruct( const std::shared_ptr& data, arrow::MemoryPool* pool) { @@ -234,6 +256,8 @@ Result> RebaseToZeroOffset( return RebaseListLike(data, pool); case arrow::Type::LARGE_LIST: return RebaseListLike(data, pool); + case arrow::Type::FIXED_SIZE_LIST: + return RebaseFixedSizeList(data, pool); case arrow::Type::STRUCT: return RebaseStruct(data, pool); case arrow::Type::DICTIONARY: diff --git a/src/paimon/common/utils/arrow/arrow_utils_test.cpp b/src/paimon/common/utils/arrow/arrow_utils_test.cpp index 571326989..4e1fdaa0c 100644 --- a/src/paimon/common/utils/arrow/arrow_utils_test.cpp +++ b/src/paimon/common/utils/arrow/arrow_utils_test.cpp @@ -559,6 +559,9 @@ std::vector NormalizeCases() { {arrow::list(arrow::utf8()), R"([["a"], null, ["bb", "ccc"], [], ["d"], null, ["e", "f"], [], ["g"], ["h"]])", {{{0}, 2}}}, + {arrow::fixed_size_list(arrow::int32(), 2), + "[[0, 1], null, [2, 3], [4, 5], [6, 7], null, [8, 9], [10, 11], [12, 13], [14, 15]]", + {{{0}, 1}}}, {arrow::struct_({int_field, text_field}), R"([{"a": 0, "b": "x"}, null, {"a": 2, "b": null}, {"a": null, "b": "yyy"}, {"a": 4, "b": "z"}, {"a": 5, "b": ""}, null, {"a": 7, "b": "w"}, From 2729d30831fc246a6979dcc8c9419788ca58a84e Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Mon, 17 Aug 2026 12:10:41 +0000 Subject: [PATCH 07/16] fix(parquet): handle VECTOR columns in nested read paths MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A file written by Paimon Rust or Python exposes a VECTOR column as Arrow FixedSizeList, so the read schema handed to the Parquet plugin keeps that type too. `HasSameNestedProjectionShape` had no FixedSizeList branch and rejected such a schema as a partial projection inside list/map, and `CollectLeafIndices` and `SkipLeafIndices` treated a FixedSizeList as a leaf column. `FieldMappingBuilder::CreateDataCastExecutors` also has to exempt FixedSizeList from the scalar cast lookup, like the other nested types, because the reader reshapes those columns itself. Co-authored-by: 小明同学 --- src/paimon/core/utils/field_mapping.cpp | 3 +- .../parquet/parquet_file_batch_reader.cpp | 21 ++- .../format/parquet/parquet_vector_io_test.cpp | 155 ++++++++++++++++-- 3 files changed, 158 insertions(+), 21 deletions(-) diff --git a/src/paimon/core/utils/field_mapping.cpp b/src/paimon/core/utils/field_mapping.cpp index 447d8d589..28847d0cd 100644 --- a/src/paimon/core/utils/field_mapping.cpp +++ b/src/paimon/core/utils/field_mapping.cpp @@ -185,7 +185,8 @@ Result>> FieldMappingBuilder::CreateDa if (!read_fields[i].Type()->Equals(data_fields[i].Type())) { auto read_type_id = read_fields[i].Type()->id(); if (read_type_id == arrow::Type::STRUCT || read_type_id == arrow::Type::LIST || - read_type_id == arrow::Type::MAP) { + read_type_id == arrow::Type::MAP || + read_type_id == arrow::Type::FIXED_SIZE_LIST) { // Nested type differs by pruning/evolution; the reader's reshape // handles it, no scalar cast. cast_executors.push_back(nullptr); diff --git a/src/paimon/format/parquet/parquet_file_batch_reader.cpp b/src/paimon/format/parquet/parquet_file_batch_reader.cpp index c0cd40e19..fcedd8a76 100644 --- a/src/paimon/format/parquet/parquet_file_batch_reader.cpp +++ b/src/paimon/format/parquet/parquet_file_batch_reader.cpp @@ -40,6 +40,7 @@ #include "fmt/format.h" #include "paimon/common/metrics/metrics_impl.h" #include "paimon/common/utils/arrow/status_utils.h" +#include "paimon/common/utils/checked_cast.h" #include "paimon/common/utils/options_utils.h" #include "paimon/common/utils/scope_guard.h" #include "paimon/common/utils/string_utils.h" @@ -113,6 +114,13 @@ bool HasSameNestedProjectionShape(const std::shared_ptr& read_t const auto& file_list = static_cast(*file_type); return HasSameNestedProjectionShape(read_list.value_type(), file_list.value_type()); } + case arrow::Type::FIXED_SIZE_LIST: { + const auto& read_vector = checked_cast(*read_type); + const auto& file_vector = checked_cast(*file_type); + return read_vector.list_size() == file_vector.list_size() && + HasSameNestedProjectionShape(read_vector.value_type(), + file_vector.value_type()); + } case arrow::Type::MAP: { const auto& read_map = static_cast(*read_type); const auto& file_map = static_cast(*file_type); @@ -740,6 +748,16 @@ Status ParquetFileBatchReader::CollectLeafIndices(const std::shared_ptr(*file_type); PAIMON_RETURN_NOT_OK(CollectLeafIndices(read_list.value_type(), file_list.value_type(), leaf_index, indices)); + } else if (file_type->id() == arrow::Type::FIXED_SIZE_LIST) { + if (!HasSameNestedProjectionShape(read_type, file_type)) { + return Status::Invalid(fmt::format( + "Parquet does not support partial projection inside list/map: src {} vs target {}", + file_type->ToString(), read_type->ToString())); + } + const auto& read_vector = checked_cast(*read_type); + const auto& file_vector = checked_cast(*file_type); + PAIMON_RETURN_NOT_OK(CollectLeafIndices(read_vector.value_type(), file_vector.value_type(), + leaf_index, indices)); } else if (file_type->id() == arrow::Type::MAP) { if (!HasSameNestedProjectionShape(read_type, file_type)) { return Status::Invalid(fmt::format( @@ -761,8 +779,7 @@ Status ParquetFileBatchReader::CollectLeafIndices(const std::shared_ptr& file_type, int32_t* leaf_index) { - if (file_type->id() == arrow::Type::STRUCT || file_type->id() == arrow::Type::LIST || - file_type->id() == arrow::Type::MAP) { + if (ArrowSchemaValidator::IsNestedType(file_type)) { for (int32_t i = 0; i < file_type->num_fields(); i++) { SkipLeafIndices(file_type->field(i)->type(), leaf_index); } diff --git a/src/paimon/format/parquet/parquet_vector_io_test.cpp b/src/paimon/format/parquet/parquet_vector_io_test.cpp index 8da61e47c..c263043cf 100644 --- a/src/paimon/format/parquet/parquet_vector_io_test.cpp +++ b/src/paimon/format/parquet/parquet_vector_io_test.cpp @@ -29,18 +29,27 @@ #include "arrow/ipc/json_simple.h" #include "gtest/gtest.h" #include "paimon/common/utils/arrow/arrow_input_stream_adapter.h" +#include "paimon/common/utils/arrow/arrow_output_stream_adapter.h" #include "paimon/common/utils/arrow/mem_utils.h" #include "paimon/common/utils/checked_cast.h" #include "paimon/core/io/vector_file_batch_reader.h" +#include "paimon/defs.h" #include "paimon/format/parquet/parquet_file_batch_reader.h" #include "paimon/format/parquet/parquet_format_defs.h" #include "paimon/format/parquet/parquet_format_writer.h" #include "paimon/fs/file_system.h" #include "paimon/memory/memory_pool.h" +#include "paimon/predicate/literal.h" +#include "paimon/predicate/predicate_builder.h" #include "paimon/testing/utils/read_result_collector.h" #include "paimon/testing/utils/testharness.h" +#include "parquet/arrow/writer.h" #include "parquet/properties.h" +namespace paimon { +class Predicate; +} // namespace paimon + namespace paimon::parquet::test { class ParquetVectorIoTest : public ::testing::Test { @@ -57,25 +66,8 @@ class ParquetVectorIoTest : public ::testing::Test { const std::shared_ptr& write_type, const std::shared_ptr& read_type, const std::string& json) { - arrow::Result> write_array_result = - arrow::ipc::internal::json::ArrayFromJSON(write_type, json); - ASSERT_TRUE(write_array_result.ok()) << write_array_result.status().ToString(); - std::shared_ptr write_array = std::move(write_array_result).ValueOrDie(); - auto c_array = std::make_unique(); - ASSERT_TRUE(arrow::ExportArray(*write_array, c_array.get()).ok()); - std::string file_path = dir_->Str() + "/" + file_name; - ASSERT_OK_AND_ASSIGN(std::shared_ptr out, - fs_->Create(file_path, /*overwrite=*/false)); - ::parquet::WriterProperties::Builder properties_builder; - ASSERT_OK_AND_ASSIGN( - std::unique_ptr writer, - ParquetFormatWriter::Create(out, arrow::schema(write_type->fields()), - properties_builder.build(), - DEFAULT_PARQUET_WRITER_MAX_MEMORY_USE, arrow_pool_)); - ASSERT_OK(writer->AddBatch(c_array.get())); - ASSERT_OK(writer->Finish()); - ASSERT_OK(out->Close()); + WriteWithFormatWriter(file_path, write_type, json, /*max_row_group_length=*/1024); ASSERT_OK_AND_ASSIGN(std::shared_ptr in, fs_->Open(file_path)); ASSERT_OK_AND_ASSIGN(int64_t length, in->Length()); @@ -114,6 +106,79 @@ class ParquetVectorIoTest : public ::testing::Test { << actual->ToString(); } + /// Writes the JSON rows through the Paimon Parquet writer, which stores VECTOR values as + /// Parquet LIST. + void WriteWithFormatWriter(const std::string& file_path, + const std::shared_ptr& write_type, + const std::string& json, int64_t max_row_group_length) { + arrow::Result> write_array_result = + arrow::ipc::internal::json::ArrayFromJSON(write_type, json); + ASSERT_TRUE(write_array_result.ok()) << write_array_result.status().ToString(); + std::shared_ptr write_array = std::move(write_array_result).ValueOrDie(); + auto c_array = std::make_unique(); + ASSERT_TRUE(arrow::ExportArray(*write_array, c_array.get()).ok()); + + ASSERT_OK_AND_ASSIGN(std::shared_ptr out, + fs_->Create(file_path, /*overwrite=*/false)); + ::parquet::WriterProperties::Builder properties_builder; + properties_builder.max_row_group_length(max_row_group_length); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr writer, + ParquetFormatWriter::Create(out, arrow::schema(write_type->fields()), + properties_builder.build(), + DEFAULT_PARQUET_WRITER_MAX_MEMORY_USE, arrow_pool_)); + ASSERT_OK(writer->AddBatch(c_array.get())); + ASSERT_OK(writer->Finish()); + ASSERT_OK(out->Close()); + } + + /// Writes `array` with the plain Arrow Parquet writer, so FixedSizeList columns keep their + /// Arrow type in the file schema the way Paimon Rust and Python writers store them. + void WriteWithArrowWriter(const std::string& file_path, + const std::shared_ptr& type, + const std::string& json) { + arrow::Result> array_result = + arrow::ipc::internal::json::ArrayFromJSON(type, json); + ASSERT_TRUE(array_result.ok()) << array_result.status().ToString(); + arrow::Result> batch_result = + arrow::RecordBatch::FromStructArray(std::move(array_result).ValueOrDie()); + ASSERT_TRUE(batch_result.ok()) << batch_result.status().ToString(); + arrow::Result> table_result = + arrow::Table::FromRecordBatches({std::move(batch_result).ValueOrDie()}); + ASSERT_TRUE(table_result.ok()) << table_result.status().ToString(); + + ASSERT_OK_AND_ASSIGN(std::shared_ptr out, + fs_->Create(file_path, /*overwrite=*/false)); + auto arrow_out = std::make_shared(out); + ::parquet::WriterProperties::Builder properties_builder; + arrow::Status status = ::parquet::arrow::WriteTable( + *std::move(table_result).ValueOrDie(), arrow_pool_.get(), arrow_out, + /*chunk_size=*/1024, properties_builder.build()); + ASSERT_TRUE(status.ok()) << status.ToString(); + ASSERT_OK(out->Close()); + } + + std::unique_ptr CreateVectorReader( + const std::string& file_path, const std::shared_ptr& read_schema, + const std::shared_ptr& predicate, + const std::map& options, int32_t batch_size) { + std::unique_ptr vector_reader; + ASSERT_OK_AND_ASSIGN(std::shared_ptr in, fs_->Open(file_path)); + ASSERT_OK_AND_ASSIGN(int64_t length, in->Length()); + auto in_stream = std::make_shared(in, length, arrow_pool_); + ASSERT_OK_AND_ASSIGN(std::unique_ptr reader, + ParquetFileBatchReader::Create(std::move(in_stream), options, + batch_size, /*file_metadata=*/nullptr, + /*storage_read_bytes=*/nullptr, + arrow_pool_)); + vector_reader = std::make_unique(std::move(reader), pool_); + auto c_schema = std::make_unique(); + ASSERT_TRUE(arrow::ExportSchema(*read_schema, c_schema.get()).ok()); + ASSERT_OK(vector_reader->SetReadSchema(c_schema.get(), predicate, + /*selection_bitmap=*/std::nullopt)); + return vector_reader; + } + void ReadFixtureAndCheck( const std::string& file_name, arrow::Type::type expected_file_vector_type, int32_t vector_length, const std::vector& expected_ids, @@ -226,6 +291,60 @@ TEST_F(ParquetVectorIoTest, WriteAndReadNestedDoubleVector) { [2, [null, null, [["b", null]]]]])"); } +// Vectors nested in a LIST keep their Arrow type when a third-party writer stores them as +// FixedSizeList, so the Parquet reader must accept a FixedSizeList read type as well. +TEST_F(ParquetVectorIoTest, ReadNestedFixedSizeListFile) { + auto vector_type = + arrow::fixed_size_list(arrow::field("item", arrow::float32(), /*nullable=*/false), 3); + auto logical_type = checked_pointer_cast(arrow::struct_({ + arrow::field("id", arrow::int32()), + arrow::field("history", arrow::list(vector_type)), + })); + const std::string json = R"([[1, [[1.0, 2.0, 3.0], null]], [2, null]])"; + std::string file_path = dir_->Str() + "/nested-fixed-size-list.parquet"; + WriteWithArrowWriter(file_path, logical_type, json); + + std::unique_ptr reader = + CreateVectorReader(file_path, arrow::schema(logical_type->fields()), /*predicate=*/nullptr, + /*options=*/{}, /*batch_size=*/10); + ASSERT_OK_AND_ASSIGN(std::shared_ptr actual, + paimon::test::ReadResultCollector::CollectResult(reader.get())); + arrow::Result> expected_result = + arrow::ipc::internal::json::ArrayFromJSON(logical_type, json); + ASSERT_TRUE(expected_result.ok()) << expected_result.status().ToString(); + ASSERT_TRUE(std::make_shared(std::move(expected_result).ValueOrDie()) + ->Equals(actual)) + << actual->ToString(); +} + +TEST_F(ParquetVectorIoTest, ReadVectorWithPredicatePushdown) { + auto vector_type = + arrow::fixed_size_list(arrow::field("item", arrow::float32(), /*nullable=*/false), 3); + auto logical_type = checked_pointer_cast(arrow::struct_( + {arrow::field("id", arrow::int32()), arrow::field("embedding", vector_type)})); + // One row per row group, so the predicate on `id` prunes row groups while reading. + std::string file_path = dir_->Str() + "/vector-predicate.parquet"; + WriteWithFormatWriter(file_path, logical_type, + R"([[1, [1.0, 2.0, 3.0]], [2, null], [3, [4.0, 5.0, 6.0]], + [4, [7.0, 8.0, 9.0]]])", + /*max_row_group_length=*/1); + + std::shared_ptr predicate = PredicateBuilder::GreaterThan( + /*field_index=*/0, /*field_name=*/"id", FieldType::INT, Literal(2)); + std::unique_ptr reader = + CreateVectorReader(file_path, arrow::schema(logical_type->fields()), predicate, + /*options=*/{}, /*batch_size=*/10); + ASSERT_OK_AND_ASSIGN(std::shared_ptr actual, + paimon::test::ReadResultCollector::CollectResult(reader.get())); + arrow::Result> expected_result = + arrow::ipc::internal::json::ArrayFromJSON( + logical_type, R"([[3, [4.0, 5.0, 6.0]], [4, [7.0, 8.0, 9.0]]])"); + ASSERT_TRUE(expected_result.ok()) << expected_result.status().ToString(); + ASSERT_TRUE(std::make_shared(std::move(expected_result).ValueOrDie()) + ->Equals(actual)) + << actual->ToString(); +} + TEST_F(ParquetVectorIoTest, ReadJavaFixture) { ReadFixtureAndCheck( "java_vector.parquet", arrow::Type::LIST, /*vector_length=*/2, From 6c1235d3197fc13a832c714f0457bcf2c279f80b Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Mon, 17 Aug 2026 12:10:48 +0000 Subject: [PATCH 08/16] style(parquet): apply VECTOR review nits MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Move the `needs_vector_conversion` writer parameter before the memory pool, build VECTOR test arrays from JSON instead of Arrow builders, and drop the license header from the VECTOR fixture notes. Co-authored-by: 小明同学 --- .../core/io/vector_file_batch_reader_test.cpp | 18 +++++------------- .../format/parquet/parquet_format_writer.cpp | 10 +++++----- .../format/parquet/parquet_format_writer.h | 4 ++-- .../parquet/parquet_vector_converter_test.cpp | 13 ++++--------- .../parquet/vector_compatibility/README.md | 19 ------------------- 5 files changed, 16 insertions(+), 48 deletions(-) diff --git a/src/paimon/core/io/vector_file_batch_reader_test.cpp b/src/paimon/core/io/vector_file_batch_reader_test.cpp index ed7ef38da..66d118519 100644 --- a/src/paimon/core/io/vector_file_batch_reader_test.cpp +++ b/src/paimon/core/io/vector_file_batch_reader_test.cpp @@ -173,19 +173,11 @@ TEST(VectorFileBatchReaderTest, RejectInvalidVectorValues) { } TEST(VectorFileBatchReaderTest, RejectInvalidFixedSizeListVectorValues) { - auto values_builder = std::make_shared(); - arrow::FixedSizeListBuilder vector_builder(arrow::default_memory_pool(), values_builder, 3); - ASSERT_TRUE(values_builder->Append(1.0f).ok()); - ASSERT_TRUE(values_builder->AppendNull().ok()); - ASSERT_TRUE(values_builder->Append(3.0f).ok()); - ASSERT_TRUE(vector_builder.Append().ok()); - std::shared_ptr vector_array; - ASSERT_TRUE(vector_builder.Finish(&vector_array).ok()); - arrow::Result> struct_result = - arrow::StructArray::Make({vector_array}, {"embedding"}); - ASSERT_TRUE(struct_result.ok()) << struct_result.status().ToString(); - std::shared_ptr physical_array = std::move(struct_result).ValueOrDie(); - auto physical_type = AsStructType(physical_array->type()); + auto physical_type = AsStructType( + arrow::struct_({arrow::field("embedding", arrow::fixed_size_list(arrow::float32(), 3))})); + auto physical_array = + arrow::ipc::internal::json::ArrayFromJSON(physical_type, R"([[[1.0, null, 3.0]]])") + .ValueOrDie(); auto mock_reader = std::make_unique(physical_array, physical_type, /*read_batch_size=*/10); mock_reader->EnableRandomizeBatchSize(false); diff --git a/src/paimon/format/parquet/parquet_format_writer.cpp b/src/paimon/format/parquet/parquet_format_writer.cpp index 339ed2147..dc0947913 100644 --- a/src/paimon/format/parquet/parquet_format_writer.cpp +++ b/src/paimon/format/parquet/parquet_format_writer.cpp @@ -67,8 +67,9 @@ Result> ParquetFormatWriter::Create( ::parquet::arrow::FileWriter::Open(*write_schema, pool.get(), out, writer_properties, arrow_writer_properties)); return std::unique_ptr( - new ParquetFormatWriter(std::move(file_writer), out, schema, max_memory_use, pool, - !logical_type->Equals(write_type))); + new ParquetFormatWriter(std::move(file_writer), out, schema, max_memory_use, + /*needs_vector_conversion=*/!logical_type->Equals(write_type), + pool)); } Status ParquetFormatWriter::AddBatch(ArrowArray* batch) { @@ -132,9 +133,8 @@ Result ParquetFormatWriter::GetEstimateLength() const { ParquetFormatWriter::ParquetFormatWriter(std::unique_ptr<::parquet::arrow::FileWriter> writer, const std::shared_ptr& out, const std::shared_ptr& schema, - uint64_t max_memory_use, - const std::shared_ptr& pool, - bool needs_vector_conversion) + uint64_t max_memory_use, bool needs_vector_conversion, + const std::shared_ptr& pool) : pool_(pool), out_(out), writer_(std::move(writer)), diff --git a/src/paimon/format/parquet/parquet_format_writer.h b/src/paimon/format/parquet/parquet_format_writer.h index 3d5956d66..f8f441195 100644 --- a/src/paimon/format/parquet/parquet_format_writer.h +++ b/src/paimon/format/parquet/parquet_format_writer.h @@ -72,8 +72,8 @@ class ParquetFormatWriter : public FormatWriter { ParquetFormatWriter(std::unique_ptr<::parquet::arrow::FileWriter> writer, const std::shared_ptr& out, const std::shared_ptr& schema, uint64_t max_memory_use, - const std::shared_ptr& pool, - bool needs_vector_conversion); + bool needs_vector_conversion, + const std::shared_ptr& pool); Result GetEstimateLength() const; diff --git a/src/paimon/format/parquet/parquet_vector_converter_test.cpp b/src/paimon/format/parquet/parquet_vector_converter_test.cpp index 602fab08a..c6653ceaa 100644 --- a/src/paimon/format/parquet/parquet_vector_converter_test.cpp +++ b/src/paimon/format/parquet/parquet_vector_converter_test.cpp @@ -29,15 +29,10 @@ namespace paimon::parquet::test { TEST(ParquetVectorConverterTest, ConvertNullableVectorToList) { - auto values_builder = std::make_shared(); - arrow::FixedSizeListBuilder vector_builder(arrow::default_memory_pool(), values_builder, 3); - ASSERT_TRUE(values_builder->AppendValues({1.0f, 2.0f, 3.0f}).ok()); - ASSERT_TRUE(vector_builder.Append().ok()); - ASSERT_TRUE(vector_builder.AppendNull().ok()); - ASSERT_TRUE(values_builder->AppendValues({4.0f, 5.0f, 6.0f}).ok()); - ASSERT_TRUE(vector_builder.Append().ok()); - std::shared_ptr vector_array; - ASSERT_TRUE(vector_builder.Finish(&vector_array).ok()); + auto vector_type = arrow::fixed_size_list(arrow::float32(), 3); + auto vector_array = arrow::ipc::internal::json::ArrayFromJSON( + vector_type, R"([[1.0, 2.0, 3.0], null, [4.0, 5.0, 6.0]])") + .ValueOrDie(); ASSERT_OK_AND_ASSIGN( std::shared_ptr converted, diff --git a/test/test_data/parquet/vector_compatibility/README.md b/test/test_data/parquet/vector_compatibility/README.md index 8aad4adfe..4ffae4e0b 100644 --- a/test/test_data/parquet/vector_compatibility/README.md +++ b/test/test_data/parquet/vector_compatibility/README.md @@ -1,22 +1,3 @@ - - # VECTOR Parquet compatibility fixtures These files pin the two physical Arrow schemas produced by Java and Rust writers for Paimon From 8d68ae0d373551393ff25b2c3c7e66b6a854f748 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Mon, 17 Aug 2026 12:34:29 +0000 Subject: [PATCH 09/16] style: apply clang-format to VECTOR changes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reformat the lines added by the previous commits with the clang-format version pinned in .pre-commit-config.yaml, and pass the reader helper in the Parquet VECTOR test through an out parameter so it can use the ASSERT_* macros. Co-authored-by: 小明同学 --- .../common/utils/arrow/vector_utils.cpp | 4 +- .../common/utils/arrow/vector_utils_test.cpp | 30 +++++++-------- src/paimon/core/utils/field_mapping.cpp | 3 +- .../parquet/parquet_file_batch_reader.cpp | 3 +- .../format/parquet/parquet_format_writer.cpp | 7 ++-- .../parquet/parquet_vector_converter.cpp | 8 ++-- .../format/parquet/parquet_vector_io_test.cpp | 37 ++++++++++--------- 7 files changed, 42 insertions(+), 50 deletions(-) diff --git a/src/paimon/common/utils/arrow/vector_utils.cpp b/src/paimon/common/utils/arrow/vector_utils.cpp index 620373603..7046e0d74 100644 --- a/src/paimon/common/utils/arrow/vector_utils.cpp +++ b/src/paimon/common/utils/arrow/vector_utils.cpp @@ -120,8 +120,8 @@ Status VectorUtils::ValidateVectorElements(const arrow::Array& array) { return ValidateFixedSizeListVector( checked_cast(array)); default: - return Status::Invalid(fmt::format("Cannot validate VECTOR values of type {}", - array.type()->ToString())); + return Status::Invalid( + fmt::format("Cannot validate VECTOR values of type {}", array.type()->ToString())); } } diff --git a/src/paimon/common/utils/arrow/vector_utils_test.cpp b/src/paimon/common/utils/arrow/vector_utils_test.cpp index 131f81638..063056f37 100644 --- a/src/paimon/common/utils/arrow/vector_utils_test.cpp +++ b/src/paimon/common/utils/arrow/vector_utils_test.cpp @@ -44,8 +44,7 @@ TEST(VectorUtilsTest, TestContainsVector) { ASSERT_TRUE(VectorUtils::ContainsVectorType(vector_type)); ASSERT_TRUE(VectorUtils::ContainsVectorType(arrow::list(vector_type))); ASSERT_TRUE(VectorUtils::ContainsVectorType(arrow::map(arrow::utf8(), vector_type))); - ASSERT_TRUE( - VectorUtils::ContainsVectorType(arrow::struct_({arrow::field("v", vector_type)}))); + ASSERT_TRUE(VectorUtils::ContainsVectorType(arrow::struct_({arrow::field("v", vector_type)}))); ASSERT_FALSE(VectorUtils::ContainsVectorType(arrow::list(arrow::float32()))); ASSERT_FALSE(VectorUtils::ContainsVectorType(nullptr)); @@ -63,10 +62,9 @@ TEST(VectorUtilsTest, TestValidateVectorElements) { auto vector_type = arrow::fixed_size_list(arrow::float32(), 3); ASSERT_OK(VectorUtils::ValidateVectorElements( *ArrayFromJSON(vector_type, R"([[1.0, 2.0, 3.0], null, [4.0, 5.0, 6.0]])"))); - ASSERT_NOK_WITH_MSG( - VectorUtils::ValidateVectorElements( - *ArrayFromJSON(vector_type, R"([[1.0, 2.0, 3.0], [4.0, null, 6.0]])")), - "VECTOR cannot contain null elements, found one at row 1 position 1"); + ASSERT_NOK_WITH_MSG(VectorUtils::ValidateVectorElements( + *ArrayFromJSON(vector_type, R"([[1.0, 2.0, 3.0], [4.0, null, 6.0]])")), + "VECTOR cannot contain null elements, found one at row 1 position 1"); // A sliced array must be validated against its own rows only. std::shared_ptr sliced = @@ -91,9 +89,9 @@ TEST(VectorUtilsTest, TestValidateVectorElements) { TEST(VectorUtilsTest, TestValidateVectorElementsRejectsTruncatedValues) { auto vector_type = arrow::fixed_size_list(arrow::float32(), 3); std::shared_ptr values = ArrayFromJSON(arrow::float32(), "[1.0, null, 3.0]"); - auto truncated = arrow::MakeArray( - arrow::ArrayData::Make(vector_type, /*length=*/2, {nullptr}, {values->data()}, - /*null_count=*/0)); + auto truncated = arrow::MakeArray(arrow::ArrayData::Make(vector_type, /*length=*/2, {nullptr}, + {values->data()}, + /*null_count=*/0)); ASSERT_NOK_WITH_MSG(VectorUtils::ValidateVectorElements(*truncated), "VECTOR holds 3 elements while 2 rows of dimension 3 require 6"); @@ -108,14 +106,12 @@ TEST(VectorUtilsTest, TestValidateNestedVectorElements) { }); ASSERT_OK(VectorUtils::ValidateNestedVectorElements( ArrayFromJSON(nested_type, R"([[1, [[1.0, 2.0], null], [["a", [3.0, 4.0]]]]])"))); - ASSERT_NOK_WITH_MSG( - VectorUtils::ValidateNestedVectorElements( - ArrayFromJSON(nested_type, R"([[1, [[1.0, null]], [["a", [3.0, 4.0]]]]])")), - "VECTOR cannot contain null elements"); - ASSERT_NOK_WITH_MSG( - VectorUtils::ValidateNestedVectorElements( - ArrayFromJSON(nested_type, R"([[1, [[1.0, 2.0]], [["a", [null, 4.0]]]]])")), - "VECTOR cannot contain null elements"); + ASSERT_NOK_WITH_MSG(VectorUtils::ValidateNestedVectorElements(ArrayFromJSON( + nested_type, R"([[1, [[1.0, null]], [["a", [3.0, 4.0]]]]])")), + "VECTOR cannot contain null elements"); + ASSERT_NOK_WITH_MSG(VectorUtils::ValidateNestedVectorElements(ArrayFromJSON( + nested_type, R"([[1, [[1.0, 2.0]], [["a", [null, 4.0]]]]])")), + "VECTOR cannot contain null elements"); ASSERT_OK(VectorUtils::ValidateNestedVectorElements( ArrayFromJSON(arrow::list(arrow::float32()), R"([[1.0, null]])"))); ASSERT_OK(VectorUtils::ValidateNestedVectorElements(nullptr)); diff --git a/src/paimon/core/utils/field_mapping.cpp b/src/paimon/core/utils/field_mapping.cpp index 28847d0cd..be7287dd6 100644 --- a/src/paimon/core/utils/field_mapping.cpp +++ b/src/paimon/core/utils/field_mapping.cpp @@ -185,8 +185,7 @@ Result>> FieldMappingBuilder::CreateDa if (!read_fields[i].Type()->Equals(data_fields[i].Type())) { auto read_type_id = read_fields[i].Type()->id(); if (read_type_id == arrow::Type::STRUCT || read_type_id == arrow::Type::LIST || - read_type_id == arrow::Type::MAP || - read_type_id == arrow::Type::FIXED_SIZE_LIST) { + read_type_id == arrow::Type::MAP || read_type_id == arrow::Type::FIXED_SIZE_LIST) { // Nested type differs by pruning/evolution; the reader's reshape // handles it, no scalar cast. cast_executors.push_back(nullptr); diff --git a/src/paimon/format/parquet/parquet_file_batch_reader.cpp b/src/paimon/format/parquet/parquet_file_batch_reader.cpp index fcedd8a76..487e81ebf 100644 --- a/src/paimon/format/parquet/parquet_file_batch_reader.cpp +++ b/src/paimon/format/parquet/parquet_file_batch_reader.cpp @@ -118,8 +118,7 @@ bool HasSameNestedProjectionShape(const std::shared_ptr& read_t const auto& read_vector = checked_cast(*read_type); const auto& file_vector = checked_cast(*file_type); return read_vector.list_size() == file_vector.list_size() && - HasSameNestedProjectionShape(read_vector.value_type(), - file_vector.value_type()); + HasSameNestedProjectionShape(read_vector.value_type(), file_vector.value_type()); } case arrow::Type::MAP: { const auto& read_map = static_cast(*read_type); diff --git a/src/paimon/format/parquet/parquet_format_writer.cpp b/src/paimon/format/parquet/parquet_format_writer.cpp index dc0947913..6e69e6945 100644 --- a/src/paimon/format/parquet/parquet_format_writer.cpp +++ b/src/paimon/format/parquet/parquet_format_writer.cpp @@ -66,10 +66,9 @@ Result> ParquetFormatWriter::Create( std::unique_ptr<::parquet::arrow::FileWriter> file_writer, ::parquet::arrow::FileWriter::Open(*write_schema, pool.get(), out, writer_properties, arrow_writer_properties)); - return std::unique_ptr( - new ParquetFormatWriter(std::move(file_writer), out, schema, max_memory_use, - /*needs_vector_conversion=*/!logical_type->Equals(write_type), - pool)); + return std::unique_ptr(new ParquetFormatWriter( + std::move(file_writer), out, schema, max_memory_use, + /*needs_vector_conversion=*/!logical_type->Equals(write_type), pool)); } Status ParquetFormatWriter::AddBatch(ArrowArray* batch) { diff --git a/src/paimon/format/parquet/parquet_vector_converter.cpp b/src/paimon/format/parquet/parquet_vector_converter.cpp index b5e4acf1e..92a5e27d2 100644 --- a/src/paimon/format/parquet/parquet_vector_converter.cpp +++ b/src/paimon/format/parquet/parquet_vector_converter.cpp @@ -52,11 +52,9 @@ std::shared_ptr ParquetVectorConverter::GetWriteType( case arrow::Type::MAP: { const auto& map_type = checked_cast(*logical_type); return std::make_shared( - map_type.value_field()->WithType( - arrow::struct_({map_type.key_field()->WithType( - GetWriteType(map_type.key_type())), - map_type.item_field()->WithType( - GetWriteType(map_type.item_type()))})), + map_type.value_field()->WithType(arrow::struct_( + {map_type.key_field()->WithType(GetWriteType(map_type.key_type())), + map_type.item_field()->WithType(GetWriteType(map_type.item_type()))})), map_type.keys_sorted()); } default: diff --git a/src/paimon/format/parquet/parquet_vector_io_test.cpp b/src/paimon/format/parquet/parquet_vector_io_test.cpp index c263043cf..c01f97ecb 100644 --- a/src/paimon/format/parquet/parquet_vector_io_test.cpp +++ b/src/paimon/format/parquet/parquet_vector_io_test.cpp @@ -158,25 +158,26 @@ class ParquetVectorIoTest : public ::testing::Test { ASSERT_OK(out->Close()); } - std::unique_ptr CreateVectorReader( - const std::string& file_path, const std::shared_ptr& read_schema, - const std::shared_ptr& predicate, - const std::map& options, int32_t batch_size) { - std::unique_ptr vector_reader; + void CreateVectorReader(const std::string& file_path, + const std::shared_ptr& read_schema, + const std::shared_ptr& predicate, + const std::map& options, int32_t batch_size, + std::unique_ptr* vector_reader_out) { ASSERT_OK_AND_ASSIGN(std::shared_ptr in, fs_->Open(file_path)); ASSERT_OK_AND_ASSIGN(int64_t length, in->Length()); auto in_stream = std::make_shared(in, length, arrow_pool_); - ASSERT_OK_AND_ASSIGN(std::unique_ptr reader, - ParquetFileBatchReader::Create(std::move(in_stream), options, - batch_size, /*file_metadata=*/nullptr, - /*storage_read_bytes=*/nullptr, - arrow_pool_)); - vector_reader = std::make_unique(std::move(reader), pool_); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr reader, + ParquetFileBatchReader::Create(std::move(in_stream), options, batch_size, + /*file_metadata=*/nullptr, + /*storage_read_bytes=*/nullptr, arrow_pool_)); + std::unique_ptr vector_reader = + std::make_unique(std::move(reader), pool_); auto c_schema = std::make_unique(); ASSERT_TRUE(arrow::ExportSchema(*read_schema, c_schema.get()).ok()); ASSERT_OK(vector_reader->SetReadSchema(c_schema.get(), predicate, /*selection_bitmap=*/std::nullopt)); - return vector_reader; + *vector_reader_out = std::move(vector_reader); } void ReadFixtureAndCheck( @@ -304,9 +305,9 @@ TEST_F(ParquetVectorIoTest, ReadNestedFixedSizeListFile) { std::string file_path = dir_->Str() + "/nested-fixed-size-list.parquet"; WriteWithArrowWriter(file_path, logical_type, json); - std::unique_ptr reader = - CreateVectorReader(file_path, arrow::schema(logical_type->fields()), /*predicate=*/nullptr, - /*options=*/{}, /*batch_size=*/10); + std::unique_ptr reader; + CreateVectorReader(file_path, arrow::schema(logical_type->fields()), /*predicate=*/nullptr, + /*options=*/{}, /*batch_size=*/10, &reader); ASSERT_OK_AND_ASSIGN(std::shared_ptr actual, paimon::test::ReadResultCollector::CollectResult(reader.get())); arrow::Result> expected_result = @@ -331,9 +332,9 @@ TEST_F(ParquetVectorIoTest, ReadVectorWithPredicatePushdown) { std::shared_ptr predicate = PredicateBuilder::GreaterThan( /*field_index=*/0, /*field_name=*/"id", FieldType::INT, Literal(2)); - std::unique_ptr reader = - CreateVectorReader(file_path, arrow::schema(logical_type->fields()), predicate, - /*options=*/{}, /*batch_size=*/10); + std::unique_ptr reader; + CreateVectorReader(file_path, arrow::schema(logical_type->fields()), predicate, + /*options=*/{}, /*batch_size=*/10, &reader); ASSERT_OK_AND_ASSIGN(std::shared_ptr actual, paimon::test::ReadResultCollector::CollectResult(reader.get())); arrow::Result> expected_result = From 0951b31591d442e9a3291b20186cda3d100ed5c9 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Mon, 17 Aug 2026 12:48:21 +0000 Subject: [PATCH 10/16] fix(parquet): write nullable VECTOR values as zero length null lists MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Writing a VECTOR column holding a null value failed with "Lists with non-zero length null components are not supported": Arrow 17 casts a null FixedSizeList row to a null LIST slot that still spans `list_size` values, and the Parquet writer rejects that layout. Keep the plain cast when a VECTOR has no null value, and otherwise rebuild the LIST with compacted values so its null slots have a zero length, matching what Paimon Java writes. Both paths stay inside the Parquet format layer with a TODO to drop the second one after the Arrow upgrade. Also cover the table level read path with a predicate pushed down on a non-vector column next to a VECTOR column. Co-authored-by: 小明同学 --- .../parquet/parquet_vector_converter.cpp | 109 ++++++++++++++++-- .../parquet/parquet_vector_converter_test.cpp | 8 +- test/inte/write_and_read_inte_test.cpp | 71 ++++++++++++ 3 files changed, 178 insertions(+), 10 deletions(-) diff --git a/src/paimon/format/parquet/parquet_vector_converter.cpp b/src/paimon/format/parquet/parquet_vector_converter.cpp index 92a5e27d2..5b6446d22 100644 --- a/src/paimon/format/parquet/parquet_vector_converter.cpp +++ b/src/paimon/format/parquet/parquet_vector_converter.cpp @@ -18,9 +18,14 @@ #include "paimon/format/parquet/parquet_vector_converter.h" +#include +#include #include +#include #include "arrow/array.h" +#include "arrow/array/array_nested.h" +#include "arrow/array/builder_primitive.h" #include "arrow/compute/api.h" #include "arrow/type.h" #include "paimon/common/utils/arrow/status_utils.h" @@ -29,6 +34,75 @@ #include "paimon/status.h" namespace paimon::parquet { +namespace { + +Result> CastToListType( + const std::shared_ptr& array, const std::shared_ptr& write_type, + arrow::MemoryPool* pool) { + arrow::compute::ExecContext exec_context(pool); + arrow::TypeHolder type_holder(write_type.get()); + arrow::compute::CastOptions options = arrow::compute::CastOptions::Safe(); + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW( + std::shared_ptr result, + arrow::compute::Cast(*array, type_holder, options, &exec_context)); + return result; +} + +/// Rebuilds a nullable VECTOR as a LIST whose null slots have a zero length, dropping the +/// values Arrow keeps for them. +/// +/// TODO(ChaomingZhangCN): Cast the whole array once Arrow is upgraded. Arrow 17 casts a null +/// FixedSizeList row to a null LIST slot spanning `list_size` values, and the Parquet writer +/// rejects a LIST with non-zero length null slots. +Result> CompactNullVectorsToList( + const arrow::FixedSizeListArray& vector_array, + const std::shared_ptr& write_type, arrow::MemoryPool* pool) { + const auto& vector_type = checked_cast(*vector_array.type()); + const int32_t vector_length = vector_type.list_size(); + if (vector_array.length() > std::numeric_limits::max() / vector_length) { + return Status::Invalid("VECTOR values exceed the maximum Parquet LIST offset"); + } + + arrow::Int32Builder offsets_builder(pool); + arrow::Int64Builder indices_builder(pool); + arrow::BooleanBuilder validity_builder(pool); + PAIMON_RETURN_NOT_OK_FROM_ARROW(offsets_builder.Reserve(vector_array.length() + 1)); + PAIMON_RETURN_NOT_OK_FROM_ARROW(indices_builder.Reserve(vector_array.length() * vector_length)); + PAIMON_RETURN_NOT_OK_FROM_ARROW(validity_builder.Reserve(vector_array.length())); + PAIMON_RETURN_NOT_OK_FROM_ARROW(offsets_builder.Append(0)); + + int32_t offset = 0; + for (int64_t i = 0; i < vector_array.length(); ++i) { + bool valid = !vector_array.IsNull(i); + PAIMON_RETURN_NOT_OK_FROM_ARROW(validity_builder.Append(valid)); + if (valid) { + int64_t value_offset = (vector_array.offset() + i) * vector_length; + for (int32_t j = 0; j < vector_length; ++j) { + PAIMON_RETURN_NOT_OK_FROM_ARROW(indices_builder.Append(value_offset + j)); + } + offset += vector_length; + } + PAIMON_RETURN_NOT_OK_FROM_ARROW(offsets_builder.Append(offset)); + } + + std::shared_ptr offsets; + std::shared_ptr indices; + std::shared_ptr validity; + PAIMON_RETURN_NOT_OK_FROM_ARROW(offsets_builder.Finish(&offsets)); + PAIMON_RETURN_NOT_OK_FROM_ARROW(indices_builder.Finish(&indices)); + PAIMON_RETURN_NOT_OK_FROM_ARROW(validity_builder.Finish(&validity)); + + arrow::compute::ExecContext exec_context(pool); + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW( + arrow::Datum values, + arrow::compute::Take(arrow::Datum(vector_array.values()), arrow::Datum(indices), + arrow::compute::TakeOptions::NoBoundsCheck(), &exec_context)); + return std::make_shared( + write_type, vector_array.length(), offsets->data()->buffers[1], values.make_array(), + validity->data()->buffers[1], vector_array.null_count()); +} + +} // namespace std::shared_ptr ParquetVectorConverter::GetWriteType( const std::shared_ptr& logical_type) { @@ -67,15 +141,34 @@ Result> ParquetVectorConverter::ConvertToWriteType if (!VectorUtils::ContainsVectorType(array->type())) { return array; } - PAIMON_RETURN_NOT_OK(VectorUtils::ValidateNestedVectorElements(array)); std::shared_ptr write_type = GetWriteType(array->type()); - arrow::compute::ExecContext exec_context(pool); - arrow::TypeHolder type_holder(write_type.get()); - arrow::compute::CastOptions options = arrow::compute::CastOptions::Safe(); - PAIMON_ASSIGN_OR_RAISE_FROM_ARROW( - std::shared_ptr result, - arrow::compute::Cast(*array, type_holder, options, &exec_context)); - return result; + switch (array->type_id()) { + case arrow::Type::FIXED_SIZE_LIST: { + PAIMON_RETURN_NOT_OK(VectorUtils::ValidateVectorElements(*array)); + const auto& vector_array = checked_cast(*array); + if (vector_array.null_count() == 0) { + return CastToListType(array, write_type, pool); + } + return CompactNullVectorsToList(vector_array, write_type, pool); + } + case arrow::Type::STRUCT: + case arrow::Type::LIST: + case arrow::Type::MAP: { + std::vector> children; + children.reserve(array->data()->child_data.size()); + for (const auto& child_data : array->data()->child_data) { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr child, + ConvertToWriteType(arrow::MakeArray(child_data), pool)); + children.push_back(child->data()); + } + std::shared_ptr data = array->data()->Copy(); + data->child_data = std::move(children); + data->type = write_type; + return arrow::MakeArray(data); + } + default: + return array; + } } } // namespace paimon::parquet diff --git a/src/paimon/format/parquet/parquet_vector_converter_test.cpp b/src/paimon/format/parquet/parquet_vector_converter_test.cpp index c6653ceaa..6e1c0b0df 100644 --- a/src/paimon/format/parquet/parquet_vector_converter_test.cpp +++ b/src/paimon/format/parquet/parquet_vector_converter_test.cpp @@ -41,9 +41,13 @@ TEST(ParquetVectorConverterTest, ConvertNullableVectorToList) { auto list_array = checked_pointer_cast(converted); ASSERT_EQ(list_array->value_length(0), 3); ASSERT_TRUE(list_array->IsNull(1)); - ASSERT_EQ(list_array->value_length(1), 3); + // The Parquet writer rejects a null LIST slot spanning values, so the values Arrow keeps for + // a null VECTOR row are dropped. + ASSERT_EQ(list_array->value_length(1), 0); ASSERT_EQ(list_array->value_length(2), 3); - ASSERT_EQ(list_array->values()->length(), 9); + ASSERT_EQ(list_array->values()->length(), 6); + auto values = checked_pointer_cast(list_array->values()); + ASSERT_FLOAT_EQ(values->Value(3), 4.0f); } TEST(ParquetVectorConverterTest, ConvertNestedVectorsToList) { diff --git a/test/inte/write_and_read_inte_test.cpp b/test/inte/write_and_read_inte_test.cpp index 57da416cd..04c461396 100644 --- a/test/inte/write_and_read_inte_test.cpp +++ b/test/inte/write_and_read_inte_test.cpp @@ -364,6 +364,77 @@ TEST_P(WriteAndReadInteTest, TestAppendVector) { ASSERT_TRUE(std::make_shared(expected)->Equals(actual)); } +// Pushing a predicate down on a non-vector column must not disturb the VECTOR column, whose +// read schema differs from the type stored in the data file. +TEST_P(WriteAndReadInteTest, TestAppendVectorWithPredicate) { + auto [file_format, file_system] = GetParam(); + if (file_format != "parquet") { + return; + } + + auto vector_type = + arrow::fixed_size_list(arrow::field("item", arrow::float32(), /*nullable=*/false), 3); + arrow::FieldVector fields = {arrow::field("id", arrow::int32()), + arrow::field("embedding", vector_type)}; + std::map options = { + {Options::MANIFEST_FORMAT, "avro"}, + {Options::FILE_FORMAT, file_format}, + {Options::TARGET_FILE_SIZE, "1048576"}, + {Options::BUCKET, "-1"}, + {Options::FILE_SYSTEM, file_system}, + // One row per row group, so the predicate prunes row groups instead of rows. + {"parquet.write.max-row-group-length", "1"}, + }; + if (file_system == "jindo") { + options = AddOptionsForJindo(options); + } + ASSERT_OK_AND_ASSIGN(auto helper, + TestHelper::Create(test_dir_, arrow::schema(fields), /*partition_keys=*/{}, + /*primary_keys=*/{}, options, + /*is_streaming_mode=*/false)); + std::string table_path = PathUtil::JoinPath(test_dir_, "foo.db/bar"); + const std::string data_json = R"([ + [1, [1.0, 2.0, 3.0]], + [2, null], + [3, [4.0, 5.0, 6.0]], + [4, [7.0, 8.0, 9.0]] + ])"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr batch, + TestHelper::MakeRecordBatch(arrow::struct_(fields), data_json, + /*partition_map=*/{}, /*bucket=*/0, {})); + ASSERT_OK(helper->WriteAndCommit(std::move(batch), /*commit_identifier=*/0, + /*expected_commit_messages=*/std::nullopt)); + + auto predicate = PredicateBuilder::GreaterThan(/*field_index=*/0, /*field_name=*/"id", + FieldType::INT, Literal(2)); + ScanContextBuilder scan_context_builder(table_path); + scan_context_builder.SetOptions(options) + .AddOption(Options::SCAN_MODE, StartupMode::LatestFull().ToString()) + .SetPredicate(predicate); + ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); + ASSERT_FALSE(result_plan->Splits().empty()); + + ReadContextBuilder read_context_builder(table_path); + read_context_builder.SetOptions(options).SetPredicate(predicate); + ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(result_plan->Splits())); + ASSERT_OK_AND_ASSIGN(auto actual, ReadResultCollector::CollectResult(batch_reader.get())); + + arrow::FieldVector fields_with_row_kind = fields; + fields_with_row_kind.insert(fields_with_row_kind.begin(), + arrow::field("_VALUE_KIND", arrow::int8())); + auto expected = std::make_shared( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields_with_row_kind), R"([ + [0, 3, [4.0, 5.0, 6.0]], + [0, 4, [7.0, 8.0, 9.0]] + ])") + .ValueOrDie()); + ASSERT_TRUE(expected->Equals(actual)) << actual->ToString(); +} + TEST_P(WriteAndReadInteTest, TestPKSimple) { arrow::FieldVector fields = { arrow::field("pk", arrow::utf8()), From 4d2ba7d45c3225e442eca339662b2f3903eabc60 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Tue, 18 Aug 2026 04:22:45 +0000 Subject: [PATCH 11/16] feat(schema): reject VECTOR fields in data-evolution tables MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Data evolution is out of scope for this phase the same way primary-key tables are, so a schema that enables it together with a VECTOR field must be rejected rather than fail later at runtime. Co-authored-by: 小明同学 --- docs/source/user_guide/data_types.rst | 12 +++++++++--- src/paimon/core/schema/schema_validation.cpp | 4 ++++ .../core/schema/schema_validation_test.cpp | 19 +++++++++++++++++++ 3 files changed, 32 insertions(+), 3 deletions(-) diff --git a/docs/source/user_guide/data_types.rst b/docs/source/user_guide/data_types.rst index 9b0fb7613..9fdecf6e5 100644 --- a/docs/source/user_guide/data_types.rst +++ b/docs/source/user_guide/data_types.rst @@ -197,9 +197,15 @@ and `Arrow DataTypes `` - Map diff --git a/src/paimon/core/schema/schema_validation.cpp b/src/paimon/core/schema/schema_validation.cpp index d4874f08e..7342826d4 100644 --- a/src/paimon/core/schema/schema_validation.cpp +++ b/src/paimon/core/schema/schema_validation.cpp @@ -678,6 +678,10 @@ Status SchemaValidation::ValidateVectorFields(const TableSchema& schema, return Status::NotImplemented( "VECTOR fields in primary-key tables are not implemented yet."); } + if (options.DataEvolutionEnabled()) { + return Status::NotImplemented( + "VECTOR fields in data-evolution tables are not implemented yet."); + } PAIMON_RETURN_NOT_OK( ValidateVectorFileFormat(Options::FILE_FORMAT, options.GetFileFormat()->Identifier())); return ValidatePerLevelOption(options.ToMap(), Options::FILE_FORMAT_PER_LEVEL, diff --git a/src/paimon/core/schema/schema_validation_test.cpp b/src/paimon/core/schema/schema_validation_test.cpp index 10d0ffacd..47603497b 100644 --- a/src/paimon/core/schema/schema_validation_test.cpp +++ b/src/paimon/core/schema/schema_validation_test.cpp @@ -88,6 +88,25 @@ TEST(SchemaValidationTest, TestVectorType) { /*partition_keys=*/{}, /*primary_keys=*/{"id"}, primary_key_options)); ASSERT_NOK_WITH_MSG(SchemaValidation::ValidateTableSchema(*table_schema), "VECTOR fields in primary-key tables are not implemented yet."); + + std::map data_evolution_options = { + {Options::BUCKET, "-1"}, + {Options::FILE_FORMAT, "parquet"}, + {Options::ROW_TRACKING_ENABLED, "true"}, + {Options::DATA_EVOLUTION_ENABLED, "true"}, + }; + ASSERT_OK_AND_ASSIGN(table_schema, + TableSchema::Create(/*schema_id=*/0, schema, + /*partition_keys=*/{}, + /*primary_keys=*/{}, data_evolution_options)); + ASSERT_NOK_WITH_MSG(SchemaValidation::ValidateTableSchema(*table_schema), + "VECTOR fields in data-evolution tables are not implemented yet."); + ASSERT_OK_AND_ASSIGN(table_schema, + TableSchema::Create(/*schema_id=*/0, nested_schema, + /*partition_keys=*/{}, + /*primary_keys=*/{}, data_evolution_options)); + ASSERT_NOK_WITH_MSG(SchemaValidation::ValidateTableSchema(*table_schema), + "VECTOR fields in data-evolution tables are not implemented yet."); } TEST(SchemaValidationTest, TestRowTracking) { From 7740bcec2308c38d30cd08d3e82c9da10d8a7ce1 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Tue, 18 Aug 2026 04:22:55 +0000 Subject: [PATCH 12/16] test(parquet): cover nullable and FixedSizeList VECTOR files MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `WriteWithArrowWriter` did not store the Arrow schema, so the file exposed the column as LIST and `ReadNestedFixedSizeListFile` silently took the LIST to VECTOR conversion instead of the nested FixedSizeList path it is meant to cover. Store the schema, assert the layout the file really has, and read a vector nested in a LIST without null values. Add the nullable cross-language fixtures: parquet-mr writes the column as a 3-level LIST, which reads back fine, while a writer that stores the Arrow schema exposes it as FixedSizeList, which Arrow 17 cannot read once a value is null. Pin both behaviors. Reuse the shared reader helper in the remaining checks, and read the id and element values through typed arrays: `GetValues(1)` already accounts for `ArrayData::offset`, so adding the offset again applied it twice. Co-authored-by: 小明同学 --- .../format/parquet/parquet_vector_io_test.cpp | 135 +++++++++++------- .../parquet/vector_compatibility/README.md | 18 ++- .../java_vector_nullable.parquet | Bin 0 -> 765 bytes .../rust_vector_nullable.parquet | Bin 0 -> 932 bytes 4 files changed, 99 insertions(+), 54 deletions(-) create mode 100644 test/test_data/parquet/vector_compatibility/java_vector_nullable.parquet create mode 100644 test/test_data/parquet/vector_compatibility/rust_vector_nullable.parquet diff --git a/src/paimon/format/parquet/parquet_vector_io_test.cpp b/src/paimon/format/parquet/parquet_vector_io_test.cpp index c01f97ecb..fe551fb45 100644 --- a/src/paimon/format/parquet/parquet_vector_io_test.cpp +++ b/src/paimon/format/parquet/parquet_vector_io_test.cpp @@ -69,32 +69,17 @@ class ParquetVectorIoTest : public ::testing::Test { std::string file_path = dir_->Str() + "/" + file_name; WriteWithFormatWriter(file_path, write_type, json, /*max_row_group_length=*/1024); - ASSERT_OK_AND_ASSIGN(std::shared_ptr in, fs_->Open(file_path)); - ASSERT_OK_AND_ASSIGN(int64_t length, in->Length()); - auto in_stream = std::make_shared(in, length, arrow_pool_); - ASSERT_OK_AND_ASSIGN( - std::unique_ptr reader, - ParquetFileBatchReader::Create(std::move(in_stream), /*options=*/{}, - /*batch_size=*/10, /*file_metadata=*/nullptr, - /*storage_read_bytes=*/nullptr, arrow_pool_)); - ASSERT_OK_AND_ASSIGN(std::unique_ptr c_file_schema, reader->GetFileSchema()); - arrow::Result> file_type_result = - arrow::ImportType(c_file_schema.get()); - ASSERT_TRUE(file_type_result.ok()) << file_type_result.status().ToString(); - auto file_type = - checked_pointer_cast(std::move(file_type_result).ValueOrDie()); + std::shared_ptr file_type; + ReadFileType(file_path, &file_type); std::shared_ptr physical_value_type = file_type->field(1)->type(); if (physical_value_type->id() == arrow::Type::STRUCT) { physical_value_type = physical_value_type->field(0)->type(); } ASSERT_EQ(physical_value_type->id(), arrow::Type::LIST); - std::unique_ptr vector_reader = - std::make_unique(std::move(reader), pool_); - auto c_schema = std::make_unique(); - ASSERT_TRUE(arrow::ExportSchema(*arrow::schema(read_type->fields()), c_schema.get()).ok()); - ASSERT_OK(vector_reader->SetReadSchema(c_schema.get(), /*predicate=*/nullptr, - /*selection_bitmap=*/std::nullopt)); + std::unique_ptr vector_reader; + CreateVectorReader(file_path, arrow::schema(read_type->fields()), /*predicate=*/nullptr, + /*options=*/{}, /*batch_size=*/10, &vector_reader); ASSERT_OK_AND_ASSIGN(std::shared_ptr actual, paimon::test::ReadResultCollector::CollectResult(vector_reader.get())); @@ -132,8 +117,9 @@ class ParquetVectorIoTest : public ::testing::Test { ASSERT_OK(out->Close()); } - /// Writes `array` with the plain Arrow Parquet writer, so FixedSizeList columns keep their - /// Arrow type in the file schema the way Paimon Rust and Python writers store them. + /// Writes `array` with the plain Arrow Parquet writer, storing the Arrow schema so that + /// FixedSizeList columns are read back as FixedSizeList, the way Paimon Rust and Python + /// writers store them. void WriteWithArrowWriter(const std::string& file_path, const std::shared_ptr& type, const std::string& json) { @@ -151,13 +137,33 @@ class ParquetVectorIoTest : public ::testing::Test { fs_->Create(file_path, /*overwrite=*/false)); auto arrow_out = std::make_shared(out); ::parquet::WriterProperties::Builder properties_builder; + std::shared_ptr<::parquet::ArrowWriterProperties> arrow_properties = + ::parquet::ArrowWriterProperties::Builder().store_schema()->build(); arrow::Status status = ::parquet::arrow::WriteTable( *std::move(table_result).ValueOrDie(), arrow_pool_.get(), arrow_out, - /*chunk_size=*/1024, properties_builder.build()); + /*chunk_size=*/1024, properties_builder.build(), arrow_properties); ASSERT_TRUE(status.ok()) << status.ToString(); ASSERT_OK(out->Close()); } + void ReadFileType(const std::string& file_path, + std::shared_ptr* file_type_out) { + ASSERT_OK_AND_ASSIGN(std::shared_ptr in, fs_->Open(file_path)); + ASSERT_OK_AND_ASSIGN(int64_t length, in->Length()); + auto in_stream = std::make_shared(in, length, arrow_pool_); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr reader, + ParquetFileBatchReader::Create(std::move(in_stream), /*options=*/{}, + /*batch_size=*/10, /*file_metadata=*/nullptr, + /*storage_read_bytes=*/nullptr, arrow_pool_)); + ASSERT_OK_AND_ASSIGN(std::unique_ptr c_file_schema, reader->GetFileSchema()); + arrow::Result> file_type_result = + arrow::ImportType(c_file_schema.get()); + ASSERT_TRUE(file_type_result.ok()) << file_type_result.status().ToString(); + *file_type_out = + checked_pointer_cast(std::move(file_type_result).ValueOrDie()); + } + void CreateVectorReader(const std::string& file_path, const std::shared_ptr& read_schema, const std::shared_ptr& predicate, @@ -186,20 +192,8 @@ class ParquetVectorIoTest : public ::testing::Test { const std::vector>>& expected_vectors) { std::string file_path = paimon::test::GetDataDir() + "/parquet/vector_compatibility/" + file_name; - ASSERT_OK_AND_ASSIGN(std::shared_ptr in, fs_->Open(file_path)); - ASSERT_OK_AND_ASSIGN(int64_t length, in->Length()); - auto in_stream = std::make_shared(in, length, arrow_pool_); - ASSERT_OK_AND_ASSIGN( - std::unique_ptr reader, - ParquetFileBatchReader::Create(std::move(in_stream), /*options=*/{}, - /*batch_size=*/10, /*file_metadata=*/nullptr, - /*storage_read_bytes=*/nullptr, arrow_pool_)); - ASSERT_OK_AND_ASSIGN(std::unique_ptr c_file_schema, reader->GetFileSchema()); - arrow::Result> file_type_result = - arrow::ImportType(c_file_schema.get()); - ASSERT_TRUE(file_type_result.ok()) << file_type_result.status().ToString(); - auto file_type = - checked_pointer_cast(std::move(file_type_result).ValueOrDie()); + std::shared_ptr file_type; + ReadFileType(file_path, &file_type); std::shared_ptr file_vector_field = file_type->GetFieldByName("embedding"); ASSERT_TRUE(file_vector_field); ASSERT_EQ(file_vector_field->type()->id(), expected_file_vector_type); @@ -210,12 +204,9 @@ class ParquetVectorIoTest : public ::testing::Test { arrow::field("element", arrow::float32(), /*nullable=*/false), vector_length); auto logical_schema = arrow::schema({file_id_field, file_vector_field->WithType(vector_type)}); - std::unique_ptr vector_reader = - std::make_unique(std::move(reader), pool_); - auto c_read_schema = std::make_unique(); - ASSERT_TRUE(arrow::ExportSchema(*logical_schema, c_read_schema.get()).ok()); - ASSERT_OK(vector_reader->SetReadSchema(c_read_schema.get(), /*predicate=*/nullptr, - /*selection_bitmap=*/std::nullopt)); + std::unique_ptr vector_reader; + CreateVectorReader(file_path, logical_schema, /*predicate=*/nullptr, /*options=*/{}, + /*batch_size=*/10, &vector_reader); ASSERT_OK_AND_ASSIGN(std::shared_ptr actual, paimon::test::ReadResultCollector::CollectResult(vector_reader.get())); ASSERT_EQ(actual->num_chunks(), 1); @@ -227,13 +218,13 @@ class ParquetVectorIoTest : public ::testing::Test { ASSERT_TRUE(vector_field); ASSERT_EQ(id_field->type_id(), arrow::Type::INT32); ASSERT_EQ(vector_field->type_id(), arrow::Type::FIXED_SIZE_LIST); + auto ids = checked_pointer_cast(id_field); auto vector_array = checked_pointer_cast(vector_field); - ASSERT_EQ(id_field->length(), static_cast(expected_ids.size())); + ASSERT_EQ(ids->length(), static_cast(expected_ids.size())); ASSERT_EQ(vector_array->length(), static_cast(expected_vectors.size())); - const int32_t* id_values = id_field->data()->GetValues(1); - for (int64_t i = 0; i < id_field->length(); ++i) { - ASSERT_FALSE(id_field->IsNull(i)); - ASSERT_EQ(id_values[id_field->offset() + i], expected_ids[i]); + for (int64_t i = 0; i < ids->length(); ++i) { + ASSERT_FALSE(ids->IsNull(i)); + ASSERT_EQ(ids->Value(i), expected_ids[i]); if (!expected_vectors[i]) { ASSERT_TRUE(vector_array->IsNull(i)); continue; @@ -241,10 +232,10 @@ class ParquetVectorIoTest : public ::testing::Test { ASSERT_FALSE(vector_array->IsNull(i)); ASSERT_EQ(vector_array->value_length(i), static_cast(expected_vectors[i]->size())); - std::shared_ptr values = vector_array->value_slice(i); - const float* vector_values = values->data()->GetValues(1); - for (int64_t j = 0; j < vector_array->value_length(i); ++j) { - ASSERT_FLOAT_EQ(vector_values[j], expected_vectors[i].value()[j]); + auto values = checked_pointer_cast(vector_array->value_slice(i)); + for (int64_t j = 0; j < values->length(); ++j) { + ASSERT_FALSE(values->IsNull(j)); + ASSERT_FLOAT_EQ(values->Value(j), expected_vectors[i].value()[j]); } } } @@ -301,10 +292,19 @@ TEST_F(ParquetVectorIoTest, ReadNestedFixedSizeListFile) { arrow::field("id", arrow::int32()), arrow::field("history", arrow::list(vector_type)), })); - const std::string json = R"([[1, [[1.0, 2.0, 3.0], null]], [2, null]])"; + const std::string json = R"([[1, [[1.0, 2.0, 3.0], [4.0, 5.0, 6.0]]], [2, []]])"; std::string file_path = dir_->Str() + "/nested-fixed-size-list.parquet"; WriteWithArrowWriter(file_path, logical_type, json); + // Without this the file would expose the column as list> and the read would take + // the LIST to VECTOR conversion instead of the nested FixedSizeList path under test. + std::shared_ptr file_type; + ReadFileType(file_path, &file_type); + std::shared_ptr file_history_field = file_type->GetFieldByName("history"); + ASSERT_TRUE(file_history_field); + ASSERT_EQ(file_history_field->type()->id(), arrow::Type::LIST); + ASSERT_EQ(file_history_field->type()->field(0)->type()->id(), arrow::Type::FIXED_SIZE_LIST); + std::unique_ptr reader; CreateVectorReader(file_path, arrow::schema(logical_type->fields()), /*predicate=*/nullptr, /*options=*/{}, /*batch_size=*/10, &reader); @@ -361,4 +361,33 @@ TEST_F(ParquetVectorIoTest, ReadRustFixture) { {{{1.0f, 2.0f, 3.0f}}, {{7.0f, 8.0f, 9.0f}}, {{4.0f, 5.0f, 6.0f}}}); } +TEST_F(ParquetVectorIoTest, ReadNullableJavaFixture) { + ReadFixtureAndCheck("java_vector_nullable.parquet", arrow::Type::LIST, /*vector_length=*/3, + /*expected_ids=*/{1, 2, 3}, + /*expected_vectors=*/ + {{{1.0f, 2.0f, 3.0f}}, std::nullopt, {{4.0f, 5.0f, 6.0f}}}); +} + +// A writer that stores the Arrow schema, such as Paimon Rust or Python, exposes the VECTOR column +// as FixedSizeList. Arrow 17 cannot read a null value from such a column: Parquet stores a null +// list slot with no values, while FixedSizeListReader::AssembleArray in +// parquet/arrow/reader.cc requires every slot to span exactly `list_size` values. +// +// TODO(ChaomingZhangCN): Turn this into a read check once Arrow is upgraded. +TEST_F(ParquetVectorIoTest, ReadNullableRustFixtureIsUnsupported) { + std::string file_path = + paimon::test::GetDataDir() + "/parquet/vector_compatibility/rust_vector_nullable.parquet"; + std::shared_ptr file_type; + ReadFileType(file_path, &file_type); + std::shared_ptr file_vector_field = file_type->GetFieldByName("embedding"); + ASSERT_TRUE(file_vector_field); + ASSERT_EQ(file_vector_field->type()->id(), arrow::Type::FIXED_SIZE_LIST); + + std::unique_ptr reader; + CreateVectorReader(file_path, arrow::schema(file_type->fields()), /*predicate=*/nullptr, + /*options=*/{}, /*batch_size=*/10, &reader); + ASSERT_NOK_WITH_MSG(paimon::test::ReadResultCollector::CollectResult(reader.get()), + "Expected all lists to be of size=3"); +} + } // namespace paimon::parquet::test diff --git a/test/test_data/parquet/vector_compatibility/README.md b/test/test_data/parquet/vector_compatibility/README.md index 4ffae4e0b..15eb2ef30 100644 --- a/test/test_data/parquet/vector_compatibility/README.md +++ b/test/test_data/parquet/vector_compatibility/README.md @@ -1,22 +1,38 @@ # VECTOR Parquet compatibility fixtures These files pin the two physical Arrow schemas produced by Java and Rust writers for Paimon -VECTOR columns. +VECTOR columns, with and without null vectors. - `java_vector.parquet` was copied from Apache Paimon Rust commit `403a2b2e9bfc4ea66cd7e633619f1460efd18bc8`, path `crates/paimon/testdata/pkvector/pk_vector_ivf_flat/bucket-0/data-932a1249-f7e0-4a03-8e1f-ab8c85cbb76f-0.parquet`. The fixture documentation records Apache Paimon Java commit `7234e4c34` and `PkVectorFixtureGenerator` as its source. Its VECTOR column is exposed as Arrow `list`. +- `java_vector_nullable.parquet` was generated with parquet-mr 1.15.1 (`parquet-avro` + `AvroParquetWriter` with `parquet.avro.write-old-list-structure=false`, so the column uses the + standard 3-level `list` / `element` layout Paimon Java writes). The rows are `(1, [1, 2, 3])`, + `(2, null)` and `(3, [4, 5, 6])`. The file carries no `ARROW:schema` key, so the VECTOR column + is exposed as Arrow `list`. - `rust_vector.parquet` was generated with Apache Arrow Rust 58.4.0 using `FixedSizeListBuilder` and `parquet::arrow::ArrowWriter`, the same Arrow and Parquet representation used by Apache Paimon Rust. Its VECTOR column is exposed as Arrow `fixed_size_list[3]`. The rows are `(1, [1, 2, 3])`, `(2, [7, 8, 9])`, and `(3, [4, 5, 6])`. +- `rust_vector_nullable.parquet` was generated the same way, with the rows `(1, [1, 2, 3])`, + `(2, null)` and `(3, [4, 5, 6])`. + +A file that stores the Arrow schema, as the Rust writer does, is read back as +`fixed_size_list`. Arrow 17 cannot read a null value from such a column, because Parquet stores a +null list slot with no values while `FixedSizeListReader::AssembleArray` in +`parquet/arrow/reader.cc` requires every slot to span exactly `list_size` values. Reading +`rust_vector_nullable.parquet` therefore fails until Arrow is upgraded, which +`ParquetVectorIoTest.ReadNullableRustFixtureIsUnsupported` pins. SHA-256 checksums: ```text 2b2325cc2266301beaa2c78ec666cb5e0ee62283049de2a7231e3c9ae07bf3ca java_vector.parquet +42352e11daf5a291e8a8c4cfc8d0f0f6f8c9099cf7dcf24c28d9e159a29e0d8a java_vector_nullable.parquet b5ba47e766ad72fca9c8485aa718ad27709c1fb4d34fb3670aa35e2001cbdbb0 rust_vector.parquet +f86058b1bc6cf803003446ca0abb7923e928d455fd39a7288c6d3acdd5fd10e9 rust_vector_nullable.parquet ``` diff --git a/test/test_data/parquet/vector_compatibility/java_vector_nullable.parquet b/test/test_data/parquet/vector_compatibility/java_vector_nullable.parquet new file mode 100644 index 0000000000000000000000000000000000000000..fa900ac0dc1c19629f8c58d610eddfa820fdc5bf GIT binary patch literal 765 zcmYjP&1%~~5FT%gs+5uvnq4H&!7NxfK^0L_|6+)9F(s4`dQC%-rBy1>Pr8;JhhXrj z&yic-qU6+LPrdXp3hlLY)^eP%GuoY*Z)Uz3bawpSqd*NjzyALFy=hSmO`Ylh6#yWp z8>r|z!SmzGpRX@1x`n1jwN*G$fQ`L9fW;BM1}LZt)H~Gsfw@ggqpGUmwJb(V1}4)` zpbV;13@S9mATe!CH(LZ=fN3$E4w*`}*eRW<7eYR~eLfXIk;{)Vzou1m)xjWf2u)&a zigYBxFwQP1q1wAXW;CltHHpUsB{&-*pNT}IA}at%Sf*oxg*EFs62ux5y==&aw%#xK zmgE^Umh4Ll>EIHuFZ6fINr2rGy2HL#Xb)6D-K^tyokuoH1`nOF$rhWjnSF{))ZFU3 zIAWyn#CqCfy%AtPRi6c+17P1OOtW>oc5p$C@#@N#pC_Vl{i)2|aqvl`zHwK%<;BgF z;5{xykjs!eJo8g!Kkg8HQ7n>h%zNjz58+rrSE+GU@VT=Nt#`aeobzmwBpe~D3|~tB z2E%b7QY_1(B=@n#g~LM;`IDsJJ(V%Pn1iu>EfFT&G!I4MDt1Oy%>c&9YNXextWAX+ z$9Y!sT(9YRwZ>=?Ct)pUA2i#ePUJ^Xv(@bi?{G8<+8wXe?uNnsVf3aGc;5bjziT=c N016xvfHVB@{{w+Cp{M`= literal 0 HcmV?d00001 diff --git a/test/test_data/parquet/vector_compatibility/rust_vector_nullable.parquet b/test/test_data/parquet/vector_compatibility/rust_vector_nullable.parquet new file mode 100644 index 0000000000000000000000000000000000000000..8610e46e1065f7f248148c0383b5c2a2c7985043 GIT binary patch literal 932 zcmZuw&2Cab6h3z#S4>Dlo6h7;!lDbjsRS!cifIf<8K9*qiG>9X#MDr^MN)n$m&%r< zYoEZC56~xY-f%Ae{W7%vh_UXIO z8$x?=t_New39WFkF!KVoRDgx{-GqK7C`mf4iZY8p_HG<`#I6+j&2h3jolfR&!*}n4 zQCG$RJ|a&HdQ=eJ!(enC^m>DFUkd5gi^wU&z4&kt&ZNMYhMO&eWM-^b_f*->7*-+qV1Yw!Tg}hZUhXuP+5d>Zg#K>X##Th)EL>>HgoA zY#XccD72{10}Zqx!qH%`1o#5q<#yj)K?)(q;4JM`U#j#UYnOR z&F1UV=h3FTTk(`-J}(<%8c>#3#>`EnZ(cG4YYe}#G;lZO1-pP&7B_Y)6ULADgGX5K z?V0M_95RGF%WspwAIV{(Z5he_wU-I--clzNvm_3<>TOWgIIGBvL3i~^IJ9(W;6XDPg{B%z6|NH|iJGlk` literal 0 HcmV?d00001 From 25ccfb14b11088256fb1cd0474443d995924f743 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Tue, 18 Aug 2026 04:23:01 +0000 Subject: [PATCH 13/16] test(inte): read and write VECTOR values nested in STRUCT, LIST and MAP MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The table level coverage only wrote a top level VECTOR column, so nesting was exercised at the Parquet layer only. Co-authored-by: 小明同学 --- test/inte/write_and_read_inte_test.cpp | 51 ++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/test/inte/write_and_read_inte_test.cpp b/test/inte/write_and_read_inte_test.cpp index 04c461396..fae7db325 100644 --- a/test/inte/write_and_read_inte_test.cpp +++ b/test/inte/write_and_read_inte_test.cpp @@ -364,6 +364,57 @@ TEST_P(WriteAndReadInteTest, TestAppendVector) { ASSERT_TRUE(std::make_shared(expected)->Equals(actual)); } +TEST_P(WriteAndReadInteTest, TestAppendNestedVector) { + auto [file_format, file_system] = GetParam(); + if (file_format != "parquet") { + return; + } + + auto vector_type = + arrow::fixed_size_list(arrow::field("item", arrow::float32(), /*nullable=*/false), 2); + arrow::FieldVector fields = { + arrow::field("id", arrow::int32()), + arrow::field("payload", arrow::struct_({arrow::field("embedding", vector_type)})), + arrow::field("history", arrow::list(vector_type)), + arrow::field("by_name", arrow::map(arrow::utf8(), vector_type)), + }; + std::map options = { + {Options::MANIFEST_FORMAT, "avro"}, {Options::FILE_FORMAT, file_format}, + {Options::TARGET_FILE_SIZE, "1024"}, {Options::BUCKET, "-1"}, + {Options::FILE_SYSTEM, file_system}, + }; + if (file_system == "jindo") { + options = AddOptionsForJindo(options); + } + ASSERT_OK_AND_ASSIGN(auto helper, + TestHelper::Create(test_dir_, arrow::schema(fields), /*partition_keys=*/{}, + /*primary_keys=*/{}, options, + /*is_streaming_mode=*/false)); + const std::string data_json = R"([ + [1, [[1.0, 2.0]], [[3.0, 4.0], null], [["a", [5.0, 6.0]], ["b", null]]], + [2, [null], null, []], + [3, null, [], [["c", [7.0, 8.0]]]] + ])"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr batch, + TestHelper::MakeRecordBatch(arrow::struct_(fields), data_json, + /*partition_map=*/{}, /*bucket=*/0, {})); + ASSERT_OK(helper->WriteAndCommit(std::move(batch), /*commit_identifier=*/0, + /*expected_commit_messages=*/std::nullopt)); + + ASSERT_OK_AND_ASSIGN(std::vector> data_splits, + helper->NewScan(StartupMode::LatestFull(), /*snapshot_id=*/std::nullopt)); + arrow::FieldVector result_fields = fields; + result_fields.insert(result_fields.begin(), arrow::field("_VALUE_KIND", arrow::int8())); + const std::string expected_json = R"([ + [0, 1, [[1.0, 2.0]], [[3.0, 4.0], null], [["a", [5.0, 6.0]], ["b", null]]], + [0, 2, [null], null, []], + [0, 3, null, [], [["c", [7.0, 8.0]]]] + ])"; + ASSERT_OK_AND_ASSIGN(bool success, helper->ReadAndCheckResult(arrow::struct_(result_fields), + data_splits, expected_json)); + ASSERT_TRUE(success); +} + // Pushing a predicate down on a non-vector column must not disturb the VECTOR column, whose // read schema differs from the type stored in the data file. TEST_P(WriteAndReadInteTest, TestAppendVectorWithPredicate) { From 56f9629afc9a1ab62b197c2338925e086555a8d0 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Tue, 18 Aug 2026 06:43:28 +0000 Subject: [PATCH 14/16] fix(io): include the Arrow C data interface in the VECTOR reader header MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `GetFileSchema()` is defined in the header and returns `Result>`, so instantiating the unique_ptr destructor needs the complete type. `file_batch_reader.h` only forward declares it, which broke the clang-tidy run over the header, as the other readers that define this method inline avoid by including the C data interface too. Co-authored-by: 小明同学 --- src/paimon/core/io/vector_file_batch_reader.h | 1 + 1 file changed, 1 insertion(+) diff --git a/src/paimon/core/io/vector_file_batch_reader.h b/src/paimon/core/io/vector_file_batch_reader.h index 79ad5e066..b7eab263c 100644 --- a/src/paimon/core/io/vector_file_batch_reader.h +++ b/src/paimon/core/io/vector_file_batch_reader.h @@ -22,6 +22,7 @@ #include #include +#include "arrow/c/abi.h" #include "paimon/reader/file_batch_reader.h" namespace arrow { From 5d93d8881b082cedbd94eb850e7a78e70815f021 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Tue, 18 Aug 2026 11:41:10 +0000 Subject: [PATCH 15/16] fix(io): normalize the FixedSizeList type of VECTOR batches A file storing a VECTOR column as FixedSizeList was returned with the element field of the file, while a file storing it as LIST was cast to the element field of the read schema. Reading both encodings with one table schema therefore produced batches of unequal Arrow types, which cannot be combined into a single result. Rewrap the validated values in the requested type instead of returning the file type unchanged. --- .../core/io/vector_file_batch_reader.cpp | 8 +++- .../core/io/vector_file_batch_reader_test.cpp | 40 +++++++++++++++++++ .../format/parquet/parquet_vector_io_test.cpp | 39 ++++++++++++++++++ 3 files changed, 86 insertions(+), 1 deletion(-) diff --git a/src/paimon/core/io/vector_file_batch_reader.cpp b/src/paimon/core/io/vector_file_batch_reader.cpp index a9cf99a04..a4573eef9 100644 --- a/src/paimon/core/io/vector_file_batch_reader.cpp +++ b/src/paimon/core/io/vector_file_batch_reader.cpp @@ -168,7 +168,13 @@ Result> ConvertToReadType( vector_type.ToString())); } PAIMON_RETURN_NOT_OK(VectorUtils::ValidateVectorElements(*array)); - return array; + // Writers disagree on the element field, for example `element: float not null` + // for Paimon Rust against the `item: float` of a Paimon schema. Restore the + // requested type so that files storing VECTOR as LIST and files storing it as + // FixedSizeList produce batches of one type. + std::shared_ptr data = array->data()->Copy(); + data->type = read_type; + return arrow::MakeArray(data); } return CastListToVector( array, checked_pointer_cast(read_type), pool); diff --git a/src/paimon/core/io/vector_file_batch_reader_test.cpp b/src/paimon/core/io/vector_file_batch_reader_test.cpp index 66d118519..e334e62aa 100644 --- a/src/paimon/core/io/vector_file_batch_reader_test.cpp +++ b/src/paimon/core/io/vector_file_batch_reader_test.cpp @@ -112,6 +112,46 @@ TEST(VectorFileBatchReaderTest, KeepFixedSizeListFileSchema) { ASSERT_TRUE(logical_array->Equals(std::move(actual_result).ValueOrDie())); } +// Paimon Rust names the element field of a VECTOR column `element` and marks it non-nullable, +// while a Paimon schema names it `item`. Batches must carry the requested type either way, +// otherwise they cannot be combined with batches read from a file storing VECTOR as LIST. +TEST(VectorFileBatchReaderTest, NormalizeFixedSizeListElementField) { + auto file_vector = + arrow::fixed_size_list(arrow::field("element", arrow::float32(), /*nullable=*/false), 3); + auto logical_vector = arrow::fixed_size_list(arrow::float32(), 3); + auto file_type = AsStructType(arrow::struct_({ + arrow::field("embedding", file_vector), + arrow::field("history", arrow::list(file_vector)), + })); + auto logical_type = AsStructType(arrow::struct_({ + arrow::field("embedding", logical_vector), + arrow::field("history", arrow::list(logical_vector)), + })); + const std::string json = R"([ + [[1.0, 2.0, 3.0], [[4.0, 5.0, 6.0]]], + [null, []] + ])"; + auto file_array = arrow::ipc::internal::json::ArrayFromJSON(file_type, json).ValueOrDie(); + auto mock_reader = + std::make_unique(file_array, file_type, /*batch_size=*/10); + mock_reader->EnableRandomizeBatchSize(false); + VectorFileBatchReader reader(std::move(mock_reader), GetDefaultPool()); + + ArrowSchema c_read_schema; + ASSERT_TRUE(arrow::ExportSchema(*arrow::schema(logical_type->fields()), &c_read_schema).ok()); + ASSERT_OK(reader.SetReadSchema(&c_read_schema, /*predicate=*/nullptr, + /*selection_bitmap=*/std::nullopt)); + + ASSERT_OK_AND_ASSIGN(BatchReader::ReadBatch batch, reader.NextBatch()); + arrow::Result> actual_result = + arrow::ImportArray(batch.first.get(), batch.second.get()); + ASSERT_TRUE(actual_result.ok()) << actual_result.status().ToString(); + std::shared_ptr actual = std::move(actual_result).ValueOrDie(); + ASSERT_TRUE(actual->type()->Equals(logical_type)) << actual->type()->ToString(); + auto expected = arrow::ipc::internal::json::ArrayFromJSON(logical_type, json).ValueOrDie(); + ASSERT_TRUE(expected->Equals(actual)) << actual->ToString(); +} + TEST(VectorFileBatchReaderTest, ConvertNestedVectorsWithBitmap) { auto logical_vector = arrow::fixed_size_list(arrow::field("item", arrow::float64(), /*nullable=*/false), 2); diff --git a/src/paimon/format/parquet/parquet_vector_io_test.cpp b/src/paimon/format/parquet/parquet_vector_io_test.cpp index fe551fb45..5f26653fe 100644 --- a/src/paimon/format/parquet/parquet_vector_io_test.cpp +++ b/src/paimon/format/parquet/parquet_vector_io_test.cpp @@ -368,6 +368,45 @@ TEST_F(ParquetVectorIoTest, ReadNullableJavaFixture) { {{{1.0f, 2.0f, 3.0f}}, std::nullopt, {{4.0f, 5.0f, 6.0f}}}); } +// A table can hold files from several writers, and Paimon Java stores VECTOR as Parquet LIST +// while Paimon Rust stores it as FixedSizeList. Reading both with the table schema must produce +// batches of one Arrow type, otherwise they cannot be combined into a single result. +TEST_F(ParquetVectorIoTest, ReadMixedListAndFixedSizeListFixtures) { + // The Arrow type a Paimon schema builds for `id INT, embedding VECTOR`. The Rust + // fixture instead names the element field `element` and marks it non-nullable. + auto logical_schema = + arrow::schema({arrow::field("id", arrow::int32()), + arrow::field("embedding", arrow::fixed_size_list(arrow::float32(), 3))}); + std::shared_ptr logical_type = arrow::struct_(logical_schema->fields()); + + arrow::ArrayVector chunks; + for (const std::string& file_name : {"java_vector_nullable.parquet", "rust_vector.parquet"}) { + std::string file_path = + paimon::test::GetDataDir() + "/parquet/vector_compatibility/" + file_name; + std::unique_ptr reader; + CreateVectorReader(file_path, logical_schema, /*predicate=*/nullptr, /*options=*/{}, + /*batch_size=*/10, &reader); + ASSERT_OK_AND_ASSIGN(std::shared_ptr actual, + paimon::test::ReadResultCollector::CollectResult(reader.get())); + ASSERT_TRUE(actual->type()->Equals(logical_type)) + << file_name << ": " << actual->type()->ToString(); + chunks.insert(chunks.end(), actual->chunks().begin(), actual->chunks().end()); + } + + arrow::Result> merged_result = + arrow::ChunkedArray::Make(chunks); + ASSERT_TRUE(merged_result.ok()) << merged_result.status().ToString(); + arrow::Result> expected_result = + arrow::ipc::internal::json::ArrayFromJSON( + logical_type, R"([[1, [1.0, 2.0, 3.0]], [2, null], [3, [4.0, 5.0, 6.0]], + [1, [1.0, 2.0, 3.0]], [2, [7.0, 8.0, 9.0]], [3, [4.0, 5.0, 6.0]]])"); + ASSERT_TRUE(expected_result.ok()) << expected_result.status().ToString(); + std::shared_ptr merged = std::move(merged_result).ValueOrDie(); + ASSERT_TRUE(std::make_shared(std::move(expected_result).ValueOrDie()) + ->Equals(merged)) + << merged->ToString(); +} + // A writer that stores the Arrow schema, such as Paimon Rust or Python, exposes the VECTOR column // as FixedSizeList. Arrow 17 cannot read a null value from such a column: Parquet stores a null // list slot with no values, while FixedSizeListReader::AssembleArray in From e23abc1f3c3d7bf989bf8c63c195e4b33caacdfc Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Tue, 18 Aug 2026 11:54:07 +0000 Subject: [PATCH 16/16] test(parquet): fix the mixed VECTOR encoding test lifetimes A reader owns the Arrow memory pool that its batches are allocated from, so releasing each fixture reader before the collected chunks crashed while freeing them. Keep both readers alive until the comparison is done, and iterate over the file names without binding a reference to a temporary, which -Werror=range-loop-construct rejects. --- src/paimon/format/parquet/parquet_vector_io_test.cpp | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/paimon/format/parquet/parquet_vector_io_test.cpp b/src/paimon/format/parquet/parquet_vector_io_test.cpp index 5f26653fe..e177cc1b6 100644 --- a/src/paimon/format/parquet/parquet_vector_io_test.cpp +++ b/src/paimon/format/parquet/parquet_vector_io_test.cpp @@ -379,8 +379,12 @@ TEST_F(ParquetVectorIoTest, ReadMixedListAndFixedSizeListFixtures) { arrow::field("embedding", arrow::fixed_size_list(arrow::float32(), 3))}); std::shared_ptr logical_type = arrow::struct_(logical_schema->fields()); + // A reader owns the memory pool that its batches are allocated from, so it has to outlive + // the chunks collected from it. This mirrors a scan, which holds every split reader until + // the whole result has been consumed. + std::vector> readers; arrow::ArrayVector chunks; - for (const std::string& file_name : {"java_vector_nullable.parquet", "rust_vector.parquet"}) { + for (const char* file_name : {"java_vector_nullable.parquet", "rust_vector.parquet"}) { std::string file_path = paimon::test::GetDataDir() + "/parquet/vector_compatibility/" + file_name; std::unique_ptr reader; @@ -388,6 +392,7 @@ TEST_F(ParquetVectorIoTest, ReadMixedListAndFixedSizeListFixtures) { /*batch_size=*/10, &reader); ASSERT_OK_AND_ASSIGN(std::shared_ptr actual, paimon::test::ReadResultCollector::CollectResult(reader.get())); + readers.push_back(std::move(reader)); ASSERT_TRUE(actual->type()->Equals(logical_type)) << file_name << ": " << actual->type()->ToString(); chunks.insert(chunks.end(), actual->chunks().begin(), actual->chunks().end());