feat(parquet): support vector type storage

docs/source/user_guide/data_types.rst

@@ -186,6 +186,27 @@ and `Arrow DataTypes <https://arrow.apache.org/docs/format/Columnar.html#data-ty
 
        The type can be declared using ``ARRAY<t>`` where t is the data type of the contained elements.
 
+   * - ``VECTOR<t, n>``
+     - FixedSizeList
+     - Data type of a dense vector containing exactly ``n`` elements of type ``t``.
+
+       ``n`` must be positive. ``t`` can be ``BOOLEAN``, ``TINYINT``,
+       ``SMALLINT``, ``INT``, ``BIGINT``, ``FLOAT``, or ``DOUBLE``. A VECTOR
+       value may be NULL, but its elements cannot be NULL.
+
+       Paimon C++ currently supports VECTOR columns only in append-only tables
+       backed by Parquet data files. They use the standard Parquet LIST
+       representation on disk and are restored as Arrow ``FixedSizeList``
+       values on read. Primary-key tables and data-evolution tables containing
+       VECTOR fields are rejected. VECTOR columns also cannot be partition or
+       bucket keys. Dedicated vector storage is not included yet.
+
+       **Note:** A data file written by another engine that records the column as
+       Arrow ``FixedSizeList`` instead of ``LIST``, such as Paimon Rust or Python,
+       can only be read while it holds no NULL vector. Parquet stores a NULL list
+       slot with no values, which the Arrow 17 Parquet reader rejects for a
+       ``FixedSizeList`` column.
+
    * - ``MAP<kt, vt>``
      - Map
      - Data type of an associative array that maps keys (including NULL) to values (including NULL). A map cannot contain duplicate keys; each key can map to at most one value.

include/paimon/defs.h

@@ -50,6 +50,8 @@ enum class FieldType {
     STRUCT = 15,
     BLOB = 16,
     VARIANT = 17,
+    /// Fixed-length dense vector represented by Arrow FixedSizeList.
+    VECTOR = 18,
     UNKNOWN = 128,
 };
 

include/paimon/format/column_stats.h

@@ -33,8 +33,8 @@ namespace paimon {
 /// ColumnStats is an abstract base class that represents statistical information for data columns
 /// in Paimon tables. It provides min/max values and null count statistics
 ///
-/// Only primitive data types support min/max statistics. Nested types (arrays, maps, structs) only
-/// track null counts through `NestedColumnStats`.
+/// Only primitive data types support min/max statistics. Nested types (arrays, vectors, maps,
+/// structs) only track null counts through `NestedColumnStats`.
 ///
 /// @note This is an abstract base class. Use the static factory methods `CreateXXXColumnStats()` to
 /// create concrete instances for specific data types.
@@ -52,7 +52,7 @@ class PAIMON_EXPORT ColumnStats {
     /// @name CreateXXXColumnStats()
     /// %Factory methods `CreateXXXColumnStats()` to create column statistics.
     /// - min/max/null_count for primitive data types
-    /// - null_count for nested data types (arrays, maps, structs)
+    /// - null_count for nested data types (arrays, vectors, maps, structs)
     ///
     /// @{
     static std::unique_ptr<ColumnStats> CreateBooleanColumnStats(std::optional<bool> min,
@@ -88,8 +88,8 @@ class PAIMON_EXPORT ColumnStats {
     static std::unique_ptr<ColumnStats> CreateDateColumnStats(std::optional<int32_t> min,
                                                               std::optional<int32_t> max,
                                                               std::optional<int64_t> null_count);
-    /// Creates column statistics for nested data types (arrays, maps, structs), which only track
-    /// null counts.
+    /// Creates column statistics for nested data types (arrays, vectors, maps, structs), which only
+    /// track null counts.
     static std::unique_ptr<ColumnStats> CreateNestedColumnStats(const FieldType& nested_type,
                                                                 std::optional<int64_t> null_count);
     /// @}
@@ -180,7 +180,7 @@ class PAIMON_EXPORT NestedColumnStats : public ColumnStats {
     NestedColumnStats(const FieldType& nested_type, std::optional<int64_t> null_count)
         : nested_type_(nested_type), null_count_(null_count) {
         assert(nested_type == FieldType::ARRAY || nested_type == FieldType::MAP ||
-               nested_type == FieldType::STRUCT);
+               nested_type == FieldType::STRUCT || nested_type == FieldType::VECTOR);
     }
 
     std::optional<int64_t> NullCount() const override {

src/paimon/CMakeLists.txt

@@ -157,6 +157,7 @@ set(PAIMON_COMMON_SRCS
     common/utils/arrow/arrow_output_stream_adapter.cpp
     common/utils/arrow/arrow_utils.cpp
     common/utils/arrow/mem_utils.cpp
+    common/utils/arrow/vector_utils.cpp
     common/utils/binary_row_partition_computer.cpp
     common/utils/bit_set.cpp
     common/utils/bloom_filter.cpp
@@ -270,6 +271,7 @@ set(PAIMON_CORE_SRCS
     core/io/data_file_writer.cpp
     core/io/field_mapping_reader.cpp
     core/io/complete_row_tracking_fields_reader.cpp
+    core/io/vector_file_batch_reader.cpp
     core/io/file_index_evaluator.cpp
     core/io/key_value_data_file_record_reader.cpp
     core/io/key_value_data_file_writer_factory.cpp
@@ -602,6 +604,7 @@ if(PAIMON_BUILD_TESTS)
                     common/utils/row_range_index_test.cpp
                     common/utils/var_length_int_utils_test.cpp
                     common/utils/arrow/arrow_utils_test.cpp
+                    common/utils/arrow/vector_utils_test.cpp
                     common/utils/arrow/arrow_stream_adapter_test.cpp
                     common/utils/arrow/mem_utils_test.cpp
                     common/utils/arrow/status_utils_test.cpp
@@ -735,6 +738,7 @@ if(PAIMON_BUILD_TESTS)
                     core/io/key_value_in_memory_record_reader_test.cpp
                     core/io/merged_key_value_record_reader_test.cpp
                     core/io/complete_row_tracking_fields_reader_test.cpp
+                    core/io/vector_file_batch_reader_test.cpp
                     core/io/data_file_meta_test.cpp
                     core/io/file_index_evaluator_test.cpp
                     core/io/single_file_writer_test.cpp

src/paimon/common/predicate/literal_converter.cpp

@@ -166,6 +166,7 @@ Result<Literal> LiteralConverter::ConvertLiteralsFromRow(
         case FieldType::DATE:
             return Literal(FieldType::DATE, row.GetInt(field_idx));
         case FieldType::ARRAY:
+        case FieldType::VECTOR:
         case FieldType::MAP:
         case FieldType::STRUCT:
         default:

src/paimon/common/types/data_type.cpp

@@ -29,6 +29,7 @@
 #include "paimon/common/types/array_type.h"
 #include "paimon/common/types/map_type.h"
 #include "paimon/common/types/row_type.h"
+#include "paimon/common/types/vector_type.h"
 #include "paimon/common/utils/checked_cast.h"
 #include "paimon/common/utils/date_time_utils.h"
 #include "paimon/common/utils/decimal_utils.h"
@@ -52,6 +53,8 @@ std::unique_ptr<DataType> DataType::Create(
             return std::make_unique<MapType>(type, nullable, metadata);
         case arrow::Type::type::LIST:
             return std::make_unique<ArrayType>(type, nullable, metadata);
+        case arrow::Type::type::FIXED_SIZE_LIST:
+            return std::make_unique<VectorType>(type, nullable, metadata);
         case arrow::Type::type::STRUCT:
             if (VariantTypeUtils::IsVariantMetadata(metadata)) {
                 // A variant field is physically a struct<value, metadata> but is a scalar

src/paimon/common/types/data_type_json_parser.cpp

@@ -25,6 +25,7 @@
 #include <cstdint>
 #include <limits>
 #include <map>
+#include <optional>
 #include <sstream>
 #include <utility>
 #include <vector>
@@ -33,6 +34,7 @@
 #include "paimon/common/data/blob_utils.h"
 #include "paimon/common/data/variant/variant_type_utils.h"
 #include "paimon/common/types/data_field.h"
+#include "paimon/common/types/vector_type.h"
 #include "paimon/common/utils/date_time_utils.h"
 #include "paimon/common/utils/rapidjson_util.h"
 #include "paimon/common/utils/string_utils.h"
@@ -148,6 +150,7 @@ enum class Keyword : int32_t {
     ROW,
     BLOB,
     VARIANT,
+    VECTOR,
     // NULL is keyword in c++
     NULL_,
     RAW,
@@ -197,6 +200,7 @@ const std::map<std::string, Keyword>& Keywords() {
         {"ROW", Keyword::ROW},
         {"BLOB", Keyword::BLOB},
         {"VARIANT", Keyword::VARIANT},
+        {"VECTOR", Keyword::VECTOR},
         {"NULL", Keyword::NULL_},
         {"RAW", Keyword::RAW},
         {"LEGACY", Keyword::LEGACY},
@@ -249,6 +253,7 @@ class TokenParser {
     Result<std::shared_ptr<arrow::DataType>> ParseDoubleType();
     Result<std::shared_ptr<arrow::DataType>> ParseTimestampType();
     Result<std::shared_ptr<arrow::DataType>> ParseTimestampLtzType();
+    Result<std::shared_ptr<arrow::DataType>> ParseVectorType();
     Result<int32_t> ParseOptionalPrecision(int32_t default_precision);
 
  private:
@@ -526,6 +531,8 @@ Result<std::shared_ptr<arrow::DataType>> TokenParser::ParseTypeByKeyword(
             return ParseTimestampType();
         case Keyword::TIMESTAMP_LTZ:
             return ParseTimestampLtzType();
+        case Keyword::VECTOR:
+            return ParseVectorType();
         default:
             return Status::Invalid(fmt::format("Unsupported type: {}", GetToken().value));
     }
@@ -607,6 +614,31 @@ Result<std::shared_ptr<arrow::DataType>> TokenParser::ParseTimestampLtzType() {
     return ts_type;
 }
 
+Result<std::shared_ptr<arrow::DataType>> TokenParser::ParseVectorType() {
+    PAIMON_RETURN_NOT_OK(NextToken(TokenType::BEGIN_SUBTYPE));
+    bool element_nullable = true;
+    AtomicTypeAttributes element_attributes;
+    PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<arrow::DataType> element_type,
+                           ParseTypeWithNullability(&element_nullable, &element_attributes));
+    if (element_attributes.is_blob || element_attributes.is_variant ||
+        !VectorType::IsValidElementType(element_type)) {
+        return Status::Invalid(
+            fmt::format("Invalid element type for vector: {}", element_type->ToString()));
+    }
+    PAIMON_RETURN_NOT_OK(NextToken(TokenType::LIST_SEPARATOR));
+    PAIMON_RETURN_NOT_OK(NextToken(TokenType::LITERAL_INT));
+    const std::string& length_token = GetToken().value;
+    std::optional<int32_t> length = StringUtils::StringToValue<int32_t>(length_token);
+    if (!length || length.value() < 1) {
+        return Status::Invalid(
+            fmt::format("Vector length must be between 1 and {} (both inclusive), but was {}",
+                        std::numeric_limits<int32_t>::max(), length_token));
+    }
+    PAIMON_RETURN_NOT_OK(NextToken(TokenType::END_SUBTYPE));
+    return arrow::fixed_size_list(arrow::field("item", element_type, element_nullable),
+                                  length.value());
+}
+
 Result<int32_t> TokenParser::ParseOptionalPrecision(int32_t default_precision) {
     auto precision = default_precision;
     if (HasNextToken({TokenType::BEGIN_PARAMETER})) {
@@ -659,6 +691,8 @@ Result<std::shared_ptr<arrow::Field>> DataTypeJsonParser::ParseComplexTypeField(
 
     if (StringUtils::StartsWith(type_str, "ARRAY")) {
         return ParseArrayType(name, type_json_value, nullable);
+    } else if (StringUtils::StartsWith(type_str, "VECTOR")) {
+        return ParseVectorType(name, type_json_value, nullable);
     } else if (StringUtils::StartsWith(type_str, "MAP")) {
         return ParseMapType(name, type_json_value, nullable);
     } else if (StringUtils::StartsWith(type_str, "ROW")) {
@@ -681,6 +715,27 @@ Result<std::shared_ptr<arrow::Field>> DataTypeJsonParser::ParseArrayType(
     return arrow::field(name, arrow::list(element_field), nullable);
 }
 
+Result<std::shared_ptr<arrow::Field>> DataTypeJsonParser::ParseVectorType(
+    const std::string& name, const rapidjson::Value& type_json_value, bool nullable) {
+    if (!type_json_value.HasMember("element") || !type_json_value.HasMember("length")) {
+        return Status::Invalid("vector data type must have element and length");
+    }
+    if (!type_json_value["length"].IsInt()) {
+        return Status::Invalid("vector length must be an integer");
+    }
+    int32_t length = type_json_value["length"].GetInt();
+    if (length < 1) {
+        return Status::Invalid("Vector length must be between 1 and 2147483647 (both inclusive)");
+    }
+    PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<arrow::Field> element_field,
+                           ParseType("item", type_json_value["element"]));
+    if (!VectorType::IsValidElementType(element_field->type())) {
+        return Status::Invalid(
+            fmt::format("Invalid element type for vector: {}", element_field->type()->ToString()));
+    }
+    return arrow::field(name, arrow::fixed_size_list(element_field, length), nullable);
+}
+
 Result<std::shared_ptr<arrow::Field>> DataTypeJsonParser::ParseMapType(
     const std::string& name, const rapidjson::Value& type_json_value, bool nullable) {
     if (!type_json_value.HasMember("key") || !type_json_value.HasMember("value")) {

src/paimon/common/types/data_type_json_parser.h

@@ -50,6 +50,8 @@ class DataTypeJsonParser {
 
     static Result<std::shared_ptr<arrow::Field>> ParseArrayType(
         const std::string& name, const rapidjson::Value& type_json_value, bool nullable);
+    static Result<std::shared_ptr<arrow::Field>> ParseVectorType(
+        const std::string& name, const rapidjson::Value& type_json_value, bool nullable);
     static Result<std::shared_ptr<arrow::Field>> ParseMapType(
         const std::string& name, const rapidjson::Value& type_json_value, bool nullable);
     static Result<std::shared_ptr<arrow::Field>> ParseRowType(

src/paimon/common/types/data_type_json_parser_test.cpp

@@ -48,6 +48,56 @@ TEST(DataTypeJsonParserTest, ParseTypeArrayTypeSuccess) {
     ASSERT_NE(field, nullptr);
 }
 
+TEST(DataTypeJsonParserTest, ParseVectorTypeSuccess) {
+    const char* json = R"({
+        "type": "VECTOR NOT NULL",
+        "element": "FLOAT",
+        "length": 3
+    })";
+    rapidjson::Document doc;
+    doc.Parse(json);
+
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<arrow::Field> field,
+                         DataTypeJsonParser::ParseType("embedding", doc));
+    ASSERT_FALSE(field->nullable());
+    ASSERT_EQ(field->type()->id(), arrow::Type::FIXED_SIZE_LIST);
+    auto vector_type = checked_pointer_cast<arrow::FixedSizeListType>(field->type());
+    ASSERT_EQ(vector_type->list_size(), 3);
+    ASSERT_TRUE(vector_type->value_type()->Equals(arrow::float32()));
+
+    rapidjson::Document sql_doc;
+    rapidjson::Value sql_value("VECTOR<BIGINT NOT NULL, 5>", sql_doc.GetAllocator());
+    ASSERT_OK_AND_ASSIGN(field, DataTypeJsonParser::ParseType("embedding", sql_value));
+    vector_type = checked_pointer_cast<arrow::FixedSizeListType>(field->type());
+    ASSERT_TRUE(field->nullable());
+    ASSERT_EQ(vector_type->list_size(), 5);
+    ASSERT_FALSE(vector_type->value_field()->nullable());
+    ASSERT_TRUE(vector_type->value_type()->Equals(arrow::int64()));
+}
+
+TEST(DataTypeJsonParserTest, ParseVectorTypeFailure) {
+    for (const char* json : {
+             R"({"type":"VECTOR","element":"FLOAT","length":0})",
+             R"({"type":"VECTOR","element":"STRING","length":3})",
+             R"({"type":"VECTOR","element":"FLOAT"})",
+             R"({"type":"VECTOR","element":"FLOAT","length":"3"})",
+         }) {
+        rapidjson::Document doc;
+        doc.Parse(json);
+        ASSERT_NOK(DataTypeJsonParser::ParseType("embedding", doc));
+    }
+
+    rapidjson::Document sql_doc;
+    rapidjson::Value sql_value("VECTOR<STRING, 3>", sql_doc.GetAllocator());
+    ASSERT_NOK_WITH_MSG(DataTypeJsonParser::ParseType("embedding", sql_value),
+                        "Invalid element type for vector");
+    sql_value.SetString("VECTOR<DOUBLE, 3>", sql_doc.GetAllocator());
+    ASSERT_OK(DataTypeJsonParser::ParseType("embedding", sql_value));
+    sql_value.SetString("VECTOR<FLOAT, 999999999999999999999999>", sql_doc.GetAllocator());
+    ASSERT_NOK_WITH_MSG(DataTypeJsonParser::ParseType("embedding", sql_value),
+                        "Vector length must be between 1 and 2147483647");
+}
+
 TEST(DataTypeJsonParserTest, ParseTypeMapTypeSuccess) {
     const std::string name = "map_field";
     const char* json = R"({

src/paimon/common/types/data_type_test.cpp

@@ -148,4 +148,18 @@ TEST(DataTypeTest, NestedTypeSerializationUsesChildMetadata) {
               R"({"type":"ARRAY","element":"INT"})");
 }
 
+TEST(DataTypeTest, VectorTypeSerialization) {
+    auto vector_field = arrow::field(
+        "embedding", arrow::fixed_size_list(arrow::field("item", arrow::float32()), 3), false);
+    auto data_type =
+        DataType::Create(vector_field->type(), vector_field->nullable(), vector_field->metadata());
+    rapidjson::Document doc;
+    auto value = data_type->ToJson(&doc.GetAllocator());
+    rapidjson::StringBuffer buffer;
+    rapidjson::Writer<rapidjson::StringBuffer> writer(buffer);
+    value.Accept(writer);
+    ASSERT_EQ(std::string(buffer.GetString()),
+              R"({"type":"VECTOR NOT NULL","element":"FLOAT","length":3})");
+}
+
 }  // namespace paimon::test