refactor(btree): refactor btree global index and add tests

include/paimon/defs.h

@@ -434,24 +434,6 @@ struct PAIMON_EXPORT Options {
     /// "lookup.cache-max-disk-size" - Max disk size for lookup cache, you can use this option
     /// to limit the use of local disks. Default value is unlimited (INT64_MAX).
     static const char LOOKUP_CACHE_MAX_DISK_SIZE[];
-    /// "btree-index.compression" - The compression algorithm to use for BTreeIndex.
-    /// Default value is "none".
-    static const char BTREE_INDEX_COMPRESSION[];
-    /// "btree-index.compression-level" - The compression level of the compression algorithm.
-    /// Default value is 1.
-    static const char BTREE_INDEX_COMPRESSION_LEVEL[];
-    /// "btree-index.block-size" - The block size to use for BTreeIndex.
-    /// Default value is 64 KB.
-    static const char BTREE_INDEX_BLOCK_SIZE[];
-    /// "btree-index.cache-size" - The cache size to use for BTreeIndex.
-    /// Default value is 128 MB.
-    static const char BTREE_INDEX_CACHE_SIZE[];
-    /// "btree-index.high-priority-pool-ratio" - The high priority pool ratio to use for BTreeIndex.
-    /// Default value is 0.1.
-    static const char BTREE_INDEX_HIGH_PRIORITY_POOL_RATIO[];
-    /// "btree-index.records-per-range" - The expected number of records per BTree Index File.
-    /// Default value is 1000000.
-    static const char BTREE_INDEX_RECORDS_PER_RANGE[];
 };
 
 static constexpr int64_t BATCH_WRITE_COMMIT_IDENTIFIER = std::numeric_limits<int64_t>::max();

include/paimon/file_index/file_index_reader.h

@@ -62,12 +62,6 @@ class PAIMON_EXPORT FileIndexReader : public FunctionVisitor<std::shared_ptr<Fil
     Result<std::shared_ptr<FileIndexResult>> VisitContains(const Literal& literal) override;
 
     Result<std::shared_ptr<FileIndexResult>> VisitLike(const Literal& literal) override;
-
-    Result<std::shared_ptr<FileIndexResult>> VisitAnd(
-        const std::vector<Result<std::shared_ptr<FileIndexResult>>>& children) override;
-
-    Result<std::shared_ptr<FileIndexResult>> VisitOr(
-        const std::vector<Result<std::shared_ptr<FileIndexResult>>>& children) override;
 };
 
 }  // namespace paimon

include/paimon/global_index/global_index_reader.h

@@ -48,20 +48,6 @@ class PAIMON_EXPORT GlobalIndexReader : public FunctionVisitor<std::shared_ptr<G
     virtual Result<std::shared_ptr<GlobalIndexResult>> VisitFullTextSearch(
         const std::shared_ptr<FullTextSearch>& full_text_search) = 0;
 
-    /// VisitAnd performs logical AND across multiple child results.
-    /// Default implementation returns "not supported" error.
-    Result<std::shared_ptr<GlobalIndexResult>> VisitAnd(
-        const std::vector<Result<std::shared_ptr<GlobalIndexResult>>>& children) override {
-        return Status::NotImplemented("AND operations not supported by this index type");
-    }
-
-    /// VisitOr performs logical OR across multiple child results.
-    /// Default implementation returns "not supported" error.
-    Result<std::shared_ptr<GlobalIndexResult>> VisitOr(
-        const std::vector<Result<std::shared_ptr<GlobalIndexResult>>>& children) override {
-        return Status::NotImplemented("OR operations not supported by this index type");
-    }
-
     /// @return true if the reader is thread-safe; false otherwise.
     virtual bool IsThreadSafe() const = 0;
 

include/paimon/predicate/function_visitor.h

@@ -74,41 +74,5 @@ class PAIMON_EXPORT FunctionVisitor {
 
     /// Evaluates whether string values like the given string.
     virtual Result<T> VisitLike(const Literal& literal) = 0;
-
-    /// Evaluates the BETWEEN predicate with the given lower and upper bounds.
-    virtual Result<T> VisitBetween(const Literal& from, const Literal& to) {
-        // Default implementation: BETWEEN is equivalent to >= AND <=
-        auto lower_result = VisitGreaterOrEqual(from);
-        if (!lower_result.ok()) {
-            return lower_result.status();
-        }
-        auto upper_result = VisitLessOrEqual(to);
-        if (!upper_result.ok()) {
-            return upper_result.status();
-        }
-        return VisitAnd({std::move(lower_result).value(), std::move(upper_result).value()});
-    }
-
-    /// Evaluates the NOT BETWEEN predicate with the given lower and upper bounds.
-    virtual Result<T> VisitNotBetween(const Literal& from, const Literal& to) {
-        // Default implementation: NOT BETWEEN is equivalent to < OR >
-        auto lower_result = VisitLessThan(from);
-        if (!lower_result.ok()) {
-            return lower_result.status();
-        }
-        auto upper_result = VisitGreaterThan(to);
-        if (!upper_result.ok()) {
-            return upper_result.status();
-        }
-        return VisitOr({std::move(lower_result).value(), std::move(upper_result).value()});
-    }
-
-    // ----------------- Compound functions ------------------------
-
-    /// Evaluates the AND predicate across multiple child results.
-    virtual Result<T> VisitAnd(const std::vector<Result<T>>& children) = 0;
-
-    /// Evaluates the OR predicate across multiple child results.
-    virtual Result<T> VisitOr(const std::vector<Result<T>>& children) = 0;
 };
 }  // namespace paimon

src/paimon/CMakeLists.txt

@@ -413,8 +413,7 @@ if(PAIMON_BUILD_TESTS)
                     common/global_index/bitmap/bitmap_global_index_test.cpp
                     common/global_index/btree/btree_index_meta_test.cpp
                     common/global_index/btree/btree_file_footer_test.cpp
-                    common/global_index/btree/btree_global_indexer_test.cpp
-                    common/global_index/btree/btree_global_index_writer_test.cpp
+                    common/global_index/btree/key_serializer_test.cpp
                     common/global_index/btree/btree_global_index_integration_test.cpp
                     common/global_index/btree/btree_compatibility_test.cpp
                     common/global_index/rangebitmap/range_bitmap_global_index_test.cpp

src/paimon/common/data/binary_row.cpp

@@ -18,6 +18,7 @@
 
 #include <cstdint>
 
+#include "fmt/format.h"
 #include "paimon/common/data/binary_data_read_utils.h"
 #include "paimon/common/memory/memory_segment.h"
 #include "paimon/common/memory/memory_segment_utils.h"
@@ -300,4 +301,8 @@ int32_t BinaryRow::HashCode() const {
     return MemorySegmentUtils::HashByWords({segment_}, offset_, size_in_bytes_, nullptr);
 }
 
+std::string BinaryRow::ToString() const {
+    return fmt::format("BinaryRow@{:#x}", static_cast<uint32_t>(HashCode()));
+}
+
 }  // namespace paimon

src/paimon/common/data/binary_row.h

@@ -112,11 +112,7 @@ class BinaryRow final : public BinarySection, public InternalRow, public DataSet
     bool operator==(const BinaryRow& other) const;
     // TODO(liancheng.lsz): single column to be implemented
 
-    std::string ToString() const override {
-        std::stringstream ss;
-        ss << std::hex << static_cast<uint32_t>(HashCode());
-        return "BinaryRow@" + ss.str();
-    }
+    std::string ToString() const override;
 
     int32_t HashCode() const override;
 

src/paimon/common/defs.cpp

@@ -123,11 +123,4 @@ const char Options::LOOKUP_CACHE_HIGH_PRIO_POOL_RATIO[] = "lookup.cache.high-pri
 const char Options::BUCKET_FUNCTION_TYPE[] = "bucket-function.type";
 const char Options::LOOKUP_CACHE_FILE_RETENTION[] = "lookup.cache-file-retention";
 const char Options::LOOKUP_CACHE_MAX_DISK_SIZE[] = "lookup.cache-max-disk-size";
-const char Options::BTREE_INDEX_COMPRESSION[] = "btree-index.compression";
-const char Options::BTREE_INDEX_COMPRESSION_LEVEL[] = "btree-index.compression-level";
-const char Options::BTREE_INDEX_BLOCK_SIZE[] = "btree-index.block-size";
-const char Options::BTREE_INDEX_CACHE_SIZE[] = "btree-index.cache-size";
-const char Options::BTREE_INDEX_HIGH_PRIORITY_POOL_RATIO[] = "btree-index.high-priority-pool-ratio";
-const char Options::BTREE_INDEX_RECORDS_PER_RANGE[] = "btree-index.records-per-range";
-
 }  // namespace paimon

src/paimon/common/file_index/empty/empty_file_index_reader.h

@@ -65,16 +65,6 @@ class EmptyFileIndexReader : public FileIndexReader {
         const std::vector<Literal>& literals) override {
         return FileIndexResult::Skip();
     }
-
-    Result<std::shared_ptr<FileIndexResult>> VisitBetween(const Literal& from,
-                                                          const Literal& to) override {
-        return FileIndexResult::Skip();
-    }
-
-    Result<std::shared_ptr<FileIndexResult>> VisitNotBetween(const Literal& from,
-                                                             const Literal& to) override {
-        return FileIndexResult::Remain();
-    }
 };
 
 }  // namespace paimon

src/paimon/common/file_index/file_index_reader.cpp

@@ -98,44 +98,4 @@ Result<std::shared_ptr<FileIndexResult>> FileIndexReader::VisitNotIn(
     }
     return file_index_result;
 }
-
-Result<std::shared_ptr<FileIndexResult>> FileIndexReader::VisitAnd(
-    const std::vector<Result<std::shared_ptr<FileIndexResult>>>& children) {
-    if (children.empty()) {
-        return Status::Invalid("VisitAnd called with no children");
-    }
-
-    // Start with the first child
-    PAIMON_RETURN_NOT_OK(children[0]);
-    auto current = children[0].value();
-
-    // AND with remaining children
-    for (size_t i = 1; i < children.size(); ++i) {
-        PAIMON_RETURN_NOT_OK(children[i]);
-        auto child = children[i].value();
-        PAIMON_ASSIGN_OR_RAISE(current, current->And(child));
-    }
-
-    return current;
-}
-
-Result<std::shared_ptr<FileIndexResult>> FileIndexReader::VisitOr(
-    const std::vector<Result<std::shared_ptr<FileIndexResult>>>& children) {
-    if (children.empty()) {
-        return Status::Invalid("VisitOr called with no children");
-    }
-
-    // Start with the first child
-    PAIMON_RETURN_NOT_OK(children[0]);
-    auto current = children[0].value();
-
-    // OR with remaining children
-    for (size_t i = 1; i < children.size(); ++i) {
-        PAIMON_RETURN_NOT_OK(children[i]);
-        auto child = children[i].value();
-        PAIMON_ASSIGN_OR_RAISE(current, current->Or(child));
-    }
-
-    return current;
-}
 }  // namespace paimon

src/paimon/common/global_index/CMakeLists.txt

@@ -21,6 +21,7 @@ set(PAIMON_GLOBAL_INDEX_SRC
     btree/btree_global_index_reader.cpp
     btree/btree_global_index_writer.cpp
     btree/btree_index_meta.cpp
+    btree/key_serializer.cpp
     rangebitmap/range_bitmap_global_index.cpp
     rangebitmap/range_bitmap_global_index_factory.cpp)