[Issue #1080] support loadable duckdb extension for official duckdb releases and fix some bugs

cpp/pixels-cpp.properties → cpp/etc/pixels-cpp.properties

@@ -21,7 +21,7 @@ pixel.column.size.path=/home/whz/pixels/clickbench-size-e0.csv
 
 # the work thread to run parquet. -1 means using all CPU cores
 parquet.threads=-1
-
+pixels.doublebuffer=true
 # storage device identifier directory depth
 # this parameter defines the directory depth that determines the storage device.
 # for example, we have three SSDs, the path is /data/ssd1, /data/ssd2 and /data/ssd3, so the depth is 2
@@ -57,10 +57,9 @@ pixel.bufferpool.bufferpoolSize=356596096
 
 pixel.bufferpool.fixedsize=false
 pixel.bufferpool.bufferNum=20
-pixel.bufferpool.hugepage=true
+pixel.bufferpool.hugepage=false
 
 # 0 interrupt-dirven
 # 1 IORING_SETUP_IOPOLL
 # 2 IORING_SETUP_SQPOLL
 pixels.io_uring.mode=0
-

cpp/include/PixelsScanFunction.hpp

@@ -70,10 +70,8 @@
 #include "duckdb/catalog/catalog.hpp"
 #include "duckdb/common/constants.hpp"
 #include "duckdb/common/enums/file_compression_type.hpp"
-//#include "duckdb/common/field_writer.hpp"
 #include "duckdb/common/file_system.hpp"
 #include "duckdb/common/hive_partitioning.hpp"
-//#include "duckdb/common/types/chunk_collection.hpp"
 #include "duckdb/function/copy_function.hpp"
 #include "duckdb/function/table_function.hpp"
 #include "duckdb/main/client_context.hpp"
@@ -87,6 +85,8 @@
 #include "duckdb/storage/statistics/base_statistics.hpp"
 #include "duckdb/catalog/catalog_entry/table_function_catalog_entry.hpp"
 #include "duckdb/common/multi_file/multi_file_reader.hpp"
+#include "filter/table_filter.hpp"
+#include "filter/constant_filter.hpp"
 
 #endif
 
@@ -117,9 +117,8 @@ namespace duckdb
                                             PixelsReadLocalState &scan_data, PixelsReadGlobalState &parallel_state,
                                             bool is_init_state = false);
 
-        static PixelsReaderOption
-        GetPixelsReaderOption(PixelsReadLocalState &local_state, PixelsReadGlobalState &global_state);
-
+        static PixelsReaderOption GetPixelsReaderOption(PixelsReadLocalState &local_state, PixelsReadGlobalState &global_state);
+        static pixels::TableFilterSet ConvertDuckDBFilter(TableFilterSet *filters);
     private:
         static void TransformDuckdbType(const std::shared_ptr <TypeDescription> &type,
                                         vector <LogicalType> &return_types);
@@ -133,4 +132,4 @@ namespace duckdb
     };
 
 } // namespace duckdb
-#endif // EXAMPLE_C_PIXELSSCANFUNCTION_HPP
+#endif // EXAMPLE_C_PIXELSSCANFUNCTION_HPP

cpp/pixels-common/CMakeLists.txt

@@ -53,4 +53,4 @@ link_directories(${CMAKE_CURRENT_BINARY_DIR}/liburing/src)
 message(${CMAKE_CURRENT_BINARY_DIR}/liburing/src)
 target_link_libraries(pixels-common
         ${Protobuf_LIBRARIES}
-		${CMAKE_CURRENT_BINARY_DIR}/liburing/src/liburing.a)
+		${CMAKE_CURRENT_BINARY_DIR}/liburing/src/liburing.a)

cpp/pixels-common/lib/utils/ConfigFactory.cpp

@@ -53,8 +53,8 @@ ConfigFactory::ConfigFactory()
     {
         pixelsHome += "/";
     }
-    std::ifstream infile(pixelsHome + "etc/pixels-cpp.properties");
-    std::cout << "pixels properties file is " << pixelsHome + "etc/pixels-cpp.properties" << std::endl;
+    std::ifstream infile(pixelsHome + "cpp/etc/pixels-cpp.properties");
+    std::cout << "pixels properties file is " << pixelsHome + "cpp/etc/pixels-cpp.properties" << std::endl;
     std::string line;
     while (std::getline(infile, line))
     {

cpp/pixels-core/include/PixelsBitMask.h

@@ -26,13 +26,6 @@
 #define DUCKDB_PIXELSBITMASK_H
 
 #include <bitset>
-#include "duckdb/planner/table_filter.hpp"
-#include "duckdb/common/vector_size.hpp"
-#include "duckdb/planner/filter/constant_filter.hpp"
-#include "duckdb/planner/filter/null_filter.hpp"
-#include "duckdb/planner/filter/conjunction_filter.hpp"
-#include "duckdb/common/operator/comparison_operators.hpp"
-
 #include "vector/ColumnVector.h"
 #include "TypeDescription.h"
 
@@ -68,4 +61,4 @@ class PixelsBitMask
     uint8_t get(long index);
 };
 
-#endif //DUCKDB_PIXELSBITMASK_H
+#endif //DUCKDB_PIXELSBITMASK_H

cpp/pixels-core/include/PixelsFilter.h

@@ -26,38 +26,54 @@
 #define DUCKDB_PIXELSFILTER_H
 
 #include <bitset>
-#include "duckdb/planner/table_filter.hpp"
-#include "duckdb/common/vector_size.hpp"
-#include "duckdb/planner/filter/constant_filter.hpp"
-#include "duckdb/planner/filter/null_filter.hpp"
-#include "duckdb/planner/filter/conjunction_filter.hpp"
-#include "duckdb/common/operator/comparison_operators.hpp"
 #include "PixelsBitMask.h"
 #include "vector/ColumnVector.h"
 #include "TypeDescription.h"
 #include <immintrin.h>
 #include <avxintrin.h>
+#include "filter/table_filter.hpp"
+#include "filter/constant_filter.hpp"
+#include "filter/conjunction_filter.hpp"
+#include "filter/comparison_operators.hpp"
 
 #define ENABLE_SIMD_FILTER
 
 class PixelsFilter
 {
 public:
-    static void ApplyFilter(std::shared_ptr <ColumnVector> vector, duckdb::TableFilter &filter,
+    static void ApplyFilter(std::shared_ptr <ColumnVector> vector, const pixels::TableFilter &filter,
                             PixelsBitMask &filterMask,
                             std::shared_ptr <TypeDescription> type);
 
     template<class T, class OP>
     static int CompareAvx2(void *data, T constant);
 
-    template<class T, class OP>
+    /*template<class T, class OP>
     static void TemplatedFilterOperation(std::shared_ptr <ColumnVector> vector,
-                                         const duckdb::Value &constant, PixelsBitMask &filter_mask,
-                                         std::shared_ptr <TypeDescription> type);
+                                         const pixels::Scalar &constant, PixelsBitMask &filter_mask,
+                                         std::shared_ptr <TypeDescription> type);*/
+    template<class OP>
+    static void IntFilterOperation(std::shared_ptr <ColumnVector> vector,
+                                    const pixels::Scalar &constant, PixelsBitMask &filter_mask);
+
+    template<class OP>
+    static void LongFilterOperation(std::shared_ptr <ColumnVector> vector,
+                                    const pixels::Scalar &constant, PixelsBitMask &filter_mask);
+    template<class OP>
+    static void DateFilterOperation(std::shared_ptr <ColumnVector> vector,
+                                    const pixels::Scalar &constant, PixelsBitMask &filter_mask);
+
+    template<class OP>
+    static void DecimalFilterOperation(std::shared_ptr <ColumnVector> vector,
+                                    const pixels::Scalar &constant, PixelsBitMask &filter_mask);    
+
+    template<class OP>
+    static void StringFilterOperation(std::shared_ptr <ColumnVector> vector,
+                                    const pixels::Scalar &constant, PixelsBitMask &filter_mask);                                    
 
     template<class OP>
-    static void FilterOperationSwitch(std::shared_ptr <ColumnVector> vector, duckdb::Value &constant,
+    static void FilterOperationSwitch(std::shared_ptr <ColumnVector> vector, const pixels::Scalar &constant,
                                       PixelsBitMask &filter_mask, std::shared_ptr <TypeDescription> type);
 
 };
-#endif //DUCKDB_PIXELSFILTER_H
+#endif //DUCKDB_PIXELSFILTER_H

cpp/pixels-core/include/PixelsTypes.h

@@ -0,0 +1,227 @@
+/*
+ * Copyright 2026 PixelsDB.
+ *
+ * This file is part of Pixels.
+ *
+ * Pixels is free software: you can redistribute it and/or modify
+ * it under the terms of the Affero GNU General Public License as
+ * published by the Free Software Foundation, either version 3 of
+ * the License, or (at your option) any later version.
+ *
+ * Pixels is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * Affero GNU General Public License for more details.
+ *
+ * You should have received a copy of the Affero GNU General Public
+ * License along with Pixels.  If not, see
+ * <https://www.gnu.org/licenses/>.
+ */
+
+
+#pragma once
+//some common types used in pixels extension
+
+#ifndef PIXELS_TYPES_H
+#define PIXELS_TYPES_H
+#include <variant>
+#include <cstdint>
+#include <cassert>
+#include <string>
+#include <cstring>
+using idx_t = uint64_t;
+namespace pixels {
+
+enum class PhysicalType : uint8_t {//physical type of decimal, used for decimal column statistics
+    INT16,
+    INT32,
+    INT64,
+    INT128,
+};
+
+struct DecimalConfig {//decimal config, used for decimal column statistics
+        static constexpr int MAX_WIDTH_INT16 = 4;
+        static constexpr int MAX_WIDTH_INT32 = 9;
+        static constexpr int MAX_WIDTH_INT64 = 18;
+        static constexpr int MAX_WIDTH_INT128 = 38;
+};
+
+enum class ComparisonOperator : uint8_t {
+        EQUAL,
+        NOT_EQUAL,
+        GREATER_THAN,
+        GREATER_THAN_OR_EQUAL,
+        LESS_THAN,
+        LESS_THAN_OR_EQUAL,
+        IS_NULL,
+        IS_NOT_NULL
+};
+
+enum class TableFilterType : uint8_t {
+	CONSTANT_COMPARISON = 0, // constant comparison (e.g. =C, >C, >=C, <C, <=C)
+	IS_NULL = 1,             // C IS NULL 
+	IS_NOT_NULL = 2,         // C IS NOT NULL
+	CONJUNCTION_OR = 3,      // OR of different filters
+	CONJUNCTION_AND = 4,     // AND of different filters
+	STRUCT_EXTRACT = 5,      // filter applies to child-column of struct 
+	OPTIONAL_FILTER = 6,     // executing filter is not required for query correctness
+	IN_FILTER = 7,           // col IN (C1, C2, C3, ...)    
+	DYNAMIC_FILTER = 8,      // dynamic filters can be updated at run-time 
+	EXPRESSION_FILTER = 9,   // an arbitrary expression     
+    DEFAULT=10               //
+};
+
+struct Scalar {
+    enum class Type {
+        INT32,
+        INT64,
+        FLOAT,
+        DOUBLE,
+        STRING,
+        INVALID
+    };
+
+private:
+    Type type_;
+
+    int32_t i32_;
+    int64_t i64_;
+    float f_;
+    double d_;
+    std::string str_;
+
+public:
+    Scalar() : type_(Type::INVALID) {}
+
+    Scalar(int32_t v) : type_(Type::INT32), i32_(v) {}
+    Scalar(int64_t v) : type_(Type::INT64), i64_(v) {}
+    Scalar(float v) : type_(Type::FLOAT), f_(v) {}
+    Scalar(double v) : type_(Type::DOUBLE), d_(v) {}
+    Scalar(const std::string &v) : type_(Type::STRING), str_(v) {}
+    Scalar(std::string &&v) : type_(Type::STRING), str_(std::move(v)) {}
+
+    Type type() const {
+        return type_;
+    }
+
+    bool is_int32() const { return type_ == Type::INT32; }
+    bool is_int64() const { return type_ == Type::INT64; }
+    bool is_float() const { return type_ == Type::FLOAT; }
+    bool is_double() const { return type_ == Type::DOUBLE; }
+    bool is_string() const { return type_ == Type::STRING; }
+
+    int32_t get_int32() const {
+        assert(type_ == Type::INT32);
+        return i32_;
+    }
+
+    int64_t get_int64() const {
+        assert(type_ == Type::INT64);
+        return i64_;
+    }
+
+    float get_float() const {
+        assert(type_ == Type::FLOAT);
+        return f_;
+    }
+
+    double get_double() const {
+        assert(type_ == Type::DOUBLE);
+        return d_;
+    }
+
+    const std::string &get_string() const {
+        assert(type_ == Type::STRING);
+        return str_;
+    }
+
+    void set(int32_t v) {
+        type_ = Type::INT32;
+        i32_ = v;
+    }
+
+    void set(int64_t v) {
+        type_ = Type::INT64;
+        i64_ = v;
+    }
+
+    void set(float v) {
+        type_ = Type::FLOAT;
+        f_ = v;
+    }
+
+    void set(double v) {
+        type_ = Type::DOUBLE;
+        d_ = v;
+    }
+
+    void set(const std::string &v) {
+        type_ = Type::STRING;
+        str_ = v;
+    }
+
+    void set(std::string &&v) {
+        type_ = Type::STRING;
+        str_ = std::move(v);
+    }
+};
+
+struct string_t {
+public:
+    static constexpr uint32_t INLINE_LENGTH = 12;
+    string_t() {
+        value.inlined.length = 0;
+    }
+    string_t(const char *data, uint32_t len) {
+        value.inlined.length = len;
+        if (IsInlined()) {
+            // small string: store inlined, prefix is not used
+            memset(value.inlined.inlined, 0, INLINE_LENGTH);
+            if (len > 0) {
+                memcpy(value.inlined.inlined, data, len);
+            }
+        } else {
+            // large string: store pointer, prefix is the first 4 bytes of data
+            memcpy(value.pointer.prefix, data, 4);
+            value.pointer.ptr = const_cast<char *>(data);
+        }
+    }
+
+    string_t(const std::string &str)
+        : string_t(str.data(), (uint32_t)str.size()) {}
+
+    inline bool IsInlined() const {
+        return GetSize() <= INLINE_LENGTH;
+    }
+
+    inline const char *GetData() const {
+        return IsInlined() ? value.inlined.inlined : value.pointer.ptr;
+    }
+
+    inline uint32_t GetSize() const {
+        return value.inlined.length;
+    }
+
+    inline std::string ToString() const {
+        return std::string(GetData(), GetSize());
+    }
+
+private:
+    union {
+        struct {
+            uint32_t length;
+            char prefix[4];
+            char *ptr;
+        } pointer;
+
+        struct {
+            uint32_t length;
+            char inlined[INLINE_LENGTH];
+        } inlined;
+    } value;
+};
+
+
+} // namespace pixels
+
+#endif // PIXELS_TYPES_H