Write with parallel

cpp/CMakeLists.txt

@@ -150,6 +150,15 @@ if (ENABLE_ZLIB)
     add_definitions(-DENABLE_GZIP)
 endif()
 
+option(ENABLE_THREADS "Enable multi-threaded read/write (requires pthreads)" ON)
+message("cmake using: ENABLE_THREADS=${ENABLE_THREADS}")
+
+if (ENABLE_THREADS)
+    add_definitions(-DENABLE_THREADS)
+    find_package(Threads REQUIRED)
+    link_libraries(Threads::Threads)
+endif()
+
 option(ENABLE_SIMDE "Enable SIMDe (SIMD Everywhere)" OFF)
 message("cmake using: ENABLE_SIMDE=${ENABLE_SIMDE}")
 

cpp/README-zh.md

@@ -105,6 +105,35 @@ make
 
 ---
 
+## 并行写入
+
+TsFile C++ 支持基于线程池的列级并行编码，适用于表模型写入路径（`write_table`）。启用后，时间列和所有值列使用预计算的 page 边界并行写入，同时保证各列 page 对齐封盘。
+
+### 编译选项
+
+并行写入通过 `ENABLE_THREADS` CMake 选项控制（默认开启）：
+
+```bash
+cmake .. -DENABLE_THREADS=ON   # 开启（默认）
+cmake .. -DENABLE_THREADS=OFF  # 关闭——编译期剥离所有线程代码
+```
+
+### 运行时配置
+
+```cpp
+#include "common/global.h"
+
+// 运行时开启或关闭并行写入（单核机器自动禁用）
+storage::set_parallel_write_enabled(true);
+
+// 设置工作线程数（必须在创建 TsFileWriter 之前调用）
+storage::set_write_thread_count(4);
+```
+
+默认情况下，当机器 CPU 核数大于 1 时自动启用并行写入，线程数设为硬件核数（上限 64）。
+
+---
+
 ## 使用 TsFile
 
 你可以在 `./examples/cpp_examples` 目录下的 `demo_read.cpp` 和 `demo_write.cpp` 中查看读写数据的示例。

cpp/README.md

@@ -75,6 +75,33 @@ cmake .. -DToolChain=ON
 make
 ```
 
+## Parallel Write
+
+TsFile C++ supports thread pool-based parallel column encoding for the table write path (`write_table`). When enabled, each column (time and value columns) is written in parallel using precomputed page boundaries, while maintaining aligned page sealing across columns.
+
+### Build Options
+
+Parallel write is controlled by the `ENABLE_THREADS` CMake option (ON by default):
+
+```bash
+cmake .. -DENABLE_THREADS=ON   # enable (default)
+cmake .. -DENABLE_THREADS=OFF  # disable — all thread code is stripped at compile time
+```
+
+### Runtime Configuration
+
+```cpp
+#include "common/global.h"
+
+// Enable or disable parallel write at runtime (auto-disabled on single-core machines)
+storage::set_parallel_write_enabled(true);
+
+// Set the number of worker threads (must be called before creating TsFileWriter)
+storage::set_write_thread_count(4);
+```
+
+By default, parallel write is enabled when the machine has more than one CPU core, and the thread count is set to the number of hardware cores (capped at 64).
+
 ## Use TsFile
 
 You can find examples on how to read and write data in `demo_read.cpp` and `demo_write.cpp` located under `./examples/cpp_examples`. There are also examples under `./examples/c_examples`on how to use a C-style API to read and write data in a C environment. You can run `bash build.sh` under `./examples` to generate an executable output under `./examples/build`.

cpp/src/common/config/config.h

@@ -46,6 +46,8 @@ typedef struct ConfigValue {
     TSEncoding double_encoding_type_;
     TSEncoding string_encoding_type_;
     CompressionType default_compression_type_;
+    bool parallel_write_enabled_;
+    int32_t write_thread_count_;
     // When true, aligned writer enforces page size limit strictly by
     // interleaving time/value writes and sealing pages together when any side
     // becomes full.

cpp/src/common/global.cc

@@ -24,11 +24,19 @@
 #endif
 #include <stdlib.h>
 
+#include <thread>
+
+#ifdef ENABLE_THREADS
+#include "common/thread_pool.h"
+#endif
 #include "utils/injection.h"
 
 namespace common {
 
 ColumnSchema g_time_column_schema;
+#ifdef ENABLE_THREADS
+ThreadPool* g_write_thread_pool_ = nullptr;
+#endif
 ConfigValue g_config_value_;
 
 void init_config_value() {
@@ -54,12 +62,18 @@ void init_config_value() {
     g_config_value_.int64_encoding_type_ = TS_2DIFF;
     g_config_value_.float_encoding_type_ = GORILLA;
     g_config_value_.double_encoding_type_ = GORILLA;
+    g_config_value_.string_encoding_type_ = PLAIN;
     // Default compression type is LZ4
 #ifdef ENABLE_LZ4
     g_config_value_.default_compression_type_ = LZ4;
 #else
     g_config_value_.default_compression_type_ = UNCOMPRESSED;
 #endif
+    unsigned int hw_cores = std::thread::hardware_concurrency();
+    if (hw_cores == 0) hw_cores = 1;  // fallback if detection fails
+    g_config_value_.parallel_write_enabled_ = (hw_cores > 1);
+    g_config_value_.write_thread_count_ =
+        static_cast<int32_t>(std::min(hw_cores, 64u));
     // Enforce aligned page size limits strictly by default.
     g_config_value_.strict_page_size_ = true;
 }
@@ -129,6 +143,15 @@ int init_common() {
     g_time_column_schema.encoding_ = PLAIN;
     g_time_column_schema.compression_ = UNCOMPRESSED;
     g_time_column_schema.column_name_ = storage::TIME_COLUMN_NAME;
+#ifdef ENABLE_THREADS
+    // (Re)create the global write thread pool with the configured size.
+    delete g_write_thread_pool_;
+    size_t pool_size =
+        g_config_value_.write_thread_count_ > 0
+            ? static_cast<size_t>(g_config_value_.write_thread_count_)
+            : size_t{1};
+    g_write_thread_pool_ = new ThreadPool(pool_size);
+#endif
     return ret;
 }
 

cpp/src/common/global.h

@@ -163,6 +163,30 @@ FORCE_INLINE uint8_t get_global_compression() {
     return static_cast<uint8_t>(g_config_value_.default_compression_type_);
 }
 
+FORCE_INLINE void set_parallel_write_enabled(bool enabled) {
+    g_config_value_.parallel_write_enabled_ = enabled;
+}
+
+FORCE_INLINE bool get_parallel_write_enabled() {
+    return g_config_value_.parallel_write_enabled_ &&
+           g_config_value_.write_thread_count_ > 1;
+}
+
+// Set the number of threads for parallel writes.  Must be called before
+// init_common() / libtsfile_init() — the global thread pool is created
+// during initialization and is not resized at runtime.
+FORCE_INLINE int set_write_thread_count(int32_t count) {
+    if (count < 1 || count > 64) return E_INVALID_ARG;
+    g_config_value_.write_thread_count_ = count;
+    return E_OK;
+}
+
+#ifdef ENABLE_THREADS
+class ThreadPool;
+// Global write thread pool, created by init_common().
+extern ThreadPool* g_write_thread_pool_;
+#endif
+
 extern int init_common();
 extern bool is_timestamp_column_name(const char* time_col_name);
 extern void cols_to_json(ByteStream* byte_stream,

cpp/src/common/tablet.h

@@ -46,6 +46,7 @@ class TabletColIterator;
  * with their associated metadata such as column names and types.
  */
 class Tablet {
+   public:
     // Arrow-style string column: offsets + contiguous buffer.
     // string[i] = buffer + offsets[i], len = offsets[i+1] - offsets[i]
     struct StringColumn {
@@ -235,6 +236,12 @@ class Tablet {
     }
     size_t get_column_count() const { return schema_vec_->size(); }
     uint32_t get_cur_row_size() const { return cur_row_size_; }
+    int64_t get_timestamp(uint32_t row_index) const {
+        return timestamps_[row_index];
+    }
+    bool is_null(uint32_t row_index, uint32_t col_index) const {
+        return bitmaps_[col_index].test(row_index);
+    }
 
     /**
      * @brief Adds a timestamp to the specified row.

cpp/src/common/thread_pool.h

@@ -0,0 +1,123 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * License); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+#ifndef COMMON_THREAD_POOL_H
+#define COMMON_THREAD_POOL_H
+
+#ifdef ENABLE_THREADS
+
+#include <condition_variable>
+#include <functional>
+#include <future>
+#include <mutex>
+#include <queue>
+#include <thread>
+#include <type_traits>
+#include <vector>
+
+namespace common {
+
+// Unified fixed-size thread pool supporting both fire-and-forget tasks
+// (submit void + wait_all) and future-returning tasks (submit<F>).
+// Used by both write path (column-parallel encoding) and read path
+// (column-parallel decoding).
+class ThreadPool {
+   public:
+    explicit ThreadPool(size_t num_threads) : stop_(false), active_(0) {
+        for (size_t i = 0; i < num_threads; i++) {
+            workers_.emplace_back([this] { worker_loop(); });
+        }
+    }
+
+    ~ThreadPool() {
+        {
+            std::lock_guard<std::mutex> lk(mu_);
+            stop_ = true;
+        }
+        cv_work_.notify_all();
+        for (auto& w : workers_) {
+            if (w.joinable()) w.join();
+        }
+    }
+
+    // Submit a fire-and-forget task (no return value).
+    void submit(std::function<void()> task) {
+        {
+            std::lock_guard<std::mutex> lk(mu_);
+            tasks_.push(std::move(task));
+            active_++;
+        }
+        cv_work_.notify_one();
+    }
+
+    // Submit a task that returns a value via std::future.
+    template <typename F>
+    std::future<typename std::result_of<F()>::type> submit(F&& f) {
+        using RetType = typename std::result_of<F()>::type;
+        auto task =
+            std::make_shared<std::packaged_task<RetType()>>(std::forward<F>(f));
+        std::future<RetType> result = task->get_future();
+        {
+            std::lock_guard<std::mutex> lk(mu_);
+            tasks_.emplace([task]() { (*task)(); });
+            active_++;
+        }
+        cv_work_.notify_one();
+        return result;
+    }
+
+    // Block until all submitted tasks have completed.
+    void wait_all() {
+        std::unique_lock<std::mutex> lk(mu_);
+        cv_done_.wait(lk, [this] { return active_ == 0 && tasks_.empty(); });
+    }
+
+   private:
+    void worker_loop() {
+        while (true) {
+            std::function<void()> task;
+            {
+                std::unique_lock<std::mutex> lk(mu_);
+                cv_work_.wait(lk, [this] { return stop_ || !tasks_.empty(); });
+                if (stop_ && tasks_.empty()) return;
+                task = std::move(tasks_.front());
+                tasks_.pop();
+            }
+            task();
+            {
+                std::lock_guard<std::mutex> lk(mu_);
+                active_--;
+            }
+            cv_done_.notify_one();
+        }
+    }
+
+    std::vector<std::thread> workers_;
+    std::queue<std::function<void()>> tasks_;
+    std::mutex mu_;
+    std::condition_variable cv_work_;
+    std::condition_variable cv_done_;
+    bool stop_;
+    int active_;
+};
+
+}  // namespace common
+
+#endif  // ENABLE_THREADS
+
+#endif  // COMMON_THREAD_POOL_H