Suppor C++ Tsfile V4 Read. #408

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed

zwhzzz0821 wants to merge 83 commits into apache:develop from hongzhi-gao:tmp

cpp/CMakeLists.txt

-Original file line number
+Diff line change
@@ Expand Up / @@ -24,7 +24,6 @@ set(TsFile_CPP_VERSION 2.0.0.dev) @@
     set(CMAKE_CXX_FLAGS "$ENV{CXXFLAGS} -Wall -Werror")
     message("cmake using: USE_CPP11=${USE_CPP11}")
     set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11")
     if(DEFINED ENV{CXX})
@@ Expand Down @@

cpp/src/common/allocator/alloc_base.h

-Original file line number
+Diff line change
@@ Expand Up / @@ -65,7 +65,10 @@ enum AllocModID { @@
         MOD_BITENCODE_OBJ = 30,
         MOD_DICENCODE_OBJ = 31,
         MOD_ZIGZAG_OBJ = 32,
-        __LAST_MOD_ID = 33,  // prev + 1,
+        MOD_DEVICE_META_ITER = 33,
+        MOD_DEVICE_TASK_ITER = 34,
+        MOD_DEVICE_ORDER_TSBLOCK_READER = 35,
+        __LAST_MOD_ID = 36,  // prev + 1,
         __MAX_MOD_ID = 127,  // leave 1 bit to detect header size
     };
@@ Expand Down @@

cpp/src/common/allocator/my_string.h

-Original file line number
+Diff line change
@@ Expand Up / @@ -35,6 +35,13 @@ struct String { @@
         String() : buf_(nullptr), len_(0) {}
         String(char *buf, uint32_t len) : buf_(buf), len_(len) {}
+        String(const std::string& str, common::PageArena& pa) : buf_(nullptr), len_(0) {
+            dup_from(str, pa);
+        }
+        String(const std::string& str) {
+            buf_ = (char*)str.c_str();
+            len_ = str.size();
+        }
         FORCE_INLINE bool is_null() const { return buf_ == nullptr && len_ == 0; }
         FORCE_INLINE void reset() {
             len_ = 0;
@@ Expand Down Expand Up / @@ -128,6 +135,18 @@ struct String { @@
             }
         }
+        FORCE_INLINE void max(const String &that, common::PageArena &pa) {
+            if (compare(that) < 0) {
+                this->dup_from(that, pa);
+            }
+        }
+        FORCE_INLINE void min(const String &that, common::PageArena &pa) {
+            if (compare(that) > 0) {
+                this->dup_from(that, pa);
+            }
+        }
         bool operator<(const String &other) const {
             if (this->is_null() && other.is_null()) {
                 return false;
@@ Expand All / @@ -147,7 +166,7 @@ struct String { @@
             return this->len_ < other.len_;
         }
-        std::string to_std_string() { return std::string(buf_, len_); }
+        std::string to_std_string() const { return std::string(buf_, len_); }
     #ifndef NDEBUG
         friend std::ostream &operator<<(std::ostream &os, const String &s) {
@@ Expand Down @@

cpp/src/common/cache/CMakeLists.txt

-Original file line number
+Diff line change
@@ -0,0 +1,20 @@
+    #[[
+    Licensed to the Apache Software Foundation (ASF) under one
+    or more contributor license agreements.  See the NOTICE file
+    distributed with this work for additional information
+    regarding copyright ownership.  The ASF licenses this file
+    to you under the Apache License, Version 2.0 (the
+    "License"); you may not use this file except in compliance
+    with the License.  You may obtain a copy of the License at
+        https://www.apache.org/licenses/LICENSE-2.0
+    Unless required by applicable law or agreed to in writing,
+    software distributed under the License is distributed on an
+    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+    KIND, either express or implied.  See the License for the
+    specific language governing permissions and limitations
+    under the License.
+    ]]

cpp/src/common/cache/lru_cache.h

-Original file line number
+Diff line change
@@ -0,0 +1,226 @@
+    /*
+     * Licensed to the Apache Software Foundation (ASF) under one
+     * or more contributor license agreements.  See the NOTICE file
+     * distributed with this work for additional information
+     * regarding copyright ownership.  The ASF licenses this file
+     * to you under the Apache License, Version 2.0 (the
+     * License); you may not use this file except in compliance
+     * with the License.  You may obtain a copy of the License at
+     *
+     *     http://www.apache.org/licenses/LICENSE-2.0
+     *
+     * Unless required by applicable law or agreed to in writing,
+     * software distributed under the License is distributed on an
+     * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+     * KIND, either express or implied.  See the License for the
+     * specific language governing permissions and limitations
+     * under the License.
+     */
+    #ifndef COMMON_CACHE_LRU_CACHE_H
+    #define COMMON_CACHE_LRU_CACHE_H
+    #include <unordered_map>
+    #include <list>
+    #include <mutex>
+    #include <algorithm>
+    #include "utils/errno_define.h"
+    namespace common {
+    /*
+     * a noop lockable concept that can be used in place of std::mutex
+     */
+    class NullLock {
+       public:
+        void lock() {}
+        void unlock() {}
+        bool try_lock() { return true; }
+    };
+    template <typename K, typename V>
+    struct KeyValuePair {
+       public:
+        K key;
+        V value;
+        KeyValuePair(K k, V v) : key(std::move(k)), value(std::move(v)) {}
+    };
+    /**
+     *	The LRU Cache class templated by
+     *		Key - key type
+     *		Value - value type
+     *		MapType - an associative container like std::unordered_map
+     *		LockType - a lock type derived from the Lock class (default:
+     *NullLock = no synchronization)
+     *
+     *	The default NullLock based template is not thread-safe, however passing
+     *Lock=std::mutex will make it
+     *	thread-safe
+     */
+    template <class Key, class Value, class Lock = NullLock,
+              class Map = std::unordered_map<
+                  Key, typename std::list<KeyValuePair<Key, Value>>::iterator>>
+    class Cache {
+       public:
+        typedef KeyValuePair<Key, Value> node_type;
+        typedef std::list<KeyValuePair<Key, Value>> list_type;
+        typedef Map map_type;
+        typedef Lock lock_type;
+        using Guard = std::lock_guard<lock_type>;
+        /**
+         * the maxSize is the soft limit of keys and (maxSize + elasticity) is the
+         * hard limit
+         * the cache is allowed to grow till (maxSize + elasticity) and is pruned
+         * back to maxSize keys set maxSize = 0 for an unbounded cache (but in that
+         * case, you're better off using a std::unordered_map directly anyway! :)
+         */
+        explicit Cache(size_t maxSize = 64, size_t elasticity = 10)
+            : maxSize_(maxSize), elasticity_(elasticity) {}
+        virtual ~Cache() = default;
+        size_t size() const {
+            Guard g(lock_);
+            return cache_.size();
+        }
+        bool empty() const {
+            Guard g(lock_);
+            return cache_.empty();
+        }
+        void clear() {
+            Guard g(lock_);
+            cache_.clear();
+            keys_.clear();
+        }
+        void insert(const Key& k, Value v) {
+            Guard g(lock_);
+            const auto iter = cache_.find(k);
+            if (iter != cache_.end()) {
+                iter->second->value = v;
+                keys_.splice(keys_.begin(), keys_, iter->second);
+                return;
+            }
+            keys_.emplace_front(k, std::move(v));
+            cache_[k] = keys_.begin();
+            prune();
+        }
+        void emplace(const Key& k, Value&& v) {
+            Guard g(lock_);
+            keys_.emplace_front(k, std::move(v));
+            cache_[k] = keys_.begin();
+            prune();
+        }
+        /**
+          for backward compatibity. redirects to tryGetCopy()
+         */
+        bool tryGet(const Key& kIn, Value& vOut) { return tryGetCopy(kIn, vOut); }
+        bool tryGetCopy(const Key& kIn, Value& vOut) {
+            Guard g(lock_);
+            Value tmp;
+            if (!tryGetRef_nolock(kIn, tmp)) {
+                return false;
+            }
+            vOut = tmp;
+            return true;
+        }
+        bool tryGetRef(const Key& kIn, Value& vOut) {
+            Guard g(lock_);
+            return tryGetRef_nolock(kIn, vOut);
+        }
+        /**
+         *	The const reference returned here is only
+         *    guaranteed to be valid till the next insert/delete
+         *  in multi-threaded apps use getCopy() to be threadsafe
+         */
+        const Value& getRef(const Key& k) {
+            Guard g(lock_);
+            return get_nolock(k);
+        }
+        /**
+            added for backward compatibility
+         */
+        Value get(const Key& k) { return getCopy(k); }
+        /**
+         * returns a copy of the stored object (if found)
+         * safe to use/recommended in multi-threaded apps
+         */
+        Value getCopy(const Key& k) {
+            Guard g(lock_);
+            return get_nolock(k);
+        }
+        bool remove(const Key& k) {
+            Guard g(lock_);
+            auto iter = cache_.find(k);
+            if (iter == cache_.end()) {
+                return false;
+            }
+            keys_.erase(iter->second);
+            cache_.erase(iter);
+            return true;
+        }
+        bool contains(const Key& k) const {
+            Guard g(lock_);
+            return cache_.find(k) != cache_.end();
+        }
+        size_t getMaxSize() const { return maxSize_; }
+        size_t getElasticity() const { return elasticity_; }
+        size_t getMaxAllowedSize() const { return maxSize_ + elasticity_; }
+        template <typename F>
+        void cwalk(F& f) const {
+            Guard g(lock_);
+            std::for_each(keys_.begin(), keys_.end(), f);
+        }
+       protected:
+        const int get_nolock(const Key& k, Value& vOut) {
+            const auto iter = cache_.find(k);
+            if (iter == cache_.end()) {
+                return E_NOT_EXIST;
+            }
+            keys_.splice(keys_.begin(), keys_, iter->second);
+            vOut = iter->second->value;
+            return E_OK;
+        }
+        bool tryGetRef_nolock(const Key& kIn, Value& vOut) {
+            const auto iter = cache_.find(kIn);
+            if (iter == cache_.end()) {
+                return false;
+            }
+            keys_.splice(keys_.begin(), keys_, iter->second);
+            vOut = iter->second->value;
+            return true;
+        }
+        size_t prune() {
+            size_t maxAllowed = maxSize_ + elasticity_;
+            if (maxSize_ == 0 || cache_.size() < maxAllowed) {
+                return 0;
+            }
+            size_t count = 0;
+            while (cache_.size() > maxSize_) {
+                cache_.erase(keys_.back().key);
+                keys_.pop_back();
+                ++count;
+            }
+            return count;
+        }
+       private:
+        // Disallow copying.
+        Cache(const Cache&) = delete;
+        Cache& operator=(const Cache&) = delete;
+        mutable Lock lock_;
+        Map cache_;
+        list_type keys_;
+        size_t maxSize_;
+        size_t elasticity_;
+    };
+    }  // namespace common
+    #endif  // COMMON_CACHE_LRU_CACHE_H

cpp/src/common/config/config.h

-Original file line number
+Diff line change
@@ Expand Up / @@ -19,7 +19,7 @@ @@
     #ifndef COMMON_CONFIG_CONFIG_H
     #define COMMON_CONFIG_CONFIG_H
-    #include <stdint.h>
+    #include <cstdint>
     #include "common/mutex/mutex.h"
     #include "utils/db_utils.h"
@@ Expand All / @@ -46,10 +46,12 @@ typedef struct ConfigValue { @@
         CompressionType time_compress_type_;
         int32_t chunk_group_size_threshold_;
         int32_t record_count_for_next_mem_check_;
+        bool encrypt_flag_ = false;
     } ConfigValue;
     extern void init_config_value();
+    extern TSEncoding get_value_encoder(TSDataType data_type);
+    extern CompressionType get_default_compressor();
     // In the future, configuration items need to be dynamically adjusted according
     // to the level
     extern void set_config_value();
@@ Expand Down @@

cpp/src/common/db_common.h

            
                      Original file line number
                      Diff line number
                      Diff line change
                  
    @@ -23,22 +23,24 @@
  
    #include <iostream>

    #include "utils/util_define.h"

    #include "common/allocator/my_string.h"

    namespace common {

    enum TSDataType {

    enum TSDataType : uint8_t {

        BOOLEAN = 0,

        INT32 = 1,

        INT64 = 2,

        FLOAT = 3,

        DOUBLE = 4,

        TEXT = 5,

        VECTOR = 6,

        STRING = 11,

        NULL_TYPE = 254,

        INVALID_DATATYPE = 255

    };

    enum TSEncoding {

    enum TSEncoding : uint8_t {

        PLAIN = 0,

        DICTIONARY = 1,

        RLE = 2,

    @@ -53,7 +55,7 @@ enum TSEncoding {
  
        INVALID_ENCODING = 255

    };

    enum CompressionType {

    enum CompressionType : uint8_t {

        UNCOMPRESSED = 0,

        SNAPPY = 1,

        GZIP = 2,

    @@ -65,12 +67,12 @@ enum CompressionType {
  
        INVALID_COMPRESSION = 255

    };

    extern const char* s_data_type_names[7];

    extern const char* s_data_type_names[8];

    extern const char* s_encoding_names[12];

    extern const char* s_compression_names[8];

    FORCE_INLINE const char* get_data_type_name(TSDataType type) {

        ASSERT(type >= BOOLEAN && type <= VECTOR);

        ASSERT(type >= BOOLEAN && type <= STRING);

        return s_data_type_names[type];

    }

    @@ -148,6 +150,10 @@ template <>
  
    FORCE_INLINE common::TSDataType GetDataTypeFromTemplateType<double>() {

        return common::DOUBLE;

    }

    template <>

    FORCE_INLINE common::TSDataType GetDataTypeFromTemplateType<common::String>() {

        return common::STRING;

    }

    FORCE_INLINE size_t get_data_type_size(TSDataType data_type) {

        switch (data_type) {

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Suppor C++ Tsfile V4 Read. #408

Uh oh!

Diff view

Diff view

There are no files selected for viewing

Uh oh!