From dc6c8615f5f70ddc421292b1e6a22aa6e4de6e33 Mon Sep 17 00:00:00 2001 From: panlei-coder Date: Wed, 15 May 2024 20:35:52 +0800 Subject: [PATCH 1/9] feat: Distinguish between normal node startup and snapshot loading --- cmake/braft.cmake | 4 +- cmake/brpc.cmake | 2 +- cmake/openssl.cmake | 17 ++++--- pikiwidb.conf | 4 +- save_load.sh | 6 +-- src/CMakeLists.txt | 3 +- src/cmd_raft.cc | 4 +- src/db.cc | 5 ++- src/praft/praft.cc | 64 ++++++++++++++++++++++++--- src/praft/praft.h | 4 ++ src/praft/psnapshot.cc | 27 +++++++++-- src/storage/include/storage/storage.h | 2 + src/storage/src/log_index.cc | 3 ++ src/storage/src/redis.h | 1 + src/storage/src/storage.cc | 9 ++++ 15 files changed, 127 insertions(+), 28 deletions(-) diff --git a/cmake/braft.cmake b/cmake/braft.cmake index 288c637fe..ac5d68b94 100644 --- a/cmake/braft.cmake +++ b/cmake/braft.cmake @@ -16,8 +16,8 @@ ExternalProject_Add( extern_braft ${EXTERNAL_PROJECT_LOG_ARGS} DEPENDS brpc - GIT_REPOSITORY "https://github.com/pikiwidb/braft.git" - GIT_TAG master + GIT_REPOSITORY https://github.com/panlei-coder/braft.git + GIT_TAG merge-master-playback PREFIX ${BRAFT_SOURCES_DIR} UPDATE_COMMAND "" CMAKE_ARGS -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER} diff --git a/cmake/brpc.cmake b/cmake/brpc.cmake index fbace60c8..86e97e86e 100644 --- a/cmake/brpc.cmake +++ b/cmake/brpc.cmake @@ -11,7 +11,7 @@ SET(BRPC_INCLUDE_DIR "${BRPC_INSTALL_DIR}/include" CACHE PATH "brpc include dire SET(BRPC_LIBRARIES "${BRPC_INSTALL_DIR}/lib/libbrpc.a" CACHE FILEPATH "brpc library." FORCE) # Reference https://stackoverflow.com/questions/45414507/pass-a-list-of-prefix-paths-to-externalproject-add-in-cmake-args -SET(prefix_path "${CMAKE_CURRENT_BINARY_DIR}/_deps/gflags-build|${THIRD_PARTY_PATH}/install/protobuf|${THIRD_PARTY_PATH}/install/zlib|${CMAKE_CURRENT_BINARY_DIR}/_deps/leveldb-build|${CMAKE_CURRENT_BINARY_DIR}/_deps/leveldb-src") +SET(prefix_path "${CMAKE_CURRENT_BINARY_DIR}/_deps/gflags-build|${THIRD_PARTY_PATH}/install/protobuf|${THIRD_PARTY_PATH}/install/zlib|${CMAKE_CURRENT_BINARY_DIR}/_deps/leveldb-build|${CMAKE_CURRENT_BINARY_DIR}/_deps/leveldb-src|${THIRD_PARTY_PATH}/install/openssl") SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-deprecated-declarations") # If minimal .a is need, you can set WITH_DEBUG_SYMBOLS=OFF EXTERNALPROJECT_ADD( diff --git a/cmake/openssl.cmake b/cmake/openssl.cmake index cc69f4f66..1e9d7d876 100644 --- a/cmake/openssl.cmake +++ b/cmake/openssl.cmake @@ -4,17 +4,15 @@ # of patent rights can be found in the PATENTS file in the same directory. INCLUDE(ExternalProject) - + SET(OPENSSL_SOURCE_DIR ${THIRD_PARTY_PATH}/openssl) SET(OPENSSL_INSTALL_DIR ${THIRD_PARTY_PATH}/install/openssl) SET(OPENSSL_INCLUDE_DIR ${OPENSSL_INSTALL_DIR}/include) SET(OPENSSL_CONFIGURE_COMMAND ${OPENSSL_SOURCE_DIR}/config) IF (CMAKE_SYSTEM_NAME MATCHES "Darwin") - SET(OPENSSL_LIBRARY_SUFFIX "dylib") - SET(OPENSSL_LIB "lib") + SET(OPENSSL_LIB "lib") ELSEIF (CMAKE_SYSTEM_NAME MATCHES "Linux") - SET(OPENSSL_LIBRARY_SUFFIX "so") SET(OPENSSL_LIB "lib64") ELSE () MESSAGE(FATAL_ERROR "only support linux or macOS") @@ -39,11 +37,16 @@ ExternalProject_Add( ) ADD_LIBRARY(ssl STATIC IMPORTED GLOBAL) -SET_PROPERTY(TARGET ssl PROPERTY IMPORTED_LOCATION ${OPENSSL_INSTALL_DIR}/${OPENSSL_LIB}/libssl.${OPENSSL_LIBRARY_SUFFIX}) +SET_PROPERTY(TARGET ssl PROPERTY IMPORTED_LOCATION ${OPENSSL_INSTALL_DIR}/${OPENSSL_LIB}/libssl.a) SET_PROPERTY(TARGET ssl PROPERTY INTERFACE_INCLUDE_DIRECTORIES ${OPENSSL_INCLUDE_DIR}) ADD_DEPENDENCIES(ssl OpenSSL) +SET(OPENSSL_SSL_LIBRARY ${OPENSSL_INSTALL_DIR}/${OPENSSL_LIB}/libssl.a) ADD_LIBRARY(crypto STATIC IMPORTED GLOBAL) -SET_PROPERTY(TARGET crypto PROPERTY IMPORTED_LOCATION ${OPENSSL_INSTALL_DIR}/${OPENSSL_LIB}/libcrypto.${OPENSSL_LIBRARY_SUFFIX}) +SET_PROPERTY(TARGET crypto PROPERTY IMPORTED_LOCATION ${OPENSSL_INSTALL_DIR}/${OPENSSL_LIB}/libcrypto.a) SET_PROPERTY(TARGET crypto PROPERTY INTERFACE_INCLUDE_DIRECTORIES ${OPENSSL_INCLUDE_DIR}) -ADD_DEPENDENCIES(crypto OpenSSL) \ No newline at end of file +ADD_DEPENDENCIES(crypto OpenSSL) +SET(OPENSSL_CRYPTO_LIBRARY ${OPENSSL_INSTALL_DIR}/${OPENSSL_LIB}/libcrypto.a) + +SET(OPENSSL_INCLUDE_DIR ${THIRD_PARTY_PATH}/install/openssl/include) +SET(OPENSSL_ROOT_DIR ${THIRD_PARTY_PATH}/install/openssl/) diff --git a/pikiwidb.conf b/pikiwidb.conf index 3f23870e6..2423707c8 100644 --- a/pikiwidb.conf +++ b/pikiwidb.conf @@ -319,7 +319,7 @@ slowlog-max-len 128 # to the same DB is distributed among several RocksDB instances. # RocksDB instances number per DB -db-instance-num 3 +db-instance-num 1 # default is 86400 * 7 small-compaction-threshold 604800 # default is 86400 * 3 @@ -343,6 +343,6 @@ rocksdb-ttl-second 604800 rocksdb-periodic-second 259200; ############################### RAFT ############################### -use-raft no +use-raft yes # Braft relies on brpc to communicate via the default port number plus the port offset raft-port-offset 10 diff --git a/save_load.sh b/save_load.sh index 1bcad5b74..09f36c260 100755 --- a/save_load.sh +++ b/save_load.sh @@ -1,4 +1,5 @@ #!/bin/bash + killall -9 pikiwidb mkdir leader follower1 @@ -10,9 +11,8 @@ redis-cli -p 7777 raft.cluster init redis-benchmark -p 7777 -c 5 -n 10000 -r 10000 -d 1024 -t hset redis-cli -p 7777 raft.node dosnapshot +redis-benchmark -p 7777 -c 5 -n 10000 -r 10000 -d 1024 -t hset redis-cli -p 7777 raft.node dosnapshot - -sleep 10 - +redis-benchmark -p 7777 -c 5 -n 10000 -r 10000 -d 1024 -t hset redis-cli -p 8888 raft.cluster join 127.0.0.1:7777 diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 14b927e51..b48f29f18 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -21,7 +21,6 @@ TARGET_INCLUDE_DIRECTORIES(pikiwidb PRIVATE ${BRPC_INCLUDE_DIR} ) - -TARGET_LINK_LIBRARIES(pikiwidb net; dl; fmt; storage; rocksdb; pstd braft brpc ssl crypto zlib protobuf leveldb gflags z praft praft_pb "${LIB}") +TARGET_LINK_LIBRARIES(pikiwidb net; dl; fmt; storage; rocksdb; pstd braft brpc ssl crypto zlib protobuf leveldb gflags z praft praft_pb "${LIB}") SET_TARGET_PROPERTIES(pikiwidb PROPERTIES LINKER_LANGUAGE CXX) \ No newline at end of file diff --git a/src/cmd_raft.cc b/src/cmd_raft.cc index 9bfaabc19..b01664a6f 100644 --- a/src/cmd_raft.cc +++ b/src/cmd_raft.cc @@ -122,7 +122,9 @@ void RaftNodeCmd::DoCmdRemove(PClient* client) { } void RaftNodeCmd::DoCmdSnapshot(PClient* client) { - auto s = PRAFT.DoSnapshot(); + // @todo need to get self_snapshot_index + // auto self_snapshot_index = PSTORE.GetBackend(client->GetCurrentDB())->GetStorage()->GetSmallestFlushedLogIndex(); + auto s = PRAFT.DoSnapshot(); // self_snapshot_index if (s.ok()) { client->SetRes(CmdRes::kOK); } diff --git a/src/db.cc b/src/db.cc index 5f1cbbbca..306b02348 100644 --- a/src/db.cc +++ b/src/db.cc @@ -34,8 +34,9 @@ rocksdb::Status DB::Open() { storage_options.append_log_function = [&r = PRAFT](const Binlog& log, std::promise&& promise) { r.AppendLog(log, std::move(promise)); }; - storage_options.do_snapshot_function = - std::bind(&pikiwidb::PRaft::DoSnapshot, &pikiwidb::PRAFT, std::placeholders::_1, std::placeholders::_2); + storage_options.do_snapshot_function = [raft = &pikiwidb::PRAFT](auto&& self_snapshot_index, auto&& is_sync) { + raft->DoSnapshot(std::forward(self_snapshot_index), std::forward(is_sync)); + }; } storage_options.db_instance_num = g_config.db_instance_num.load(); diff --git a/src/praft/praft.cc b/src/praft/praft.cc index 26239c743..e55912e97 100644 --- a/src/praft/praft.cc +++ b/src/praft/praft.cc @@ -9,6 +9,7 @@ #include +#include "braft/raft.h" #include "braft/snapshot.h" #include "braft/util.h" #include "brpc/server.h" @@ -150,7 +151,7 @@ butil::Status PRaft::Init(std::string& group_id, bool initial_conf_is_null) { node_options_.fsm = this; node_options_.node_owns_fsm = false; node_options_.snapshot_interval_s = 0; - std::string prefix = "local://" + g_config.db_path.ToString() + "_praft"; + std::string prefix = "local://" + g_config.db_path.ToString() + std::to_string(db_id_) + "/_praft"; node_options_.log_uri = prefix + "/log"; node_options_.raft_meta_uri = prefix + "/raft_meta"; node_options_.snapshot_uri = prefix + "/snapshot"; @@ -241,6 +242,24 @@ butil::Status PRaft::GetListPeers(std::vector* peers) { return node_->list_peers(peers); } +uint64_t PRaft::GetTerm(uint64_t log_index) { + if (!node_) { + ERROR("Node is not initialized"); + return 0; + } else { + return node_->get_term(log_index); + } +} + +uint64_t PRaft::GetLastLogIndex(bool is_flush) { + if (!node_) { + ERROR("Node is not initialized"); + return 0; + } else { + return node_->get_last_log_index(is_flush); + } +} + void PRaft::SendNodeRequest(PClient* client) { assert(client); @@ -521,10 +540,17 @@ butil::Status PRaft::DoSnapshot(int64_t self_snapshot_index, bool is_sync) { if (!node_) { return ERROR_LOG_AND_STATUS("Node is not initialized"); } - braft::SynchronizedClosure done; - node_->snapshot(&done, self_snapshot_index); - done.wait(); - return done.status(); + + if (is_sync) { + braft::SynchronizedClosure done; + node_->snapshot(&done, self_snapshot_index); + done.wait(); + return done.status(); + } else { + node_->snapshot(nullptr, self_snapshot_index); + butil::Status status; + return status; + } } void PRaft::OnClusterCmdConnectionFailed([[maybe_unused]] EventLoop* loop, const char* peer_ip, int port) { @@ -629,10 +655,38 @@ void PRaft::on_snapshot_save(braft::SnapshotWriter* writer, braft::Closure* done int PRaft::on_snapshot_load(braft::SnapshotReader* reader) { CHECK(!IsLeader()) << "Leader is not supposed to load snapshot"; assert(reader); + + if (is_node_first_start_up_) { + // get replay point of one db's + /* + 1. When a node starts normally, because all memory data is flushed to disks and + snapshots are truncated to the latest, the flush-index and apply-index are the + same when the node starts, so the maximum log index should be obtained. + 2. When a node is improperly shut down and restarted, the minimum flush-index should + be obtained as the starting point for fault recovery. + */ + // @todo GetSmallestFlushedLogIndex + uint64_t replay_point = PSTORE.GetBackend(db_id_)->GetStorage()->GetSmallestFlushedLogIndex(); + node_->set_self_playback_point(replay_point); + is_node_first_start_up_ = false; + INFO("set replay_point: {}", replay_point); + + /* + If a node has just joined the cluster and does not have any data, + it does not load the local snapshot at startup. Therefore, + LoadDBFromCheckPoint is required after loading the snapshot from the leader. + */ + if (GetLastLogIndex() != 0) { + return 0; + } + } + + // 3. When a snapshot is installed on a node, you do not need to set a playback point. auto reader_path = reader->get_path(); // xx/snapshot_0000001 auto path = g_config.db_path.ToString() + std::to_string(db_id_); // db/db_id TasksVector tasks(1, {TaskType::kLoadDBFromCheckpoint, db_id_, {{TaskArg::kCheckpointPath, reader_path}}, true}); PSTORE.HandleTaskSpecificDB(tasks); + INFO("load snapshot success!"); return 0; } diff --git a/src/praft/praft.h b/src/praft/praft.h index 65cc14d4f..b85de14a5 100644 --- a/src/praft/praft.h +++ b/src/praft/praft.h @@ -137,6 +137,8 @@ class PRaft : public braft::StateMachine { std::string GetGroupID() const; braft::NodeStatus GetNodeStatus() const; butil::Status GetListPeers(std::vector* peers); + uint64_t GetTerm(uint64_t log_index); + uint64_t GetLastLogIndex(bool is_flush = false); bool IsInitialized() const { return node_ != nullptr && server_ != nullptr; } @@ -164,6 +166,8 @@ class PRaft : public braft::StateMachine { ClusterCmdContext cluster_cmd_ctx_; // context for cluster join/remove command std::string group_id_; // group id int db_id_ = 0; // db_id + + bool is_node_first_start_up_ = true; }; } // namespace pikiwidb diff --git a/src/praft/psnapshot.cc b/src/praft/psnapshot.cc index 4cfc36d55..359cc6e76 100644 --- a/src/praft/psnapshot.cc +++ b/src/praft/psnapshot.cc @@ -11,33 +11,44 @@ #include "psnapshot.h" #include "braft/local_file_meta.pb.h" +#include "braft/snapshot.h" #include "butil/files/file_path.h" #include "pstd/log.h" +#include "pstd/pstd_string.h" #include "config.h" +#include "praft.h" #include "store.h" namespace pikiwidb { -extern PConfig g_config; - braft::FileAdaptor* PPosixFileSystemAdaptor::open(const std::string& path, int oflag, const ::google::protobuf::Message* file_meta, butil::File::Error* e) { if ((oflag & IS_RDONLY) == 0) { // This is a read operation bool snapshots_exists = false; std::string snapshot_path; + int db_id = -1; // parse snapshot path butil::FilePath parse_snapshot_path(path); std::vector components; + bool is_find_db = false; parse_snapshot_path.GetComponents(&components); - for (auto component : components) { + for (const auto& component : components) { snapshot_path += component + "/"; + + if (is_find_db && pstd::String2int(component, &db_id)) { + is_find_db = false; + } + if (component.find("snapshot_") != std::string::npos) { break; + } else if (component == "db") { + is_find_db = true; } } + // check whether snapshots have been created std::lock_guard guard(mutex_); if (!snapshot_path.empty()) { @@ -55,6 +66,8 @@ braft::FileAdaptor* PPosixFileSystemAdaptor::open(const std::string& path, int o // Snapshot generation if (!snapshots_exists) { + assert(db_id >= 0); + braft::LocalSnapshotMetaTable snapshot_meta_memtable; std::string meta_path = snapshot_path + "/" PRAFT_SNAPSHOT_META_FILE; INFO("start to generate snapshot in path {}", snapshot_path); @@ -66,6 +79,14 @@ braft::FileAdaptor* PPosixFileSystemAdaptor::open(const std::string& path, int o PSTORE.HandleTaskSpecificDB(tasks); AddAllFiles(snapshot_path, &snapshot_meta_memtable, snapshot_path); + // update snapshot last log index and last_log_term + auto& new_meta = const_cast(snapshot_meta_memtable.meta()); + auto last_log_index = 30000; // @todo PSTORE.GetBackend(db_id)->GetStorage()->GetSmallestFlushedLogIndex(); + new_meta.set_last_included_index(last_log_index); + auto last_log_term = PRAFT.GetTerm(last_log_index); + new_meta.set_last_included_term(last_log_term); + INFO("Succeed to fix snapshot meta: {}, {}", last_log_index, last_log_term); + auto rc = snapshot_meta_memtable.save_to_file(fs, meta_path); if (rc == 0) { INFO("Succeed to save snapshot in path {}", snapshot_path); diff --git a/src/storage/include/storage/storage.h b/src/storage/include/storage/storage.h index 4536a7e41..368e3cfe2 100644 --- a/src/storage/include/storage/storage.h +++ b/src/storage/include/storage/storage.h @@ -1110,6 +1110,8 @@ class Storage { void GetRocksDBInfo(std::string& info); Status OnBinlogWrite(const pikiwidb::Binlog& log, LogIndex log_idx); + LogIndex GetSmallestFlushedLogIndex() const; + private: std::vector> insts_; std::unique_ptr slot_indexer_; diff --git a/src/storage/src/log_index.cc b/src/storage/src/log_index.cc index 1dede3013..f1d66300f 100644 --- a/src/storage/src/log_index.cc +++ b/src/storage/src/log_index.cc @@ -7,6 +7,8 @@ #include "log_index.h" +#include "pstd/log.h" + #include #include #include @@ -167,6 +169,7 @@ void LogIndexAndSequenceCollectorPurger::OnFlushCompleted(rocksdb::DB *db, auto count = count_.fetch_add(1); if (count % 10 == 0) { + INFO("do snapshot after flush: {}", smallest_flushed_log_index); callback_(smallest_flushed_log_index, false); } diff --git a/src/storage/src/redis.h b/src/storage/src/redis.h index b60878c29..4163700ce 100644 --- a/src/storage/src/redis.h +++ b/src/storage/src/redis.h @@ -108,6 +108,7 @@ class Redis { void UpdateAppliedLogIndexOfColumnFamily(size_t cf_idx, LogIndex logidx, SequenceNumber seqno) { log_index_of_all_cfs_.Update(cf_idx, logidx, seqno); } + LogIndex GetSmallestFlushedLogIndex() const { return log_index_of_all_cfs_.GetSmallestLogIndex(-1).smallest_flushed_log_index; } bool IsRestarting() const { return is_starting_; } void StartingPhaseEnd() { is_starting_ = false; } diff --git a/src/storage/src/storage.cc b/src/storage/src/storage.cc index 342f7aeb7..aceb76135 100644 --- a/src/storage/src/storage.cc +++ b/src/storage/src/storage.cc @@ -2408,4 +2408,13 @@ Status Storage::OnBinlogWrite(const pikiwidb::Binlog& log, LogIndex log_idx) { return s; } +LogIndex Storage::GetSmallestFlushedLogIndex() const { + LogIndex smallest_flushed_log_index = INT64_MAX; + for (auto& inst : insts_) { + smallest_flushed_log_index = std::min(smallest_flushed_log_index, inst->GetSmallestFlushedLogIndex()); + } + + return smallest_flushed_log_index; +} + } // namespace storage From 610c28c4e2c6eae59769cc208f72cb7b0410bf15 Mon Sep 17 00:00:00 2001 From: panlei-coder Date: Wed, 15 May 2024 20:44:00 +0800 Subject: [PATCH 2/9] fix: code format --- src/db.cc | 3 ++- src/praft/praft.cc | 14 +++++++------- src/praft/praft.h | 2 +- src/praft/psnapshot.cc | 6 +++--- src/storage/src/redis.h | 4 +++- src/storage/src/storage.cc | 2 +- 6 files changed, 17 insertions(+), 14 deletions(-) diff --git a/src/db.cc b/src/db.cc index 306b02348..44b5430b5 100644 --- a/src/db.cc +++ b/src/db.cc @@ -35,7 +35,8 @@ rocksdb::Status DB::Open() { r.AppendLog(log, std::move(promise)); }; storage_options.do_snapshot_function = [raft = &pikiwidb::PRAFT](auto&& self_snapshot_index, auto&& is_sync) { - raft->DoSnapshot(std::forward(self_snapshot_index), std::forward(is_sync)); + raft->DoSnapshot(std::forward(self_snapshot_index), + std::forward(is_sync)); }; } diff --git a/src/praft/praft.cc b/src/praft/praft.cc index e55912e97..78a12b373 100644 --- a/src/praft/praft.cc +++ b/src/praft/praft.cc @@ -540,7 +540,7 @@ butil::Status PRaft::DoSnapshot(int64_t self_snapshot_index, bool is_sync) { if (!node_) { return ERROR_LOG_AND_STATUS("Node is not initialized"); } - + if (is_sync) { braft::SynchronizedClosure done; node_->snapshot(&done, self_snapshot_index); @@ -659,10 +659,10 @@ int PRaft::on_snapshot_load(braft::SnapshotReader* reader) { if (is_node_first_start_up_) { // get replay point of one db's /* - 1. When a node starts normally, because all memory data is flushed to disks and - snapshots are truncated to the latest, the flush-index and apply-index are the + 1. When a node starts normally, because all memory data is flushed to disks and + snapshots are truncated to the latest, the flush-index and apply-index are the same when the node starts, so the maximum log index should be obtained. - 2. When a node is improperly shut down and restarted, the minimum flush-index should + 2. When a node is improperly shut down and restarted, the minimum flush-index should be obtained as the starting point for fault recovery. */ // @todo GetSmallestFlushedLogIndex @@ -672,13 +672,13 @@ int PRaft::on_snapshot_load(braft::SnapshotReader* reader) { INFO("set replay_point: {}", replay_point); /* - If a node has just joined the cluster and does not have any data, - it does not load the local snapshot at startup. Therefore, + If a node has just joined the cluster and does not have any data, + it does not load the local snapshot at startup. Therefore, LoadDBFromCheckPoint is required after loading the snapshot from the leader. */ if (GetLastLogIndex() != 0) { return 0; - } + } } // 3. When a snapshot is installed on a node, you do not need to set a playback point. diff --git a/src/praft/praft.h b/src/praft/praft.h index b85de14a5..c86498578 100644 --- a/src/praft/praft.h +++ b/src/praft/praft.h @@ -166,7 +166,7 @@ class PRaft : public braft::StateMachine { ClusterCmdContext cluster_cmd_ctx_; // context for cluster join/remove command std::string group_id_; // group id int db_id_ = 0; // db_id - + bool is_node_first_start_up_ = true; }; diff --git a/src/praft/psnapshot.cc b/src/praft/psnapshot.cc index 359cc6e76..3855c62e8 100644 --- a/src/praft/psnapshot.cc +++ b/src/praft/psnapshot.cc @@ -37,7 +37,7 @@ braft::FileAdaptor* PPosixFileSystemAdaptor::open(const std::string& path, int o parse_snapshot_path.GetComponents(&components); for (const auto& component : components) { snapshot_path += component + "/"; - + if (is_find_db && pstd::String2int(component, &db_id)) { is_find_db = false; } @@ -48,7 +48,7 @@ braft::FileAdaptor* PPosixFileSystemAdaptor::open(const std::string& path, int o is_find_db = true; } } - + // check whether snapshots have been created std::lock_guard guard(mutex_); if (!snapshot_path.empty()) { @@ -81,7 +81,7 @@ braft::FileAdaptor* PPosixFileSystemAdaptor::open(const std::string& path, int o // update snapshot last log index and last_log_term auto& new_meta = const_cast(snapshot_meta_memtable.meta()); - auto last_log_index = 30000; // @todo PSTORE.GetBackend(db_id)->GetStorage()->GetSmallestFlushedLogIndex(); + auto last_log_index = 30000; // @todo PSTORE.GetBackend(db_id)->GetStorage()->GetSmallestFlushedLogIndex(); new_meta.set_last_included_index(last_log_index); auto last_log_term = PRAFT.GetTerm(last_log_index); new_meta.set_last_included_term(last_log_term); diff --git a/src/storage/src/redis.h b/src/storage/src/redis.h index 4163700ce..34238a4ab 100644 --- a/src/storage/src/redis.h +++ b/src/storage/src/redis.h @@ -108,7 +108,9 @@ class Redis { void UpdateAppliedLogIndexOfColumnFamily(size_t cf_idx, LogIndex logidx, SequenceNumber seqno) { log_index_of_all_cfs_.Update(cf_idx, logidx, seqno); } - LogIndex GetSmallestFlushedLogIndex() const { return log_index_of_all_cfs_.GetSmallestLogIndex(-1).smallest_flushed_log_index; } + LogIndex GetSmallestFlushedLogIndex() const { + return log_index_of_all_cfs_.GetSmallestLogIndex(-1).smallest_flushed_log_index; + } bool IsRestarting() const { return is_starting_; } void StartingPhaseEnd() { is_starting_ = false; } diff --git a/src/storage/src/storage.cc b/src/storage/src/storage.cc index aceb76135..913eaf90b 100644 --- a/src/storage/src/storage.cc +++ b/src/storage/src/storage.cc @@ -2409,7 +2409,7 @@ Status Storage::OnBinlogWrite(const pikiwidb::Binlog& log, LogIndex log_idx) { } LogIndex Storage::GetSmallestFlushedLogIndex() const { - LogIndex smallest_flushed_log_index = INT64_MAX; + LogIndex smallest_flushed_log_index = INT64_MAX; for (auto& inst : insts_) { smallest_flushed_log_index = std::min(smallest_flushed_log_index, inst->GetSmallestFlushedLogIndex()); } From 0e452d3ddd1e0990359d44543486eb4600d597aa Mon Sep 17 00:00:00 2001 From: panlei-coder Date: Wed, 15 May 2024 21:09:05 +0800 Subject: [PATCH 3/9] fix: code format of cmd_raft.cc --- src/cmd_raft.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cmd_raft.cc b/src/cmd_raft.cc index b01664a6f..ddc717e97 100644 --- a/src/cmd_raft.cc +++ b/src/cmd_raft.cc @@ -124,7 +124,7 @@ void RaftNodeCmd::DoCmdRemove(PClient* client) { void RaftNodeCmd::DoCmdSnapshot(PClient* client) { // @todo need to get self_snapshot_index // auto self_snapshot_index = PSTORE.GetBackend(client->GetCurrentDB())->GetStorage()->GetSmallestFlushedLogIndex(); - auto s = PRAFT.DoSnapshot(); // self_snapshot_index + auto s = PRAFT.DoSnapshot(); // self_snapshot_index if (s.ok()) { client->SetRes(CmdRes::kOK); } From e38513c03d99f0ff7f6f6906f32baefd675eddce Mon Sep 17 00:00:00 2001 From: panlei-coder Date: Mon, 3 Jun 2024 18:35:22 +0800 Subject: [PATCH 4/9] fix: resolve go-test fail --- pikiwidb.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pikiwidb.conf b/pikiwidb.conf index 2423707c8..38d7371cc 100644 --- a/pikiwidb.conf +++ b/pikiwidb.conf @@ -343,6 +343,6 @@ rocksdb-ttl-second 604800 rocksdb-periodic-second 259200; ############################### RAFT ############################### -use-raft yes +use-raft no # Braft relies on brpc to communicate via the default port number plus the port offset raft-port-offset 10 From 3f8ae4114b20c2245ada8f0c526101fb26803c90 Mon Sep 17 00:00:00 2001 From: panlei-coder Date: Tue, 4 Jun 2024 16:50:31 +0800 Subject: [PATCH 5/9] fix: fix comment --- src/praft/praft.cc | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/praft/praft.cc b/src/praft/praft.cc index 78a12b373..8e1fbcbc0 100644 --- a/src/praft/praft.cc +++ b/src/praft/praft.cc @@ -246,18 +246,18 @@ uint64_t PRaft::GetTerm(uint64_t log_index) { if (!node_) { ERROR("Node is not initialized"); return 0; - } else { - return node_->get_term(log_index); } + + return node_->get_term(log_index); } uint64_t PRaft::GetLastLogIndex(bool is_flush) { if (!node_) { ERROR("Node is not initialized"); return 0; - } else { - return node_->get_last_log_index(is_flush); } + + return node_->get_last_log_index(is_flush); } void PRaft::SendNodeRequest(PClient* client) { From 6f9f6fb31a9821adabf979bc6609e7bec2de4d87 Mon Sep 17 00:00:00 2001 From: panlei-coder Date: Tue, 4 Jun 2024 16:58:13 +0800 Subject: [PATCH 6/9] fix: code format --- src/praft/praft.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/praft/praft.cc b/src/praft/praft.cc index 8e1fbcbc0..d54031dfb 100644 --- a/src/praft/praft.cc +++ b/src/praft/praft.cc @@ -256,7 +256,7 @@ uint64_t PRaft::GetLastLogIndex(bool is_flush) { ERROR("Node is not initialized"); return 0; } - + return node_->get_last_log_index(is_flush); } From e46d2138a43ebe406ef5a1b553a3b4cb2b154b66 Mon Sep 17 00:00:00 2001 From: panlei-coder Date: Mon, 15 Jul 2024 17:53:44 +0800 Subject: [PATCH 7/9] fix: fix comment --- CMakeLists.txt | 2 +- cmake/braft.cmake | 6 ++---- pikiwidb.conf | 2 +- src/cmd_raft.cc | 6 +++--- src/praft/CMakeLists.txt | 1 + src/praft/praft.cc | 10 ++++------ src/praft/praft.h | 5 +++-- src/praft/psnapshot.cc | 4 ++-- 8 files changed, 17 insertions(+), 19 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 2e05ce16c..04c1a3ecd 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -33,7 +33,7 @@ IF(CMAKE_BUILD_TYPE STREQUAL "Release") MESSAGE("Building in Release mode") ELSE() MESSAGE("Building in Debug mode") - OPTION(ADDRESS_SANITIZER "Enable AddressSanitizer (default sanitizer)" ON) + OPTION(ADDRESS_SANITIZER "Enable AddressSanitizer (default sanitizer)" OFF) OPTION(THREAD_SANITIZER "Enable ThreadSanitizer" OFF) IF(THREAD_SANITIZER) diff --git a/cmake/braft.cmake b/cmake/braft.cmake index 286663c4b..3cd92d033 100644 --- a/cmake/braft.cmake +++ b/cmake/braft.cmake @@ -17,10 +17,8 @@ ExternalProject_Add( ${EXTERNAL_PROJECT_LOG_ARGS} DEPENDS brpc # The pr on braft is not merged, so I am using my own warehouse to run the test for the time being - GIT_REPOSITORY https://github.com/panlei-coder/braft.git - GIT_TAG merge-master-playback - # GIT_REPOSITORY "https://github.com/pikiwidb/braft.git" - # GIT_TAG master + GIT_REPOSITORY "https://github.com/pikiwidb/braft.git" + GIT_TAG v1.1.2-alpha1 GIT_SHALLOW true PREFIX ${BRAFT_SOURCES_DIR} UPDATE_COMMAND "" diff --git a/pikiwidb.conf b/pikiwidb.conf index 38d7371cc..2423707c8 100644 --- a/pikiwidb.conf +++ b/pikiwidb.conf @@ -343,6 +343,6 @@ rocksdb-ttl-second 604800 rocksdb-periodic-second 259200; ############################### RAFT ############################### -use-raft no +use-raft yes # Braft relies on brpc to communicate via the default port number plus the port offset raft-port-offset 10 diff --git a/src/cmd_raft.cc b/src/cmd_raft.cc index ddc717e97..b4bc29b51 100644 --- a/src/cmd_raft.cc +++ b/src/cmd_raft.cc @@ -122,9 +122,9 @@ void RaftNodeCmd::DoCmdRemove(PClient* client) { } void RaftNodeCmd::DoCmdSnapshot(PClient* client) { - // @todo need to get self_snapshot_index - // auto self_snapshot_index = PSTORE.GetBackend(client->GetCurrentDB())->GetStorage()->GetSmallestFlushedLogIndex(); - auto s = PRAFT.DoSnapshot(); // self_snapshot_index + auto self_snapshot_index = PSTORE.GetBackend(client->GetCurrentDB())->GetStorage()->GetSmallestFlushedLogIndex(); + INFO("DoCmdSnapshot self_snapshot_index:{}", self_snapshot_index); + auto s = PRAFT.DoSnapshot(self_snapshot_index); if (s.ok()) { client->SetRes(CmdRes::kOK); } diff --git a/src/praft/CMakeLists.txt b/src/praft/CMakeLists.txt index 45cf62f8c..8ed004dae 100644 --- a/src/praft/CMakeLists.txt +++ b/src/praft/CMakeLists.txt @@ -35,6 +35,7 @@ TARGET_INCLUDE_DIRECTORIES(praft PRIVATE ${BRAFT_INCLUDE_DIR} PRIVATE ${BRPC_INCLUDE_DIR} PRIVATE ${PROTO_OUTPUT_DIR} + PRIVATE ${PROJECT_SOURCE_DIR}/src/storage/include ) IF(CMAKE_SYSTEM_NAME STREQUAL "Linux") diff --git a/src/praft/praft.cc b/src/praft/praft.cc index d54031dfb..ed729b8a4 100644 --- a/src/praft/praft.cc +++ b/src/praft/praft.cc @@ -111,7 +111,7 @@ butil::Status PRaft::Init(std::string& group_id, bool initial_conf_is_null) { // the case that it becomes the leader while the service is unreacheable by // clients. // Notice the default options of server is used here. Check out details from - // the doc of brpc if you would like change some options; + // the doc of brpc if you would like change some option; if (server_->Start(port, nullptr) != 0) { server_.reset(); return ERROR_LOG_AND_STATUS("Failed to start server"); @@ -242,7 +242,7 @@ butil::Status PRaft::GetListPeers(std::vector* peers) { return node_->list_peers(peers); } -uint64_t PRaft::GetTerm(uint64_t log_index) { +storage::LogIndex PRaft::GetTerm(uint64_t log_index) { if (!node_) { ERROR("Node is not initialized"); return 0; @@ -251,7 +251,7 @@ uint64_t PRaft::GetTerm(uint64_t log_index) { return node_->get_term(log_index); } -uint64_t PRaft::GetLastLogIndex(bool is_flush) { +storage::LogIndex PRaft::GetLastLogIndex(bool is_flush) { if (!node_) { ERROR("Node is not initialized"); return 0; @@ -548,8 +548,7 @@ butil::Status PRaft::DoSnapshot(int64_t self_snapshot_index, bool is_sync) { return done.status(); } else { node_->snapshot(nullptr, self_snapshot_index); - butil::Status status; - return status; + return butil::Status{}; } } @@ -665,7 +664,6 @@ int PRaft::on_snapshot_load(braft::SnapshotReader* reader) { 2. When a node is improperly shut down and restarted, the minimum flush-index should be obtained as the starting point for fault recovery. */ - // @todo GetSmallestFlushedLogIndex uint64_t replay_point = PSTORE.GetBackend(db_id_)->GetStorage()->GetSmallestFlushedLogIndex(); node_->set_self_playback_point(replay_point); is_node_first_start_up_ = false; diff --git a/src/praft/praft.h b/src/praft/praft.h index c86498578..60c9e475b 100644 --- a/src/praft/praft.h +++ b/src/praft/praft.h @@ -18,6 +18,7 @@ #include "braft/raft.h" #include "brpc/server.h" #include "rocksdb/status.h" +#include "storage/storage.h" #include "client.h" @@ -137,8 +138,8 @@ class PRaft : public braft::StateMachine { std::string GetGroupID() const; braft::NodeStatus GetNodeStatus() const; butil::Status GetListPeers(std::vector* peers); - uint64_t GetTerm(uint64_t log_index); - uint64_t GetLastLogIndex(bool is_flush = false); + storage::LogIndex GetTerm(uint64_t log_index); + storage::LogIndex GetLastLogIndex(bool is_flush = false); bool IsInitialized() const { return node_ != nullptr && server_ != nullptr; } diff --git a/src/praft/psnapshot.cc b/src/praft/psnapshot.cc index 3855c62e8..28fc99c06 100644 --- a/src/praft/psnapshot.cc +++ b/src/praft/psnapshot.cc @@ -81,11 +81,11 @@ braft::FileAdaptor* PPosixFileSystemAdaptor::open(const std::string& path, int o // update snapshot last log index and last_log_term auto& new_meta = const_cast(snapshot_meta_memtable.meta()); - auto last_log_index = 30000; // @todo PSTORE.GetBackend(db_id)->GetStorage()->GetSmallestFlushedLogIndex(); + auto last_log_index = PSTORE.GetBackend(db_id)->GetStorage()->GetSmallestFlushedLogIndex(); new_meta.set_last_included_index(last_log_index); auto last_log_term = PRAFT.GetTerm(last_log_index); new_meta.set_last_included_term(last_log_term); - INFO("Succeed to fix snapshot meta: {}, {}", last_log_index, last_log_term); + INFO("Succeed to fix db_{} snapshot meta: {}, {}", db_id, last_log_index, last_log_term); auto rc = snapshot_meta_memtable.save_to_file(fs, meta_path); if (rc == 0) { From 929a5dbccc169030b4fe6f3a3283dc38b8b26af9 Mon Sep 17 00:00:00 2001 From: panlei-coder Date: Tue, 16 Jul 2024 22:44:29 +0800 Subject: [PATCH 8/9] fix: Fixed the bug that obtaining the smallest flushed index --- cmake/braft.cmake | 2 +- pikiwidb.conf | 2 +- src/db.cc | 8 +- src/storage/include/storage/storage.h | 1151 ------------------------- src/storage/src/log_index.h | 4 +- src/storage/src/redis.h | 4 +- src/storage/src/storage.cc | 1 - 7 files changed, 12 insertions(+), 1160 deletions(-) delete mode 100644 src/storage/include/storage/storage.h diff --git a/cmake/braft.cmake b/cmake/braft.cmake index 3cd92d033..98a3a3f58 100644 --- a/cmake/braft.cmake +++ b/cmake/braft.cmake @@ -18,7 +18,7 @@ ExternalProject_Add( DEPENDS brpc # The pr on braft is not merged, so I am using my own warehouse to run the test for the time being GIT_REPOSITORY "https://github.com/pikiwidb/braft.git" - GIT_TAG v1.1.2-alpha1 + GIT_TAG v1.1.2-alpha2 GIT_SHALLOW true PREFIX ${BRAFT_SOURCES_DIR} UPDATE_COMMAND "" diff --git a/pikiwidb.conf b/pikiwidb.conf index 2423707c8..38d7371cc 100644 --- a/pikiwidb.conf +++ b/pikiwidb.conf @@ -343,6 +343,6 @@ rocksdb-ttl-second 604800 rocksdb-periodic-second 259200; ############################### RAFT ############################### -use-raft yes +use-raft no # Braft relies on brpc to communicate via the default port number plus the port offset raft-port-offset 10 diff --git a/src/db.cc b/src/db.cc index a3c4717d6..ab353695a 100644 --- a/src/db.cc +++ b/src/db.cc @@ -94,6 +94,13 @@ void DB::LoadDBFromCheckpoint(const std::string& checkpoint_path, bool sync [[ma std::lock_guard lock(storage_mutex_); opened_ = false; + // close the old storage, then open the new storage + std::unique_ptr old_storage = std::move(storage_); + if (old_storage != nullptr) { + old_storage->Close(); + old_storage.reset(); + } + storage_ = std::make_unique(); auto result = storage_->LoadCheckpoint(checkpoint_sub_path, db_path_); for (auto& r : result) { @@ -116,7 +123,6 @@ void DB::LoadDBFromCheckpoint(const std::string& checkpoint_path, bool sync [[ma storage_options.do_snapshot_function = std::bind(&pikiwidb::PRaft::DoSnapshot, &pikiwidb::PRAFT, std::placeholders::_1, std::placeholders::_2); } - storage_ = std::make_unique(); if (auto s = storage_->Open(storage_options, db_path_); !s.ok()) { ERROR("Storage open failed! {}", s.ToString()); diff --git a/src/storage/include/storage/storage.h b/src/storage/include/storage/storage.h deleted file mode 100644 index 8c50f2aba..000000000 --- a/src/storage/include/storage/storage.h +++ /dev/null @@ -1,1151 +0,0 @@ -// Copyright (c) 2017-present, Qihoo, Inc. All rights reserved. -// This source code is licensed under the BSD-style license found in the -// LICENSE file in the root directory of this source tree. An additional grant -// of patent rights can be found in the PATENTS file in the same directory. - -#ifndef INCLUDE_STORAGE_STORAGE_H_ -#define INCLUDE_STORAGE_STORAGE_H_ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "rocksdb/convenience.h" -#include "rocksdb/filter_policy.h" -#include "rocksdb/options.h" -#include "rocksdb/rate_limiter.h" -#include "rocksdb/slice.h" -#include "rocksdb/status.h" -#include "rocksdb/table.h" - -#include "pstd/env.h" -#include "pstd/pstd_mutex.h" -#include "storage/slot_indexer.h" - -namespace pikiwidb { -class Binlog; -} - -namespace storage { - -inline constexpr double ZSET_SCORE_MAX = std::numeric_limits::max(); -inline constexpr double ZSET_SCORE_MIN = std::numeric_limits::lowest(); - -inline const std::string PROPERTY_TYPE_ROCKSDB_CUR_SIZE_ALL_MEM_TABLES = "rocksdb.cur-size-all-mem-tables"; -inline const std::string PROPERTY_TYPE_ROCKSDB_ESTIMATE_TABLE_READER_MEM = "rocksdb.estimate-table-readers-mem"; -inline const std::string PROPERTY_TYPE_ROCKSDB_BACKGROUND_ERRORS = "rocksdb.background-errors"; - -inline constexpr size_t BATCH_DELETE_LIMIT = 100; -inline constexpr size_t COMPACT_THRESHOLD_COUNT = 2000; - -inline constexpr uint64_t kNoFlush = std::numeric_limits::max(); -inline constexpr uint64_t kFlush = 0; - -using Options = rocksdb::Options; -using BlockBasedTableOptions = rocksdb::BlockBasedTableOptions; -using Status = rocksdb::Status; -using Slice = rocksdb::Slice; -using Env = rocksdb::Env; -using LogIndex = int64_t; - -class Redis; -enum class OptionType; - -template -class LRUCache; - -using AppendLogFunction = std::function&&)>; -using DoSnapshotFunction = std::function; - -struct StorageOptions { - mutable rocksdb::Options options; - rocksdb::BlockBasedTableOptions table_options; - size_t block_cache_size = 0; - bool share_block_cache = false; - size_t statistics_max_size = 0; - size_t small_compaction_threshold = 5000; - size_t small_compaction_duration_threshold = 10000; - size_t db_instance_num = 3; // default = 3 - int db_id = 0; - AppendLogFunction append_log_function = nullptr; - DoSnapshotFunction do_snapshot_function = nullptr; - - uint32_t raft_timeout_s = std::numeric_limits::max(); - int64_t max_gap = 1000; - uint64_t mem_manager_size = 100000000; - Status ResetOptions(const OptionType& option_type, const std::unordered_map& options_map); -}; - -struct KeyValue { - std::string key; - std::string value; - bool operator==(const KeyValue& kv) const { return (kv.key == key && kv.value == value); } - bool operator<(const KeyValue& kv) const { return key < kv.key; } -}; - -struct KeyInfo { - uint64_t keys = 0; - uint64_t expires = 0; - uint64_t avg_ttl = 0; - uint64_t invaild_keys = 0; - - KeyInfo() : keys(0), expires(0), avg_ttl(0), invaild_keys(0) {} - - KeyInfo(uint64_t k, uint64_t e, uint64_t a, uint64_t i) : keys(k), expires(e), avg_ttl(a), invaild_keys(i) {} - - KeyInfo operator+(const KeyInfo& info) { - KeyInfo res; - res.keys = keys + info.keys; - res.expires = expires + info.expires; - res.avg_ttl = avg_ttl + info.avg_ttl; - res.invaild_keys = invaild_keys + info.invaild_keys; - return res; - } -}; - -struct ValueStatus { - std::string value; - Status status; - uint64_t ttl; - bool operator==(const ValueStatus& vs) const { return (vs.value == value && vs.status == status && vs.ttl == ttl); } -}; - -struct FieldValue { - std::string field; - std::string value; - FieldValue() = default; - FieldValue(const std::string& k, const std::string& v) : field(k), value(v) {} - FieldValue(std::string&& k, std::string&& v) : field(std::move(k)), value(std::move(v)) {} - bool operator==(const FieldValue& fv) const { return (fv.field == field && fv.value == value); } -}; - -struct KeyVersion { - std::string key; - uint64_t version = 0; - bool operator==(const KeyVersion& kv) const { return (kv.key == key && kv.version == version); } -}; - -struct ScoreMember { - ScoreMember() : score(0.0), member("") {} - ScoreMember(double t_score, const std::string& t_member) : score(t_score), member(t_member) {} - double score; - std::string member; - bool operator==(const ScoreMember& sm) const { return (sm.score == score && sm.member == member); } -}; - -enum BeforeOrAfter { Before, After }; - -enum DataType { kAll, kStrings, kHashes, kSets, kLists, kZSets }; - -const std::string DataTypeToString[] = {"all", "string", "hash", "set", "list", "zset"}; -const char DataTypeTag[] = {'a', 'k', 'h', 's', 'l', 'z'}; - -enum class OptionType { - kDB, - kColumnFamily, -}; - -enum ColumnFamilyType { kMeta, kData, kMetaAndData }; - -enum AGGREGATE { SUM, MIN, MAX }; - -enum BitOpType { kBitOpAnd = 1, kBitOpOr, kBitOpXor, kBitOpNot, kBitOpDefault }; - -enum Operation { - kNone = 0, - kCleanAll, - kCleanStrings, - kCleanHashes, - kCleanZSets, - kCleanSets, - kCleanLists, - kCompactRange -}; - -struct BGTask { - DataType type; - Operation operation; - std::vector argv; - - BGTask(const DataType& _type = DataType::kAll, const Operation& _opeation = Operation::kNone, - const std::vector& _argv = {}) - : type(_type), operation(_opeation), argv(_argv) {} -}; - -class Storage { - public: - Storage(); - ~Storage(); - - Status Open(const StorageOptions& storage_options, const std::string& db_path); - - Status Close(); - - std::vector> CreateCheckpoint(const std::string& checkpoint_path); - - Status CreateCheckpointInternal(const std::string& checkpoint_path, int db_index); - - std::vector> LoadCheckpoint(const std::string& checkpoint_path, const std::string& db_path); - - Status LoadCheckpointInternal(const std::string& dump_path, const std::string& db_path, int index); - - Status LoadCursorStartKey(const DataType& dtype, int64_t cursor, char* type, std::string* start_key); - - Status StoreCursorStartKey(const DataType& dtype, int64_t cursor, char type, const std::string& next_key); - - std::unique_ptr& GetDBInstance(const Slice& key); - - std::unique_ptr& GetDBInstance(const std::string& key); - - // Strings Commands - - // Set key to hold the string value. if key - // already holds a value, it is overwritten - Status Set(const Slice& key, const Slice& value); - - // Set key to hold the string value. if key exist - Status Setxx(const Slice& key, const Slice& value, int32_t* ret, uint64_t ttl = 0); - - // Get the value of key. If the key does not exist - // the special value nil is returned - Status Get(const Slice& key, std::string* value); - - // Get the value and ttl of key. If the key does not exist - // the special value nil is returned. If the key has no ttl, ttl is -1 - Status GetWithTTL(const Slice& key, std::string* value, uint64_t* ttl); - - // Atomically sets key to value and returns the old value stored at key - // Returns an error when key exists but does not hold a string value. - Status GetSet(const Slice& key, const Slice& value, std::string* old_value); - - // Sets or clears the bit at offset in the string value stored at key - Status SetBit(const Slice& key, int64_t offset, int32_t value, int32_t* ret); - - // Returns the bit value at offset in the string value stored at key - Status GetBit(const Slice& key, int64_t offset, int32_t* ret); - - // Sets the given keys to their respective values - // MSET replaces existing values with new values - Status MSet(const std::vector& kvs); - - // Returns the values of all specified keys. For every key - // that does not hold a string value or does not exist, the - // special value nil is returned - Status MGet(const std::vector& keys, std::vector* vss); - - // Returns the values of all specified keyswithTTL. For every key - // that does not hold a string value or does not exist, the - // special value nil is returned - Status MGetWithTTL(const std::vector& keys, std::vector* vss); - - // Set key to hold string value if key does not exist - // return 1 if the key was set - // return 0 if the key was not set - Status Setnx(const Slice& key, const Slice& value, int32_t* ret, uint64_t ttl = 0); - - // Sets the given keys to their respective values. - // MSETNX will not perform any operation at all even - // if just a single key already exists. - Status MSetnx(const std::vector& kvs, int32_t* ret); - - // Set key to hold string new_value if key currently hold the give value - // return 1 if the key currently hold the give value And override success - // return 0 if the key doesn't exist And override fail - // return -1 if the key currently does not hold the given value And override fail - Status Setvx(const Slice& key, const Slice& value, const Slice& new_value, int32_t* ret, uint64_t ttl = 0); - - // delete the key that holds a given value - // return 1 if the key currently hold the give value And delete success - // return 0 if the key doesn't exist And del fail - // return -1 if the key currently does not hold the given value And del fail - Status Delvx(const Slice& key, const Slice& value, int32_t* ret); - - // Set key to hold string value if key does not exist - // return the length of the string after it was modified by the command - Status Setrange(const Slice& key, int64_t start_offset, const Slice& value, int32_t* ret); - - // Returns the substring of the string value stored at key, - // determined by the offsets start and end (both are inclusive) - Status Getrange(const Slice& key, int64_t start_offset, int64_t end_offset, std::string* ret); - - Status GetrangeWithValue(const Slice& key, int64_t start_offset, int64_t end_offset, std::string* ret, - std::string* value, uint64_t* ttl); - - // If key already exists and is a string, this command appends the value at - // the end of the string - // return the length of the string after the append operation - Status Append(const Slice& key, const Slice& value, int32_t* ret); - - // Count the number of set bits (population counting) in a string. - // return the number of bits set to 1 - // note: if need to specified offset, set have_range to true - Status BitCount(const Slice& key, int64_t start_offset, int64_t end_offset, int32_t* ret, bool have_range); - - // Perform a bitwise operation between multiple keys - // and store the result in the destination key - Status BitOp(BitOpType op, const std::string& dest_key, const std::vector& src_keys, - std::string& value_to_dest, int64_t* ret); - - // Return the position of the first bit set to 1 or 0 in a string - // BitPos key 0 - Status BitPos(const Slice& key, int32_t bit, int64_t* ret); - // BitPos key 0 [start] - Status BitPos(const Slice& key, int32_t bit, int64_t start_offset, int64_t* ret); - // BitPos key 0 [start] [end] - Status BitPos(const Slice& key, int32_t bit, int64_t start_offset, int64_t end_offset, int64_t* ret); - - // Decrements the number stored at key by decrement - // return the value of key after the decrement - Status Decrby(const Slice& key, int64_t value, int64_t* ret); - - // Increments the number stored at key by increment. - // If the key does not exist, it is set to 0 before performing the operation - Status Incrby(const Slice& key, int64_t value, int64_t* ret); - - // Increment the string representing a floating point number - // stored at key by the specified increment. - Status Incrbyfloat(const Slice& key, const Slice& value, std::string* ret); - - // Set key to hold the string value and set key to timeout after a given - // number of seconds - Status Setex(const Slice& key, const Slice& value, uint64_t ttl); - - // Returns the length of the string value stored at key. An error - // is returned when key holds a non-string value. - Status Strlen(const Slice& key, int32_t* len); - - // PKSETEXAT has the same effect and semantic as SETEX, but instead of - // specifying the number of seconds representing the TTL (time to live), it - // takes an absolute Unix timestamp (seconds since January 1, 1970). A - // timestamp in the past will delete the key immediately. - Status PKSetexAt(const Slice& key, const Slice& value, uint64_t timestamp); - - // Hashes Commands - - // Sets field in the hash stored at key to value. If key does not exist, a new - // key holding a hash is created. If field already exists in the hash, it is - // overwritten. - Status HSet(const Slice& key, const Slice& field, const Slice& value, int32_t* res); - - // Returns the value associated with field in the hash stored at key. - // the value associated with field, or nil when field is not present in the - // hash or key does not exist. - Status HGet(const Slice& key, const Slice& field, std::string* value); - - // Sets the specified fields to their respective values in the hash stored at - // key. This command overwrites any specified fields already existing in the - // hash. If key does not exist, a new key holding a hash is created. - Status HMSet(const Slice& key, const std::vector& fvs); - - // Returns the values associated with the specified fields in the hash stored - // at key. - // For every field that does not exist in the hash, a nil value is returned. - // Because a non-existing keys are treated as empty hashes, running HMGET - // against a non-existing key will return a list of nil values. - Status HMGet(const Slice& key, const std::vector& fields, std::vector* vss); - - // Returns all fields and values of the hash stored at key. In the returned - // value, every field name is followed by its value, so the length of the - // reply is twice the size of the hash. - Status HGetall(const Slice& key, std::vector* fvs); - - Status HGetallWithTTL(const Slice& key, std::vector* fvs, uint64_t* ttl); - - // Returns all field names in the hash stored at key. - Status HKeys(const Slice& key, std::vector* fields); - - // Returns all values in the hash stored at key. - Status HVals(const Slice& key, std::vector* values); - - // Sets field in the hash stored at key to value, only if field does not yet - // exist. If key does not exist, a new key holding a hash is created. If field - // already exists, this operation has no effect. - Status HSetnx(const Slice& key, const Slice& field, const Slice& value, int32_t* ret); - - // Returns the number of fields contained in the hash stored at key. - // Return 0 when key does not exist. - Status HLen(const Slice& key, int32_t* ret); - - // Returns the string length of the value associated with field in the hash - // stored at key. If the key or the field do not exist, 0 is returned. - Status HStrlen(const Slice& key, const Slice& field, int32_t* len); - - // Returns if field is an existing field in the hash stored at key. - // Return Status::Ok() if the hash contains field. - // Return Status::NotFound() if the hash does not contain field, - // or key does not exist. - Status HExists(const Slice& key, const Slice& field); - - // Increments the number stored at field in the hash stored at key by - // increment. If key does not exist, a new key holding a hash is created. If - // field does not exist the value is set to 0 before the operation is - // performed. - Status HIncrby(const Slice& key, const Slice& field, int64_t value, int64_t* ret); - - // Increment the specified field of a hash stored at key, and representing a - // floating point number, by the specified increment. If the increment value - // is negative, the result is to have the hash field value decremented instead - // of incremented. If the field does not exist, it is set to 0 before - // performing the operation. An error is returned if one of the following - // conditions occur: - // - // The field contains a value of the wrong type (not a string). - // The current field content or the specified increment are not parsable as a - // double precision floating point number. - Status HIncrbyfloat(const Slice& key, const Slice& field, const Slice& by, std::string* new_value); - - // Removes the specified fields from the hash stored at key. Specified fields - // that do not exist within this hash are ignored. If key does not exist, it - // is treated as an empty hash and this command returns 0. - Status HDel(const Slice& key, const std::vector& fields, int32_t* ret); - - // See SCAN for HSCAN documentation. - Status HScan(const Slice& key, int64_t cursor, const std::string& pattern, int64_t count, - std::vector* field_values, int64_t* next_cursor); - - // Iterate over a Hash table of fields - // return next_field that the user need to use as the start_field argument - // in the next call - Status HScanx(const Slice& key, const std::string& start_field, const std::string& pattern, int64_t count, - std::vector* field_values, std::string* next_field); - - // Return random field(s) and value(s) from the hash value stored at key. - Status HRandField(const Slice& key, int64_t count, bool with_values, std::vector* res); - - // Iterate over a Hash table of fields by specified range - // return next_field that the user need to use as the start_field argument - // in the next call - Status PKHScanRange(const Slice& key, const Slice& field_start, const std::string& field_end, const Slice& pattern, - int32_t limit, std::vector* field_values, std::string* next_field); - - // part from the reversed ordering, PKHRSCANRANGE is similar to PKHScanRange - Status PKHRScanRange(const Slice& key, const Slice& field_start, const std::string& field_end, const Slice& pattern, - int32_t limit, std::vector* field_values, std::string* next_field); - - // Sets Commands - - // Add the specified members to the set stored at key. Specified members that - // are already a member of this set are ignored. If key does not exist, a new - // set is created before adding the specified members. - Status SAdd(const Slice& key, const std::vector& members, int32_t* ret); - - // Returns the set cardinality (number of elements) of the set stored at key. - Status SCard(const Slice& key, int32_t* ret); - - // Returns the members of the set resulting from the difference between the - // first set and all the successive sets. - // - // For example: - // key1 = {a, b, c, d} - // key2 = {c} - // key3 = {a, c, e} - // SDIFF key1 key2 key3 = {b, d} - Status SDiff(const std::vector& keys, std::vector* members); - - // This command is equal to SDIFF, but instead of returning the resulting set, - // it is stored in destination. - // If destination already exists, it is overwritten. - // - // For example: - // destination = {}; - // key1 = {a, b, c, d} - // key2 = {c} - // key3 = {a, c, e} - // SDIFFSTORE destination key1 key2 key3 - // destination = {b, d} - Status SDiffstore(const Slice& destination, const std::vector& keys, - std::vector& value_to_dest, int32_t* ret); - - // Returns the members of the set resulting from the intersection of all the - // given sets. - // - // For example: - // key1 = {a, b, c, d} - // key2 = {c} - // key3 = {a, c, e} - // SINTER key1 key2 key3 = {c} - Status SInter(const std::vector& keys, std::vector* members); - - // This command is equal to SINTER, but instead of returning the resulting - // set, it is stored in destination. - // If destination already exists, it is overwritten. - // - // For example: - // destination = {} - // key1 = {a, b, c, d} - // key2 = {a, c} - // key3 = {a, c, e} - // SINTERSTORE destination key1 key2 key3 - // destination = {a, c} - Status SInterstore(const Slice& destination, const std::vector& keys, - std::vector& value_to_dest, int32_t* ret); - - // Returns if member is a member of the set stored at key. - Status SIsmember(const Slice& key, const Slice& member, int32_t* ret); - - // Returns all the members of the set value stored at key. - // This has the same effect as running SINTER with one argument key. - Status SMembers(const Slice& key, std::vector* members); - - Status SMembersWithTTL(const Slice& key, std::vector* members, uint64_t* ttl); - - // Remove the specified members from the set stored at key. Specified members - // that are not a member of this set are ignored. If key does not exist, it is - // treated as an empty set and this command returns 0. - Status SRem(const Slice& key, const std::vector& members, int32_t* ret); - - // Removes and returns several random elements specified by count from the set value store at key. - Status SPop(const Slice& key, std::vector* members, int64_t count); - - // When called with just the key argument, return a random element from the - // set value stored at key. - // when called with the additional count argument, return an array of count - // distinct elements if count is positive. If called with a negative count the - // behavior changes and the command is allowed to return the same element - // multiple times. In this case the number of returned elements is the - // absolute value of the specified count - Status SRandmember(const Slice& key, int32_t count, std::vector* members); - - // Move member from the set at source to the set at destination. This - // operation is atomic. In every given moment the element will appear to be a - // member of source or destination for other clients. - // - // If the source set does not exist or does not contain the specified element, - // no operation is performed and 0 is returned. Otherwise, the element is - // removed from the source set and added to the destination set. When the - // specified element already exists in the destination set, it is only removed - // from the source set. - Status SMove(const Slice& source, const Slice& destination, const Slice& member, int32_t* ret); - - // Returns the members of the set resulting from the union of all the given - // sets. - // - // For example: - // key1 = {a, b, c, d} - // key2 = {c} - // key3 = {a, c, e} - // SUNION key1 key2 key3 = {a, b, c, d, e} - Status SUnion(const std::vector& keys, std::vector* members); - - // This command is equal to SUNION, but instead of returning the resulting - // set, it is stored in destination. - // If destination already exists, it is overwritten. - // - // For example: - // key1 = {a, b} - // key2 = {c, d} - // key3 = {c, d, e} - // SUNIONSTORE destination key1 key2 key3 - // destination = {a, b, c, d, e} - Status SUnionstore(const Slice& destination, const std::vector& keys, - std::vector& value_to_dest, int32_t* ret); - - // See SCAN for SSCAN documentation. - Status SScan(const Slice& key, int64_t cursor, const std::string& pattern, int64_t count, - std::vector* members, int64_t* next_cursor); - - // Lists Commands - - // Insert all the specified values at the head of the list stored at key. If - // key does not exist, it is created as empty list before performing the push - // operations. - Status LPush(const Slice& key, const std::vector& values, uint64_t* ret); - - // Insert all the specified values at the tail of the list stored at key. If - // key does not exist, it is created as empty list before performing the push - // operation. - Status RPush(const Slice& key, const std::vector& values, uint64_t* ret); - - // Returns the specified elements of the list stored at key. The offsets start - // and stop are zero-based indexes, with 0 being the first element of the list - // (the head of the list), 1 being the next element and so on. - Status LRange(const Slice& key, int64_t start, int64_t stop, std::vector* ret); - - Status LRangeWithTTL(const Slice& key, int64_t start, int64_t stop, std::vector* ret, uint64_t* ttl); - - // Removes the first count occurrences of elements equal to value from the - // list stored at key. The count argument influences the operation in the - // following ways - Status LTrim(const Slice& key, int64_t start, int64_t stop); - - // Returns the length of the list stored at key. If key does not exist, it is - // interpreted as an empty list and 0 is returned. An error is returned when - // the value stored at key is not a list. - Status LLen(const Slice& key, uint64_t* len); - - // Removes and returns the first elements of the list stored at key. - Status LPop(const Slice& key, int64_t count, std::vector* elements); - - // Removes and returns the last elements of the list stored at key. - Status RPop(const Slice& key, int64_t count, std::vector* elements); - - // Returns the element at index index in the list stored at key. The index is - // zero-based, so 0 means the first element, 1 the second element and so on. - // Negative indices can be used to designate elements starting at the tail of - // the list. Here, -1 means the last element, -2 means the penultimate and so - // forth. - Status LIndex(const Slice& key, int64_t index, std::string* element); - - // Inserts value in the list stored at key either before or after the - // reference value pivot. - // When key does not exist, it is considered an empty list and no operation is - // performed. - // An error is returned when key exists but does not hold a list value. - Status LInsert(const Slice& key, const BeforeOrAfter& before_or_after, const std::string& pivot, - const std::string& value, int64_t* ret); - - // Inserts value at the head of the list stored at key, only if key already - // exists and holds a list. In contrary to LPUSH, no operation will be - // performed when key does not yet exist. - Status LPushx(const Slice& key, const std::vector& values, uint64_t* len); - - // Inserts value at the tail of the list stored at key, only if key already - // exists and holds a list. In contrary to RPUSH, no operation will be - // performed when key does not yet exist. - Status RPushx(const Slice& key, const std::vector& values, uint64_t* len); - - // Removes the first count occurrences of elements equal to value from the - // list stored at key. The count argument influences the operation in the - // following ways: - // - // count > 0: Remove elements equal to value moving from head to tail. - // count < 0: Remove elements equal to value moving from tail to head. - // count = 0: Remove all elements equal to value. - // For example, LREM list -2 "hello" will remove the last two occurrences of - // "hello" in the list stored at list. - // - // Note that non-existing keys are treated like empty lists, so when key does - // not exist, the command will always return 0. - Status LRem(const Slice& key, int64_t count, const Slice& value, uint64_t* ret); - - // Sets the list element at index to value. For more information on the index - // argument, see LINDEX. - // - // An error is returned for out of range indexes. - Status LSet(const Slice& key, int64_t index, const Slice& value); - - // Atomically returns and removes the last element (tail) of the list stored - // at source, and pushes the element at the first element (head) of the list - // stored at destination. - // - // For example: consider source holding the list a,b,c, and destination - // holding the list x,y,z. Executing RPOPLPUSH results in source holding a,b - // and destination holding c,x,y,z. - // - // If source does not exist, the value nil is returned and no operation is - // performed. If source and destination are the same, the operation is - // equivalent to removing the last element from the list and pushing it as - // first element of the list, so it can be considered as a list rotation - // command. - Status RPoplpush(const Slice& source, const Slice& destination, std::string* element); - - // Zsets Commands - - // Pop the maximum count score_members which have greater score in the sorted set. - // And return the result in the score_members,If the total number of the sorted - // set less than count, it will pop out the total number of sorted set. If two - // ScoreMember's score were the same, the lexicographic predominant elements will - // be pop out. - Status ZPopMax(const Slice& key, int64_t count, std::vector* score_members); - - // Pop the minimum count score_members which have less score in the sorted set. - // And return the result in the score_members,If the total number of the sorted - // set less than count, it will pop out the total number of sorted set. If two - // ScoreMember's score were the same, the lexicographic predominant elements will - // not be pop out. - Status ZPopMin(const Slice& key, int64_t count, std::vector* score_members); - - // Adds all the specified members with the specified scores to the sorted set - // stored at key. It is possible to specify multiple score / member pairs. If - // a specified member is already a member of the sorted set, the score is - // updated and the element reinserted at the right position to ensure the - // correct ordering. - // - // If key does not exist, a new sorted set with the specified members as sole - // members is created, like if the sorted set was empty. If the key exists but - // does not hold a sorted set, an error is returned. - // The score values should be the string representation of a double precision - // floating point number. +inf and -inf values are valid values as well. - Status ZAdd(const Slice& key, const std::vector& score_members, int32_t* ret); - - // Returns the sorted set cardinality (number of elements) of the sorted set - // stored at key. - Status ZCard(const Slice& key, int32_t* ret); - - // Returns the number of elements in the sorted set at key with a score - // between min and max. - // - // The min and max arguments have the same semantic as described for - // ZRANGEBYSCORE. - // - // Note: the command has a complexity of just O(log(N)) because it uses - // elements ranks (see ZRANK) to get an idea of the range. Because of this - // there is no need to do a work proportional to the size of the range. - Status ZCount(const Slice& key, double min, double max, bool left_close, bool right_close, int32_t* ret); - - // Increments the score of member in the sorted set stored at key by - // increment. If member does not exist in the sorted set, it is added with - // increment as its score (as if its previous score was 0.0). If key does not - // exist, a new sorted set with the specified member as its sole member is - // created. - // - // An error is returned when key exists but does not hold a sorted set. - // - // The score value should be the string representation of a numeric value, and - // accepts double precision floating point numbers. It is possible to provide - // a negative value to decrement the score. - Status ZIncrby(const Slice& key, const Slice& member, double increment, double* ret); - - // Returns the specified range of elements in the sorted set stored at key. - // The elements are considered to be ordered from the lowest to the highest - // score. Lexicographical order is used for elements with equal score. - // - // See ZREVRANGE when you need the elements ordered from highest to lowest - // score (and descending lexicographical order for elements with equal score). - // - // Both start and stop are zero-based indexes, where 0 is the first element, 1 - // is the next element and so on. They can also be negative numbers indicating - // offsets from the end of the sorted set, with -1 being the last element of - // the sorted set, -2 the penultimate element and so on. - // - // start and stop are inclusive ranges, so for example ZRANGE myzset 0 1 will - // return both the first and the second element of the sorted set. - // - // Out of range indexes will not produce an error. If start is larger than the - // largest index in the sorted set, or start > stop, an empty list is - // returned. If stop is larger than the end of the sorted set Redis will treat - // it like it is the last element of the sorted set. - // - // It is possible to pass the WITHSCORES option in order to return the scores - // of the elements together with the elements. The returned list will contain - // value1,score1,...,valueN,scoreN instead of value1,...,valueN. Client - // libraries are free to return a more appropriate data type (suggestion: an - // array with (value, score) arrays/tuples). - Status ZRange(const Slice& key, int32_t start, int32_t stop, std::vector* score_members); - - Status ZRangeWithTTL(const Slice& key, int32_t start, int32_t stop, std::vector* score_members, - uint64_t* ttl); - - // Returns all the elements in the sorted set at key with a score between min - // and max (including elements with score equal to min or max). The elements - // are considered to be ordered from low to high scores. - // - // The elements having the same score are returned in lexicographical order - // (this follows from a property of the sorted set implementation in Redis and - // does not involve further computation). - // - // The optional LIMIT argument can be used to only get a range of the matching - // elements (similar to SELECT LIMIT offset, count in SQL). Keep in mind that - // if offset is large, the sorted set needs to be traversed for offset - // elements before getting to the elements to return, which can add up to O(N) - // time complexity. - // - // The optional WITHSCORES argument makes the command return both the element - // and its score, instead of the element alone. This option is available since - // Redis 2.0. - // - // Exclusive intervals and infinity - // min and max can be -inf and +inf, so that you are not required to know the - // highest or lowest score in the sorted set to get all elements from or up to - // a certain score. - // - // By default, the interval specified by min and max is closed (inclusive). It - // is possible to specify an open interval (exclusive) by prefixing the score - // with the character (. For example: - // - // ZRANGEBYSCORE zset (1 5 - // Will return all elements with 1 < score <= 5 while: - // - // ZRANGEBYSCORE zset (5 (10 - // Will return all the elements with 5 < score < 10 (5 and 10 excluded). - // - // Return value - // Array reply: list of elements in the specified score range (optionally with - // their scores). - Status ZRangebyscore(const Slice& key, double min, double max, bool left_close, bool right_close, - std::vector* score_members); - - // Returns all the elements in the sorted set at key with a score between min - // and max (including elements with score equal to min or max). The elements - // are considered to be ordered from low to high scores. - // - // The elements having the same score are returned in lexicographical order - // (this follows from a property of the sorted set implementation in Redis and - // does not involve further computation). - // - // The optional LIMIT argument can be used to only get a range of the matching - // elements (similar to SELECT LIMIT offset, count in SQL). Keep in mind that - // if offset is large, the sorted set needs to be traversed for offset - // elements before getting to the elements to return, which can add up to O(N) - // time complexity. - // - // The optional WITHSCORES argument makes the command return both the element - // and its score, instead of the element alone. This option is available since - // Redis 2.0. - // - // Exclusive intervals and infinity - // min and max can be -inf and +inf, so that you are not required to know the - // highest or lowest score in the sorted set to get all elements from or up to - // a certain score. - // - // By default, the interval specified by min and max is closed (inclusive). It - // is possible to specify an open interval (exclusive) by prefixing the score - // with the character (. For example: - // - // ZRANGEBYSCORE zset (1 5 - // Will return all elements with 1 < score <= 5 while: - // - // ZRANGEBYSCORE zset (5 (10 - // Will return all the elements with 5 < score < 10 (5 and 10 excluded). - // - // Return value - // Array reply: list of elements in the specified score range (optionally with - // their scores). - Status ZRangebyscore(const Slice& key, double min, double max, bool left_close, bool right_close, int64_t count, - int64_t offset, std::vector* score_members); - - // Returns the rank of member in the sorted set stored at key, with the scores - // ordered from low to high. The rank (or index) is 0-based, which means that - // the member with the lowest score has rank 0. - // - // Use ZREVRANK to get the rank of an element with the scores ordered from - // high to low. - Status ZRank(const Slice& key, const Slice& member, int32_t* rank); - - // Removes the specified members from the sorted set stored at key. Non - // existing members are ignored. - // - // An error is returned when key exists and does not hold a sorted set. - Status ZRem(const Slice& key, const std::vector& members, int32_t* ret); - - // Removes all elements in the sorted set stored at key with rank between - // start and stop. Both start and stop are 0 -based indexes with 0 being the - // element with the lowest score. These indexes can be negative numbers, where - // they indicate offsets starting at the element with the highest score. For - // example: -1 is the element with the highest score, -2 the element with the - // second highest score and so forth. - Status ZRemrangebyrank(const Slice& key, int32_t start, int32_t stop, int32_t* ret); - - // Removes all elements in the sorted set stored at key with a score between - // min and max (inclusive). - Status ZRemrangebyscore(const Slice& key, double min, double max, bool left_close, bool right_close, int32_t* ret); - - // Returns the specified range of elements in the sorted set stored at key. - // The elements are considered to be ordered from the highest to the lowest - // score. Descending lexicographical order is used for elements with equal - // score. - // - // Apart from the reversed ordering, ZREVRANGE is similar to ZRANGE. - Status ZRevrange(const Slice& key, int32_t start, int32_t stop, std::vector* score_members); - - // Returns all the elements in the sorted set at key with a score between max - // and min (including elements with score equal to max or min). In contrary to - // the default ordering of sorted sets, for this command the elements are - // considered to be ordered from high to low scores. - // - // The elements having the same score are returned in reverse lexicographical - // order. - // - // Apart from the reversed ordering, ZREVRANGEBYSCORE is similar to - // ZRANGEBYSCORE. - Status ZRevrangebyscore(const Slice& key, double min, double max, bool left_close, bool right_close, - std::vector* score_members); - - // Returns all the elements in the sorted set at key with a score between max - // and min (including elements with score equal to max or min). In contrary to - // the default ordering of sorted sets, for this command the elements are - // considered to be ordered from high to low scores. - // - // The elements having the same score are returned in reverse lexicographical - // order. - // - // Apart from the reversed ordering, ZREVRANGEBYSCORE is similar to - // ZRANGEBYSCORE. - Status ZRevrangebyscore(const Slice& key, double min, double max, bool left_close, bool right_close, int64_t count, - int64_t offset, std::vector* score_members); - - // Returns the rank of member in the sorted set stored at key, with the scores - // ordered from high to low. The rank (or index) is 0-based, which means that - // the member with the highest score has rank 0. - Status ZRevrank(const Slice& key, const Slice& member, int32_t* rank); - - // Returns the score of member in the sorted set at key. - // - // If member does not exist in the sorted set, or key does not exist, nil is - // returned. - Status ZScore(const Slice& key, const Slice& member, double* ret); - - // Computes the union of numkeys sorted sets given by the specified keys, and - // stores the result in destination. It is mandatory to provide the number of - // input keys (numkeys) before passing the input keys and the other (optional) - // arguments. - // - // By default, the resulting score of an element is the sum of its scores in - // the sorted sets where it exists. - // - // Using the WEIGHTS option, it is possible to specify a multiplication factor - // for each input sorted set. This means that the score of every element in - // every input sorted set is multiplied by this factor before being passed to - // the aggregation function. When WEIGHTS is not given, the multiplication - // factors default to 1. - // - // With the AGGREGATE option, it is possible to specify how the results of the - // union are aggregated. This option defaults to SUM, where the score of an - // element is summed across the inputs where it exists. When this option is - // set to either MIN or MAX, the resulting set will contain the minimum or - // maximum score of an element across the inputs where it exists. - // - // If destination already exists, it is overwritten. - Status ZUnionstore(const Slice& destination, const std::vector& keys, const std::vector& weights, - AGGREGATE agg, std::map& value_to_dest, int32_t* ret); - - // Computes the intersection of numkeys sorted sets given by the specified - // keys, and stores the result in destination. It is mandatory to provide the - // number of input keys (numkeys) before passing the input keys and the other - // (optional) arguments. - // - // By default, the resulting score of an element is the sum of its scores in - // the sorted sets where it exists. Because intersection requires an element - // to be a member of every given sorted set, this results in the score of - // every element in the resulting sorted set to be equal to the number of - // input sorted sets. - // - // For a description of the WEIGHTS and AGGREGATE options, see ZUNIONSTORE. - // - // If destination already exists, it is overwritten. - Status ZInterstore(const Slice& destination, const std::vector& keys, const std::vector& weights, - AGGREGATE agg, std::vector& value_to_dest, int32_t* ret); - - // When all the elements in a sorted set are inserted with the same score, in - // order to force lexicographical ordering, this command returns all the - // elements in the sorted set at key with a value between min and max. - // - // If the elements in the sorted set have different scores, the returned - // elements are unspecified. - // - // The elements are considered to be ordered from lower to higher strings as - // compared byte-by-byte using the memcmp() C function. Longer strings are - // considered greater than shorter strings if the common part is identical. - // - // The optional LIMIT argument can be used to only get a range of the matching - // elements (similar to SELECT LIMIT offset, count in SQL). Keep in mind that - // if offset is large, the sorted set needs to be traversed for offset - // elements before getting to the elements to return, which can add up to O(N) - // time complexity. - Status ZRangebylex(const Slice& key, const Slice& min, const Slice& max, bool left_close, bool right_close, - std::vector* members); - - // When all the elements in a sorted set are inserted with the same score, in - // order to force lexicographical ordering, this command returns the number of - // elements in the sorted set at key with a value between min and max. - // - // The min and max arguments have the same meaning as described for - // ZRANGEBYLEX. - // - // Note: the command has a complexity of just O(log(N)) because it uses - // elements ranks (see ZRANK) to get an idea of the range. Because of this - // there is no need to do a work proportional to the size of the range. - Status ZLexcount(const Slice& key, const Slice& min, const Slice& max, bool left_close, bool right_close, - int32_t* ret); - - // When all the elements in a sorted set are inserted with the same score, in - // order to force lexicographical ordering, this command removes all elements - // in the sorted set stored at key between the lexicographical range specified - // by min and max. - // - // The meaning of min and max are the same of the ZRANGEBYLEX command. - // Similarly, this command actually returns the same elements that ZRANGEBYLEX - // would return if called with the same min and max arguments. - Status ZRemrangebylex(const Slice& key, const Slice& min, const Slice& max, bool left_close, bool right_close, - int32_t* ret); - - // See SCAN for ZSCAN documentation. - Status ZScan(const Slice& key, int64_t cursor, const std::string& pattern, int64_t count, - std::vector* score_members, int64_t* next_cursor); - - // Keys Commands - - // Note: - // While any error happens, you need to check type_status for - // the error message - - // Set a timeout on key - // return -1 operation exception errors happen in database - // return >=0 success - int32_t Expire(const Slice& key, uint64_t ttl); - - // Removes the specified keys - // return -1 operation exception errors happen in database - // return >=0 the number of keys that were removed - int64_t Del(const std::vector& keys); - - // Removes the specified keys of the specified type - // return -1 operation exception errors happen in database - // return >= 0 the number of keys that were removed - int64_t DelByType(const std::vector& keys, const DataType& type); - - // Iterate over a collection of elements - // return an updated cursor that the user need to use as the cursor argument - // in the next call - int64_t Scan(const DataType& dtype, int64_t cursor, const std::string& pattern, int64_t count, - std::vector* keys); - - // Iterate over a collection of elements by specified range - // return a next_key that the user need to use as the key_start argument - // in the next call - Status PKScanRange(const DataType& data_type, const Slice& key_start, const Slice& key_end, const Slice& pattern, - int32_t limit, std::vector* keys, std::vector* kvs, std::string* next_key); - - // part from the reversed ordering, PKRSCANRANGE is similar to PKScanRange - Status PKRScanRange(const DataType& data_type, const Slice& key_start, const Slice& key_end, const Slice& pattern, - int32_t limit, std::vector* keys, std::vector* kvs, std::string* next_key); - - // Traverses the database of the specified type, removing the Key that matches - // the pattern - Status PKPatternMatchDel(const DataType& data_type, const std::string& pattern, int32_t* ret); - - // Iterate over a collection of elements - // return next_key that the user need to use as the start_key argument - // in the next call - Status Scanx(const DataType& data_type, const std::string& start_key, const std::string& pattern, int64_t count, - std::vector* keys, std::string* next_key); - - // Returns if key exists. - // return -1 operation exception errors happen in database - // return >=0 the number of keys existing - int64_t Exists(const std::vector& keys); - - // Return the key exists type count - // return param type_status: return every type status - int64_t IsExist(const Slice& key, std::map* type_status); - - // EXPIREAT has the same effect and semantic as EXPIRE, but instead of - // specifying the number of seconds representing the TTL (time to live), it - // takes an absolute Unix timestamp (seconds since January 1, 1970). A - // timestamp in the past will delete the key immediately. - // return -1 operation exception errors happen in database - // return 0 if key does not exist - // return >=1 if the timueout was set - int32_t Expireat(const Slice& key, uint64_t timestamp); - - // Remove the existing timeout on key, turning the key from volatile (a key - // with an expire set) to persistent (a key that will never expire as no - // timeout is associated). - // return -1 operation exception errors happen in database - // return 0 if key does not exist or does not have an associated timeout - // return >=1 if the timueout was set - int32_t Persist(const Slice& key, std::map* type_status); - - // Returns the remaining time to live of a key that has a timeout. - // return -3 operation exception errors happen in database - // return -2 if the key does not exist - // return -1 if the key exists but has not associated expire - // return > 0 TTL in seconds - std::map TTL(const Slice& key, std::map* type_status); - - // Reutrns the data all type of the key - // if single is true, the query will return the first one - Status GetType(const std::string& key, bool single, std::vector& types); - - // Reutrns the data all type of the key - Status Type(const std::string& key, std::vector& types); - - Status Keys(const DataType& data_type, const std::string& pattern, std::vector* keys); - - // Return OK if Rename successfully, - // NotFound if key doesn't exist, - // otherwise abort. - Status Rename(const std::string& key, const std::string& newkey); - - // Return OK if Renamenx successfully, - // NotFound if key doesn't exist, - // Corruption if newkey already exists, - // otherwise abort. - Status Renamenx(const std::string& key, const std::string& newkey); - - // Dynamic switch WAL - void DisableWal(const bool is_wal_disable); - - // Iterate through all the data in the database. - void ScanDatabase(const DataType& type); - - // HyperLogLog - enum { - kMaxKeys = 255, - kPrecision = 17, - }; - // Adds all the element arguments to the HyperLogLog data structure stored - // at the variable name specified as first argument. - Status PfAdd(const Slice& key, const std::vector& values, bool* update); - - // When called with a single key, returns the approximated cardinality - // computed by the HyperLogLog data structure stored at the specified - // variable, which is 0 if the variable does not exist. - Status PfCount(const std::vector& keys, int64_t* result); - - // Merge multiple HyperLogLog values into an unique value that will - // approximate the cardinality of the union of the observed Sets of the source - // HyperLogLog structures. - Status PfMerge(const std::vector& keys, std::string& value_to_dest); - - // Admin Commands - Status StartBGThread(); - Status RunBGTask(); - Status AddBGTask(const BGTask& bg_task); - - Status Compact(const DataType& type, bool sync = false); - Status CompactRange(const DataType& type, const std::string& start, const std::string& end, bool sync = false); - Status DoCompactRange(const DataType& type, const std::string& start, const std::string& end); - Status DoCompactSpecificKey(const DataType& type, const std::string& key); - - Status SetMaxCacheStatisticKeys(uint32_t max_cache_statistic_keys); - Status SetSmallCompactionThreshold(uint32_t small_compaction_threshold); - Status SetSmallCompactionDurationThreshold(uint32_t small_compaction_duration_threshold); - - std::string GetCurrentTaskType(); - Status GetUsage(const std::string& property, uint64_t* result); - Status GetUsage(const std::string& property, std::map* type_result); - uint64_t GetProperty(const std::string& property); - - Status GetKeyNum(std::vector* key_infos); - Status StopScanKeyNum(); - - rocksdb::DB* GetDBByIndex(int index); - - Status SetOptions(const OptionType& option_type, const std::unordered_map& options); - void GetRocksDBInfo(std::string& info); - Status OnBinlogWrite(const pikiwidb::Binlog& log, LogIndex log_idx); - - LogIndex GetSmallestFlushedLogIndex() const; - - private: - std::vector> insts_; - std::unique_ptr slot_indexer_; - std::atomic is_opened_ = false; - - std::unique_ptr> cursors_store_; - - // Storage start the background thread for compaction task - pthread_t bg_tasks_thread_id_ = 0; - pstd::Mutex bg_tasks_mutex_; - pstd::CondVar bg_tasks_cond_var_; - std::queue bg_tasks_queue_; - - std::atomic current_task_type_ = kNone; - std::atomic bg_tasks_should_exit_ = false; - - // For scan keys in data base - std::atomic scan_keynum_exit_ = false; - size_t db_instance_num_ = 3; - int db_id_ = 0; -}; - -} // namespace storage -#endif // INCLUDE_STORAGE_STORAGE_H_ diff --git a/src/storage/src/log_index.h b/src/storage/src/log_index.h index e7eb31cbc..aae1707ee 100644 --- a/src/storage/src/log_index.h +++ b/src/storage/src/log_index.h @@ -133,8 +133,8 @@ class LogIndexOfColumnFamilies { last_flush_index_.SetLogIndexSeqnoPair(lastest_flush_log_index, lastest_flush_sequence_number); } - // for gtest - LogIndexSeqnoPair &GetLastFlushIndex() { return last_flush_index_; } + // get the the latest global minimum flushed_index + const LogIndexSeqnoPair &GetLastFlushIndex() const { return last_flush_index_; } LogIndexPair &GetCFStatus(size_t cf) { return cf_[cf]; } diff --git a/src/storage/src/redis.h b/src/storage/src/redis.h index 1b3334649..3b169221e 100644 --- a/src/storage/src/redis.h +++ b/src/storage/src/redis.h @@ -110,9 +110,7 @@ class Redis { void UpdateAppliedLogIndexOfColumnFamily(size_t cf_idx, LogIndex logidx, SequenceNumber seqno) { log_index_of_all_cfs_.Update(cf_idx, logidx, seqno); } - LogIndex GetSmallestFlushedLogIndex() const { - return log_index_of_all_cfs_.GetSmallestLogIndex(-1).smallest_flushed_log_index; - } + LogIndex GetSmallestFlushedLogIndex() const { return log_index_of_all_cfs_.GetLastFlushIndex().GetLogIndex(); } bool IsRestarting() const { return is_starting_; } void StartingPhaseEnd() { is_starting_ = false; } diff --git a/src/storage/src/storage.cc b/src/storage/src/storage.cc index 9e6f76731..49bb30566 100644 --- a/src/storage/src/storage.cc +++ b/src/storage/src/storage.cc @@ -245,7 +245,6 @@ Status Storage::LoadCheckpointInternal(const std::string& checkpoint_sub_path, c int index) { auto rocksdb_path = AppendSubDirectory(db_sub_path, index); // ./db/db_id/index auto tmp_rocksdb_path = rocksdb_path + ".tmp"; // ./db/db_id/index.tmp - insts_[index].reset(); auto source_dir = AppendSubDirectory(checkpoint_sub_path, index); // 1) Rename the original db to db.tmp, and only perform the maximum possible recovery of data From 8e56daf84fb42b5cdc91188f4d7c1e4e1a6fc963 Mon Sep 17 00:00:00 2001 From: panlei-coder Date: Tue, 16 Jul 2024 22:51:52 +0800 Subject: [PATCH 9/9] fix: fix a comment --- src/praft/praft.cc | 1 - src/storage/include/storage/storage.h | 1151 +++++++++++++++++++++++++ 2 files changed, 1151 insertions(+), 1 deletion(-) create mode 100644 src/storage/include/storage/storage.h diff --git a/src/praft/praft.cc b/src/praft/praft.cc index ed729b8a4..c171ff4b0 100644 --- a/src/praft/praft.cc +++ b/src/praft/praft.cc @@ -9,7 +9,6 @@ #include -#include "braft/raft.h" #include "braft/snapshot.h" #include "braft/util.h" #include "brpc/server.h" diff --git a/src/storage/include/storage/storage.h b/src/storage/include/storage/storage.h new file mode 100644 index 000000000..8c50f2aba --- /dev/null +++ b/src/storage/include/storage/storage.h @@ -0,0 +1,1151 @@ +// Copyright (c) 2017-present, Qihoo, Inc. All rights reserved. +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. An additional grant +// of patent rights can be found in the PATENTS file in the same directory. + +#ifndef INCLUDE_STORAGE_STORAGE_H_ +#define INCLUDE_STORAGE_STORAGE_H_ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "rocksdb/convenience.h" +#include "rocksdb/filter_policy.h" +#include "rocksdb/options.h" +#include "rocksdb/rate_limiter.h" +#include "rocksdb/slice.h" +#include "rocksdb/status.h" +#include "rocksdb/table.h" + +#include "pstd/env.h" +#include "pstd/pstd_mutex.h" +#include "storage/slot_indexer.h" + +namespace pikiwidb { +class Binlog; +} + +namespace storage { + +inline constexpr double ZSET_SCORE_MAX = std::numeric_limits::max(); +inline constexpr double ZSET_SCORE_MIN = std::numeric_limits::lowest(); + +inline const std::string PROPERTY_TYPE_ROCKSDB_CUR_SIZE_ALL_MEM_TABLES = "rocksdb.cur-size-all-mem-tables"; +inline const std::string PROPERTY_TYPE_ROCKSDB_ESTIMATE_TABLE_READER_MEM = "rocksdb.estimate-table-readers-mem"; +inline const std::string PROPERTY_TYPE_ROCKSDB_BACKGROUND_ERRORS = "rocksdb.background-errors"; + +inline constexpr size_t BATCH_DELETE_LIMIT = 100; +inline constexpr size_t COMPACT_THRESHOLD_COUNT = 2000; + +inline constexpr uint64_t kNoFlush = std::numeric_limits::max(); +inline constexpr uint64_t kFlush = 0; + +using Options = rocksdb::Options; +using BlockBasedTableOptions = rocksdb::BlockBasedTableOptions; +using Status = rocksdb::Status; +using Slice = rocksdb::Slice; +using Env = rocksdb::Env; +using LogIndex = int64_t; + +class Redis; +enum class OptionType; + +template +class LRUCache; + +using AppendLogFunction = std::function&&)>; +using DoSnapshotFunction = std::function; + +struct StorageOptions { + mutable rocksdb::Options options; + rocksdb::BlockBasedTableOptions table_options; + size_t block_cache_size = 0; + bool share_block_cache = false; + size_t statistics_max_size = 0; + size_t small_compaction_threshold = 5000; + size_t small_compaction_duration_threshold = 10000; + size_t db_instance_num = 3; // default = 3 + int db_id = 0; + AppendLogFunction append_log_function = nullptr; + DoSnapshotFunction do_snapshot_function = nullptr; + + uint32_t raft_timeout_s = std::numeric_limits::max(); + int64_t max_gap = 1000; + uint64_t mem_manager_size = 100000000; + Status ResetOptions(const OptionType& option_type, const std::unordered_map& options_map); +}; + +struct KeyValue { + std::string key; + std::string value; + bool operator==(const KeyValue& kv) const { return (kv.key == key && kv.value == value); } + bool operator<(const KeyValue& kv) const { return key < kv.key; } +}; + +struct KeyInfo { + uint64_t keys = 0; + uint64_t expires = 0; + uint64_t avg_ttl = 0; + uint64_t invaild_keys = 0; + + KeyInfo() : keys(0), expires(0), avg_ttl(0), invaild_keys(0) {} + + KeyInfo(uint64_t k, uint64_t e, uint64_t a, uint64_t i) : keys(k), expires(e), avg_ttl(a), invaild_keys(i) {} + + KeyInfo operator+(const KeyInfo& info) { + KeyInfo res; + res.keys = keys + info.keys; + res.expires = expires + info.expires; + res.avg_ttl = avg_ttl + info.avg_ttl; + res.invaild_keys = invaild_keys + info.invaild_keys; + return res; + } +}; + +struct ValueStatus { + std::string value; + Status status; + uint64_t ttl; + bool operator==(const ValueStatus& vs) const { return (vs.value == value && vs.status == status && vs.ttl == ttl); } +}; + +struct FieldValue { + std::string field; + std::string value; + FieldValue() = default; + FieldValue(const std::string& k, const std::string& v) : field(k), value(v) {} + FieldValue(std::string&& k, std::string&& v) : field(std::move(k)), value(std::move(v)) {} + bool operator==(const FieldValue& fv) const { return (fv.field == field && fv.value == value); } +}; + +struct KeyVersion { + std::string key; + uint64_t version = 0; + bool operator==(const KeyVersion& kv) const { return (kv.key == key && kv.version == version); } +}; + +struct ScoreMember { + ScoreMember() : score(0.0), member("") {} + ScoreMember(double t_score, const std::string& t_member) : score(t_score), member(t_member) {} + double score; + std::string member; + bool operator==(const ScoreMember& sm) const { return (sm.score == score && sm.member == member); } +}; + +enum BeforeOrAfter { Before, After }; + +enum DataType { kAll, kStrings, kHashes, kSets, kLists, kZSets }; + +const std::string DataTypeToString[] = {"all", "string", "hash", "set", "list", "zset"}; +const char DataTypeTag[] = {'a', 'k', 'h', 's', 'l', 'z'}; + +enum class OptionType { + kDB, + kColumnFamily, +}; + +enum ColumnFamilyType { kMeta, kData, kMetaAndData }; + +enum AGGREGATE { SUM, MIN, MAX }; + +enum BitOpType { kBitOpAnd = 1, kBitOpOr, kBitOpXor, kBitOpNot, kBitOpDefault }; + +enum Operation { + kNone = 0, + kCleanAll, + kCleanStrings, + kCleanHashes, + kCleanZSets, + kCleanSets, + kCleanLists, + kCompactRange +}; + +struct BGTask { + DataType type; + Operation operation; + std::vector argv; + + BGTask(const DataType& _type = DataType::kAll, const Operation& _opeation = Operation::kNone, + const std::vector& _argv = {}) + : type(_type), operation(_opeation), argv(_argv) {} +}; + +class Storage { + public: + Storage(); + ~Storage(); + + Status Open(const StorageOptions& storage_options, const std::string& db_path); + + Status Close(); + + std::vector> CreateCheckpoint(const std::string& checkpoint_path); + + Status CreateCheckpointInternal(const std::string& checkpoint_path, int db_index); + + std::vector> LoadCheckpoint(const std::string& checkpoint_path, const std::string& db_path); + + Status LoadCheckpointInternal(const std::string& dump_path, const std::string& db_path, int index); + + Status LoadCursorStartKey(const DataType& dtype, int64_t cursor, char* type, std::string* start_key); + + Status StoreCursorStartKey(const DataType& dtype, int64_t cursor, char type, const std::string& next_key); + + std::unique_ptr& GetDBInstance(const Slice& key); + + std::unique_ptr& GetDBInstance(const std::string& key); + + // Strings Commands + + // Set key to hold the string value. if key + // already holds a value, it is overwritten + Status Set(const Slice& key, const Slice& value); + + // Set key to hold the string value. if key exist + Status Setxx(const Slice& key, const Slice& value, int32_t* ret, uint64_t ttl = 0); + + // Get the value of key. If the key does not exist + // the special value nil is returned + Status Get(const Slice& key, std::string* value); + + // Get the value and ttl of key. If the key does not exist + // the special value nil is returned. If the key has no ttl, ttl is -1 + Status GetWithTTL(const Slice& key, std::string* value, uint64_t* ttl); + + // Atomically sets key to value and returns the old value stored at key + // Returns an error when key exists but does not hold a string value. + Status GetSet(const Slice& key, const Slice& value, std::string* old_value); + + // Sets or clears the bit at offset in the string value stored at key + Status SetBit(const Slice& key, int64_t offset, int32_t value, int32_t* ret); + + // Returns the bit value at offset in the string value stored at key + Status GetBit(const Slice& key, int64_t offset, int32_t* ret); + + // Sets the given keys to their respective values + // MSET replaces existing values with new values + Status MSet(const std::vector& kvs); + + // Returns the values of all specified keys. For every key + // that does not hold a string value or does not exist, the + // special value nil is returned + Status MGet(const std::vector& keys, std::vector* vss); + + // Returns the values of all specified keyswithTTL. For every key + // that does not hold a string value or does not exist, the + // special value nil is returned + Status MGetWithTTL(const std::vector& keys, std::vector* vss); + + // Set key to hold string value if key does not exist + // return 1 if the key was set + // return 0 if the key was not set + Status Setnx(const Slice& key, const Slice& value, int32_t* ret, uint64_t ttl = 0); + + // Sets the given keys to their respective values. + // MSETNX will not perform any operation at all even + // if just a single key already exists. + Status MSetnx(const std::vector& kvs, int32_t* ret); + + // Set key to hold string new_value if key currently hold the give value + // return 1 if the key currently hold the give value And override success + // return 0 if the key doesn't exist And override fail + // return -1 if the key currently does not hold the given value And override fail + Status Setvx(const Slice& key, const Slice& value, const Slice& new_value, int32_t* ret, uint64_t ttl = 0); + + // delete the key that holds a given value + // return 1 if the key currently hold the give value And delete success + // return 0 if the key doesn't exist And del fail + // return -1 if the key currently does not hold the given value And del fail + Status Delvx(const Slice& key, const Slice& value, int32_t* ret); + + // Set key to hold string value if key does not exist + // return the length of the string after it was modified by the command + Status Setrange(const Slice& key, int64_t start_offset, const Slice& value, int32_t* ret); + + // Returns the substring of the string value stored at key, + // determined by the offsets start and end (both are inclusive) + Status Getrange(const Slice& key, int64_t start_offset, int64_t end_offset, std::string* ret); + + Status GetrangeWithValue(const Slice& key, int64_t start_offset, int64_t end_offset, std::string* ret, + std::string* value, uint64_t* ttl); + + // If key already exists and is a string, this command appends the value at + // the end of the string + // return the length of the string after the append operation + Status Append(const Slice& key, const Slice& value, int32_t* ret); + + // Count the number of set bits (population counting) in a string. + // return the number of bits set to 1 + // note: if need to specified offset, set have_range to true + Status BitCount(const Slice& key, int64_t start_offset, int64_t end_offset, int32_t* ret, bool have_range); + + // Perform a bitwise operation between multiple keys + // and store the result in the destination key + Status BitOp(BitOpType op, const std::string& dest_key, const std::vector& src_keys, + std::string& value_to_dest, int64_t* ret); + + // Return the position of the first bit set to 1 or 0 in a string + // BitPos key 0 + Status BitPos(const Slice& key, int32_t bit, int64_t* ret); + // BitPos key 0 [start] + Status BitPos(const Slice& key, int32_t bit, int64_t start_offset, int64_t* ret); + // BitPos key 0 [start] [end] + Status BitPos(const Slice& key, int32_t bit, int64_t start_offset, int64_t end_offset, int64_t* ret); + + // Decrements the number stored at key by decrement + // return the value of key after the decrement + Status Decrby(const Slice& key, int64_t value, int64_t* ret); + + // Increments the number stored at key by increment. + // If the key does not exist, it is set to 0 before performing the operation + Status Incrby(const Slice& key, int64_t value, int64_t* ret); + + // Increment the string representing a floating point number + // stored at key by the specified increment. + Status Incrbyfloat(const Slice& key, const Slice& value, std::string* ret); + + // Set key to hold the string value and set key to timeout after a given + // number of seconds + Status Setex(const Slice& key, const Slice& value, uint64_t ttl); + + // Returns the length of the string value stored at key. An error + // is returned when key holds a non-string value. + Status Strlen(const Slice& key, int32_t* len); + + // PKSETEXAT has the same effect and semantic as SETEX, but instead of + // specifying the number of seconds representing the TTL (time to live), it + // takes an absolute Unix timestamp (seconds since January 1, 1970). A + // timestamp in the past will delete the key immediately. + Status PKSetexAt(const Slice& key, const Slice& value, uint64_t timestamp); + + // Hashes Commands + + // Sets field in the hash stored at key to value. If key does not exist, a new + // key holding a hash is created. If field already exists in the hash, it is + // overwritten. + Status HSet(const Slice& key, const Slice& field, const Slice& value, int32_t* res); + + // Returns the value associated with field in the hash stored at key. + // the value associated with field, or nil when field is not present in the + // hash or key does not exist. + Status HGet(const Slice& key, const Slice& field, std::string* value); + + // Sets the specified fields to their respective values in the hash stored at + // key. This command overwrites any specified fields already existing in the + // hash. If key does not exist, a new key holding a hash is created. + Status HMSet(const Slice& key, const std::vector& fvs); + + // Returns the values associated with the specified fields in the hash stored + // at key. + // For every field that does not exist in the hash, a nil value is returned. + // Because a non-existing keys are treated as empty hashes, running HMGET + // against a non-existing key will return a list of nil values. + Status HMGet(const Slice& key, const std::vector& fields, std::vector* vss); + + // Returns all fields and values of the hash stored at key. In the returned + // value, every field name is followed by its value, so the length of the + // reply is twice the size of the hash. + Status HGetall(const Slice& key, std::vector* fvs); + + Status HGetallWithTTL(const Slice& key, std::vector* fvs, uint64_t* ttl); + + // Returns all field names in the hash stored at key. + Status HKeys(const Slice& key, std::vector* fields); + + // Returns all values in the hash stored at key. + Status HVals(const Slice& key, std::vector* values); + + // Sets field in the hash stored at key to value, only if field does not yet + // exist. If key does not exist, a new key holding a hash is created. If field + // already exists, this operation has no effect. + Status HSetnx(const Slice& key, const Slice& field, const Slice& value, int32_t* ret); + + // Returns the number of fields contained in the hash stored at key. + // Return 0 when key does not exist. + Status HLen(const Slice& key, int32_t* ret); + + // Returns the string length of the value associated with field in the hash + // stored at key. If the key or the field do not exist, 0 is returned. + Status HStrlen(const Slice& key, const Slice& field, int32_t* len); + + // Returns if field is an existing field in the hash stored at key. + // Return Status::Ok() if the hash contains field. + // Return Status::NotFound() if the hash does not contain field, + // or key does not exist. + Status HExists(const Slice& key, const Slice& field); + + // Increments the number stored at field in the hash stored at key by + // increment. If key does not exist, a new key holding a hash is created. If + // field does not exist the value is set to 0 before the operation is + // performed. + Status HIncrby(const Slice& key, const Slice& field, int64_t value, int64_t* ret); + + // Increment the specified field of a hash stored at key, and representing a + // floating point number, by the specified increment. If the increment value + // is negative, the result is to have the hash field value decremented instead + // of incremented. If the field does not exist, it is set to 0 before + // performing the operation. An error is returned if one of the following + // conditions occur: + // + // The field contains a value of the wrong type (not a string). + // The current field content or the specified increment are not parsable as a + // double precision floating point number. + Status HIncrbyfloat(const Slice& key, const Slice& field, const Slice& by, std::string* new_value); + + // Removes the specified fields from the hash stored at key. Specified fields + // that do not exist within this hash are ignored. If key does not exist, it + // is treated as an empty hash and this command returns 0. + Status HDel(const Slice& key, const std::vector& fields, int32_t* ret); + + // See SCAN for HSCAN documentation. + Status HScan(const Slice& key, int64_t cursor, const std::string& pattern, int64_t count, + std::vector* field_values, int64_t* next_cursor); + + // Iterate over a Hash table of fields + // return next_field that the user need to use as the start_field argument + // in the next call + Status HScanx(const Slice& key, const std::string& start_field, const std::string& pattern, int64_t count, + std::vector* field_values, std::string* next_field); + + // Return random field(s) and value(s) from the hash value stored at key. + Status HRandField(const Slice& key, int64_t count, bool with_values, std::vector* res); + + // Iterate over a Hash table of fields by specified range + // return next_field that the user need to use as the start_field argument + // in the next call + Status PKHScanRange(const Slice& key, const Slice& field_start, const std::string& field_end, const Slice& pattern, + int32_t limit, std::vector* field_values, std::string* next_field); + + // part from the reversed ordering, PKHRSCANRANGE is similar to PKHScanRange + Status PKHRScanRange(const Slice& key, const Slice& field_start, const std::string& field_end, const Slice& pattern, + int32_t limit, std::vector* field_values, std::string* next_field); + + // Sets Commands + + // Add the specified members to the set stored at key. Specified members that + // are already a member of this set are ignored. If key does not exist, a new + // set is created before adding the specified members. + Status SAdd(const Slice& key, const std::vector& members, int32_t* ret); + + // Returns the set cardinality (number of elements) of the set stored at key. + Status SCard(const Slice& key, int32_t* ret); + + // Returns the members of the set resulting from the difference between the + // first set and all the successive sets. + // + // For example: + // key1 = {a, b, c, d} + // key2 = {c} + // key3 = {a, c, e} + // SDIFF key1 key2 key3 = {b, d} + Status SDiff(const std::vector& keys, std::vector* members); + + // This command is equal to SDIFF, but instead of returning the resulting set, + // it is stored in destination. + // If destination already exists, it is overwritten. + // + // For example: + // destination = {}; + // key1 = {a, b, c, d} + // key2 = {c} + // key3 = {a, c, e} + // SDIFFSTORE destination key1 key2 key3 + // destination = {b, d} + Status SDiffstore(const Slice& destination, const std::vector& keys, + std::vector& value_to_dest, int32_t* ret); + + // Returns the members of the set resulting from the intersection of all the + // given sets. + // + // For example: + // key1 = {a, b, c, d} + // key2 = {c} + // key3 = {a, c, e} + // SINTER key1 key2 key3 = {c} + Status SInter(const std::vector& keys, std::vector* members); + + // This command is equal to SINTER, but instead of returning the resulting + // set, it is stored in destination. + // If destination already exists, it is overwritten. + // + // For example: + // destination = {} + // key1 = {a, b, c, d} + // key2 = {a, c} + // key3 = {a, c, e} + // SINTERSTORE destination key1 key2 key3 + // destination = {a, c} + Status SInterstore(const Slice& destination, const std::vector& keys, + std::vector& value_to_dest, int32_t* ret); + + // Returns if member is a member of the set stored at key. + Status SIsmember(const Slice& key, const Slice& member, int32_t* ret); + + // Returns all the members of the set value stored at key. + // This has the same effect as running SINTER with one argument key. + Status SMembers(const Slice& key, std::vector* members); + + Status SMembersWithTTL(const Slice& key, std::vector* members, uint64_t* ttl); + + // Remove the specified members from the set stored at key. Specified members + // that are not a member of this set are ignored. If key does not exist, it is + // treated as an empty set and this command returns 0. + Status SRem(const Slice& key, const std::vector& members, int32_t* ret); + + // Removes and returns several random elements specified by count from the set value store at key. + Status SPop(const Slice& key, std::vector* members, int64_t count); + + // When called with just the key argument, return a random element from the + // set value stored at key. + // when called with the additional count argument, return an array of count + // distinct elements if count is positive. If called with a negative count the + // behavior changes and the command is allowed to return the same element + // multiple times. In this case the number of returned elements is the + // absolute value of the specified count + Status SRandmember(const Slice& key, int32_t count, std::vector* members); + + // Move member from the set at source to the set at destination. This + // operation is atomic. In every given moment the element will appear to be a + // member of source or destination for other clients. + // + // If the source set does not exist or does not contain the specified element, + // no operation is performed and 0 is returned. Otherwise, the element is + // removed from the source set and added to the destination set. When the + // specified element already exists in the destination set, it is only removed + // from the source set. + Status SMove(const Slice& source, const Slice& destination, const Slice& member, int32_t* ret); + + // Returns the members of the set resulting from the union of all the given + // sets. + // + // For example: + // key1 = {a, b, c, d} + // key2 = {c} + // key3 = {a, c, e} + // SUNION key1 key2 key3 = {a, b, c, d, e} + Status SUnion(const std::vector& keys, std::vector* members); + + // This command is equal to SUNION, but instead of returning the resulting + // set, it is stored in destination. + // If destination already exists, it is overwritten. + // + // For example: + // key1 = {a, b} + // key2 = {c, d} + // key3 = {c, d, e} + // SUNIONSTORE destination key1 key2 key3 + // destination = {a, b, c, d, e} + Status SUnionstore(const Slice& destination, const std::vector& keys, + std::vector& value_to_dest, int32_t* ret); + + // See SCAN for SSCAN documentation. + Status SScan(const Slice& key, int64_t cursor, const std::string& pattern, int64_t count, + std::vector* members, int64_t* next_cursor); + + // Lists Commands + + // Insert all the specified values at the head of the list stored at key. If + // key does not exist, it is created as empty list before performing the push + // operations. + Status LPush(const Slice& key, const std::vector& values, uint64_t* ret); + + // Insert all the specified values at the tail of the list stored at key. If + // key does not exist, it is created as empty list before performing the push + // operation. + Status RPush(const Slice& key, const std::vector& values, uint64_t* ret); + + // Returns the specified elements of the list stored at key. The offsets start + // and stop are zero-based indexes, with 0 being the first element of the list + // (the head of the list), 1 being the next element and so on. + Status LRange(const Slice& key, int64_t start, int64_t stop, std::vector* ret); + + Status LRangeWithTTL(const Slice& key, int64_t start, int64_t stop, std::vector* ret, uint64_t* ttl); + + // Removes the first count occurrences of elements equal to value from the + // list stored at key. The count argument influences the operation in the + // following ways + Status LTrim(const Slice& key, int64_t start, int64_t stop); + + // Returns the length of the list stored at key. If key does not exist, it is + // interpreted as an empty list and 0 is returned. An error is returned when + // the value stored at key is not a list. + Status LLen(const Slice& key, uint64_t* len); + + // Removes and returns the first elements of the list stored at key. + Status LPop(const Slice& key, int64_t count, std::vector* elements); + + // Removes and returns the last elements of the list stored at key. + Status RPop(const Slice& key, int64_t count, std::vector* elements); + + // Returns the element at index index in the list stored at key. The index is + // zero-based, so 0 means the first element, 1 the second element and so on. + // Negative indices can be used to designate elements starting at the tail of + // the list. Here, -1 means the last element, -2 means the penultimate and so + // forth. + Status LIndex(const Slice& key, int64_t index, std::string* element); + + // Inserts value in the list stored at key either before or after the + // reference value pivot. + // When key does not exist, it is considered an empty list and no operation is + // performed. + // An error is returned when key exists but does not hold a list value. + Status LInsert(const Slice& key, const BeforeOrAfter& before_or_after, const std::string& pivot, + const std::string& value, int64_t* ret); + + // Inserts value at the head of the list stored at key, only if key already + // exists and holds a list. In contrary to LPUSH, no operation will be + // performed when key does not yet exist. + Status LPushx(const Slice& key, const std::vector& values, uint64_t* len); + + // Inserts value at the tail of the list stored at key, only if key already + // exists and holds a list. In contrary to RPUSH, no operation will be + // performed when key does not yet exist. + Status RPushx(const Slice& key, const std::vector& values, uint64_t* len); + + // Removes the first count occurrences of elements equal to value from the + // list stored at key. The count argument influences the operation in the + // following ways: + // + // count > 0: Remove elements equal to value moving from head to tail. + // count < 0: Remove elements equal to value moving from tail to head. + // count = 0: Remove all elements equal to value. + // For example, LREM list -2 "hello" will remove the last two occurrences of + // "hello" in the list stored at list. + // + // Note that non-existing keys are treated like empty lists, so when key does + // not exist, the command will always return 0. + Status LRem(const Slice& key, int64_t count, const Slice& value, uint64_t* ret); + + // Sets the list element at index to value. For more information on the index + // argument, see LINDEX. + // + // An error is returned for out of range indexes. + Status LSet(const Slice& key, int64_t index, const Slice& value); + + // Atomically returns and removes the last element (tail) of the list stored + // at source, and pushes the element at the first element (head) of the list + // stored at destination. + // + // For example: consider source holding the list a,b,c, and destination + // holding the list x,y,z. Executing RPOPLPUSH results in source holding a,b + // and destination holding c,x,y,z. + // + // If source does not exist, the value nil is returned and no operation is + // performed. If source and destination are the same, the operation is + // equivalent to removing the last element from the list and pushing it as + // first element of the list, so it can be considered as a list rotation + // command. + Status RPoplpush(const Slice& source, const Slice& destination, std::string* element); + + // Zsets Commands + + // Pop the maximum count score_members which have greater score in the sorted set. + // And return the result in the score_members,If the total number of the sorted + // set less than count, it will pop out the total number of sorted set. If two + // ScoreMember's score were the same, the lexicographic predominant elements will + // be pop out. + Status ZPopMax(const Slice& key, int64_t count, std::vector* score_members); + + // Pop the minimum count score_members which have less score in the sorted set. + // And return the result in the score_members,If the total number of the sorted + // set less than count, it will pop out the total number of sorted set. If two + // ScoreMember's score were the same, the lexicographic predominant elements will + // not be pop out. + Status ZPopMin(const Slice& key, int64_t count, std::vector* score_members); + + // Adds all the specified members with the specified scores to the sorted set + // stored at key. It is possible to specify multiple score / member pairs. If + // a specified member is already a member of the sorted set, the score is + // updated and the element reinserted at the right position to ensure the + // correct ordering. + // + // If key does not exist, a new sorted set with the specified members as sole + // members is created, like if the sorted set was empty. If the key exists but + // does not hold a sorted set, an error is returned. + // The score values should be the string representation of a double precision + // floating point number. +inf and -inf values are valid values as well. + Status ZAdd(const Slice& key, const std::vector& score_members, int32_t* ret); + + // Returns the sorted set cardinality (number of elements) of the sorted set + // stored at key. + Status ZCard(const Slice& key, int32_t* ret); + + // Returns the number of elements in the sorted set at key with a score + // between min and max. + // + // The min and max arguments have the same semantic as described for + // ZRANGEBYSCORE. + // + // Note: the command has a complexity of just O(log(N)) because it uses + // elements ranks (see ZRANK) to get an idea of the range. Because of this + // there is no need to do a work proportional to the size of the range. + Status ZCount(const Slice& key, double min, double max, bool left_close, bool right_close, int32_t* ret); + + // Increments the score of member in the sorted set stored at key by + // increment. If member does not exist in the sorted set, it is added with + // increment as its score (as if its previous score was 0.0). If key does not + // exist, a new sorted set with the specified member as its sole member is + // created. + // + // An error is returned when key exists but does not hold a sorted set. + // + // The score value should be the string representation of a numeric value, and + // accepts double precision floating point numbers. It is possible to provide + // a negative value to decrement the score. + Status ZIncrby(const Slice& key, const Slice& member, double increment, double* ret); + + // Returns the specified range of elements in the sorted set stored at key. + // The elements are considered to be ordered from the lowest to the highest + // score. Lexicographical order is used for elements with equal score. + // + // See ZREVRANGE when you need the elements ordered from highest to lowest + // score (and descending lexicographical order for elements with equal score). + // + // Both start and stop are zero-based indexes, where 0 is the first element, 1 + // is the next element and so on. They can also be negative numbers indicating + // offsets from the end of the sorted set, with -1 being the last element of + // the sorted set, -2 the penultimate element and so on. + // + // start and stop are inclusive ranges, so for example ZRANGE myzset 0 1 will + // return both the first and the second element of the sorted set. + // + // Out of range indexes will not produce an error. If start is larger than the + // largest index in the sorted set, or start > stop, an empty list is + // returned. If stop is larger than the end of the sorted set Redis will treat + // it like it is the last element of the sorted set. + // + // It is possible to pass the WITHSCORES option in order to return the scores + // of the elements together with the elements. The returned list will contain + // value1,score1,...,valueN,scoreN instead of value1,...,valueN. Client + // libraries are free to return a more appropriate data type (suggestion: an + // array with (value, score) arrays/tuples). + Status ZRange(const Slice& key, int32_t start, int32_t stop, std::vector* score_members); + + Status ZRangeWithTTL(const Slice& key, int32_t start, int32_t stop, std::vector* score_members, + uint64_t* ttl); + + // Returns all the elements in the sorted set at key with a score between min + // and max (including elements with score equal to min or max). The elements + // are considered to be ordered from low to high scores. + // + // The elements having the same score are returned in lexicographical order + // (this follows from a property of the sorted set implementation in Redis and + // does not involve further computation). + // + // The optional LIMIT argument can be used to only get a range of the matching + // elements (similar to SELECT LIMIT offset, count in SQL). Keep in mind that + // if offset is large, the sorted set needs to be traversed for offset + // elements before getting to the elements to return, which can add up to O(N) + // time complexity. + // + // The optional WITHSCORES argument makes the command return both the element + // and its score, instead of the element alone. This option is available since + // Redis 2.0. + // + // Exclusive intervals and infinity + // min and max can be -inf and +inf, so that you are not required to know the + // highest or lowest score in the sorted set to get all elements from or up to + // a certain score. + // + // By default, the interval specified by min and max is closed (inclusive). It + // is possible to specify an open interval (exclusive) by prefixing the score + // with the character (. For example: + // + // ZRANGEBYSCORE zset (1 5 + // Will return all elements with 1 < score <= 5 while: + // + // ZRANGEBYSCORE zset (5 (10 + // Will return all the elements with 5 < score < 10 (5 and 10 excluded). + // + // Return value + // Array reply: list of elements in the specified score range (optionally with + // their scores). + Status ZRangebyscore(const Slice& key, double min, double max, bool left_close, bool right_close, + std::vector* score_members); + + // Returns all the elements in the sorted set at key with a score between min + // and max (including elements with score equal to min or max). The elements + // are considered to be ordered from low to high scores. + // + // The elements having the same score are returned in lexicographical order + // (this follows from a property of the sorted set implementation in Redis and + // does not involve further computation). + // + // The optional LIMIT argument can be used to only get a range of the matching + // elements (similar to SELECT LIMIT offset, count in SQL). Keep in mind that + // if offset is large, the sorted set needs to be traversed for offset + // elements before getting to the elements to return, which can add up to O(N) + // time complexity. + // + // The optional WITHSCORES argument makes the command return both the element + // and its score, instead of the element alone. This option is available since + // Redis 2.0. + // + // Exclusive intervals and infinity + // min and max can be -inf and +inf, so that you are not required to know the + // highest or lowest score in the sorted set to get all elements from or up to + // a certain score. + // + // By default, the interval specified by min and max is closed (inclusive). It + // is possible to specify an open interval (exclusive) by prefixing the score + // with the character (. For example: + // + // ZRANGEBYSCORE zset (1 5 + // Will return all elements with 1 < score <= 5 while: + // + // ZRANGEBYSCORE zset (5 (10 + // Will return all the elements with 5 < score < 10 (5 and 10 excluded). + // + // Return value + // Array reply: list of elements in the specified score range (optionally with + // their scores). + Status ZRangebyscore(const Slice& key, double min, double max, bool left_close, bool right_close, int64_t count, + int64_t offset, std::vector* score_members); + + // Returns the rank of member in the sorted set stored at key, with the scores + // ordered from low to high. The rank (or index) is 0-based, which means that + // the member with the lowest score has rank 0. + // + // Use ZREVRANK to get the rank of an element with the scores ordered from + // high to low. + Status ZRank(const Slice& key, const Slice& member, int32_t* rank); + + // Removes the specified members from the sorted set stored at key. Non + // existing members are ignored. + // + // An error is returned when key exists and does not hold a sorted set. + Status ZRem(const Slice& key, const std::vector& members, int32_t* ret); + + // Removes all elements in the sorted set stored at key with rank between + // start and stop. Both start and stop are 0 -based indexes with 0 being the + // element with the lowest score. These indexes can be negative numbers, where + // they indicate offsets starting at the element with the highest score. For + // example: -1 is the element with the highest score, -2 the element with the + // second highest score and so forth. + Status ZRemrangebyrank(const Slice& key, int32_t start, int32_t stop, int32_t* ret); + + // Removes all elements in the sorted set stored at key with a score between + // min and max (inclusive). + Status ZRemrangebyscore(const Slice& key, double min, double max, bool left_close, bool right_close, int32_t* ret); + + // Returns the specified range of elements in the sorted set stored at key. + // The elements are considered to be ordered from the highest to the lowest + // score. Descending lexicographical order is used for elements with equal + // score. + // + // Apart from the reversed ordering, ZREVRANGE is similar to ZRANGE. + Status ZRevrange(const Slice& key, int32_t start, int32_t stop, std::vector* score_members); + + // Returns all the elements in the sorted set at key with a score between max + // and min (including elements with score equal to max or min). In contrary to + // the default ordering of sorted sets, for this command the elements are + // considered to be ordered from high to low scores. + // + // The elements having the same score are returned in reverse lexicographical + // order. + // + // Apart from the reversed ordering, ZREVRANGEBYSCORE is similar to + // ZRANGEBYSCORE. + Status ZRevrangebyscore(const Slice& key, double min, double max, bool left_close, bool right_close, + std::vector* score_members); + + // Returns all the elements in the sorted set at key with a score between max + // and min (including elements with score equal to max or min). In contrary to + // the default ordering of sorted sets, for this command the elements are + // considered to be ordered from high to low scores. + // + // The elements having the same score are returned in reverse lexicographical + // order. + // + // Apart from the reversed ordering, ZREVRANGEBYSCORE is similar to + // ZRANGEBYSCORE. + Status ZRevrangebyscore(const Slice& key, double min, double max, bool left_close, bool right_close, int64_t count, + int64_t offset, std::vector* score_members); + + // Returns the rank of member in the sorted set stored at key, with the scores + // ordered from high to low. The rank (or index) is 0-based, which means that + // the member with the highest score has rank 0. + Status ZRevrank(const Slice& key, const Slice& member, int32_t* rank); + + // Returns the score of member in the sorted set at key. + // + // If member does not exist in the sorted set, or key does not exist, nil is + // returned. + Status ZScore(const Slice& key, const Slice& member, double* ret); + + // Computes the union of numkeys sorted sets given by the specified keys, and + // stores the result in destination. It is mandatory to provide the number of + // input keys (numkeys) before passing the input keys and the other (optional) + // arguments. + // + // By default, the resulting score of an element is the sum of its scores in + // the sorted sets where it exists. + // + // Using the WEIGHTS option, it is possible to specify a multiplication factor + // for each input sorted set. This means that the score of every element in + // every input sorted set is multiplied by this factor before being passed to + // the aggregation function. When WEIGHTS is not given, the multiplication + // factors default to 1. + // + // With the AGGREGATE option, it is possible to specify how the results of the + // union are aggregated. This option defaults to SUM, where the score of an + // element is summed across the inputs where it exists. When this option is + // set to either MIN or MAX, the resulting set will contain the minimum or + // maximum score of an element across the inputs where it exists. + // + // If destination already exists, it is overwritten. + Status ZUnionstore(const Slice& destination, const std::vector& keys, const std::vector& weights, + AGGREGATE agg, std::map& value_to_dest, int32_t* ret); + + // Computes the intersection of numkeys sorted sets given by the specified + // keys, and stores the result in destination. It is mandatory to provide the + // number of input keys (numkeys) before passing the input keys and the other + // (optional) arguments. + // + // By default, the resulting score of an element is the sum of its scores in + // the sorted sets where it exists. Because intersection requires an element + // to be a member of every given sorted set, this results in the score of + // every element in the resulting sorted set to be equal to the number of + // input sorted sets. + // + // For a description of the WEIGHTS and AGGREGATE options, see ZUNIONSTORE. + // + // If destination already exists, it is overwritten. + Status ZInterstore(const Slice& destination, const std::vector& keys, const std::vector& weights, + AGGREGATE agg, std::vector& value_to_dest, int32_t* ret); + + // When all the elements in a sorted set are inserted with the same score, in + // order to force lexicographical ordering, this command returns all the + // elements in the sorted set at key with a value between min and max. + // + // If the elements in the sorted set have different scores, the returned + // elements are unspecified. + // + // The elements are considered to be ordered from lower to higher strings as + // compared byte-by-byte using the memcmp() C function. Longer strings are + // considered greater than shorter strings if the common part is identical. + // + // The optional LIMIT argument can be used to only get a range of the matching + // elements (similar to SELECT LIMIT offset, count in SQL). Keep in mind that + // if offset is large, the sorted set needs to be traversed for offset + // elements before getting to the elements to return, which can add up to O(N) + // time complexity. + Status ZRangebylex(const Slice& key, const Slice& min, const Slice& max, bool left_close, bool right_close, + std::vector* members); + + // When all the elements in a sorted set are inserted with the same score, in + // order to force lexicographical ordering, this command returns the number of + // elements in the sorted set at key with a value between min and max. + // + // The min and max arguments have the same meaning as described for + // ZRANGEBYLEX. + // + // Note: the command has a complexity of just O(log(N)) because it uses + // elements ranks (see ZRANK) to get an idea of the range. Because of this + // there is no need to do a work proportional to the size of the range. + Status ZLexcount(const Slice& key, const Slice& min, const Slice& max, bool left_close, bool right_close, + int32_t* ret); + + // When all the elements in a sorted set are inserted with the same score, in + // order to force lexicographical ordering, this command removes all elements + // in the sorted set stored at key between the lexicographical range specified + // by min and max. + // + // The meaning of min and max are the same of the ZRANGEBYLEX command. + // Similarly, this command actually returns the same elements that ZRANGEBYLEX + // would return if called with the same min and max arguments. + Status ZRemrangebylex(const Slice& key, const Slice& min, const Slice& max, bool left_close, bool right_close, + int32_t* ret); + + // See SCAN for ZSCAN documentation. + Status ZScan(const Slice& key, int64_t cursor, const std::string& pattern, int64_t count, + std::vector* score_members, int64_t* next_cursor); + + // Keys Commands + + // Note: + // While any error happens, you need to check type_status for + // the error message + + // Set a timeout on key + // return -1 operation exception errors happen in database + // return >=0 success + int32_t Expire(const Slice& key, uint64_t ttl); + + // Removes the specified keys + // return -1 operation exception errors happen in database + // return >=0 the number of keys that were removed + int64_t Del(const std::vector& keys); + + // Removes the specified keys of the specified type + // return -1 operation exception errors happen in database + // return >= 0 the number of keys that were removed + int64_t DelByType(const std::vector& keys, const DataType& type); + + // Iterate over a collection of elements + // return an updated cursor that the user need to use as the cursor argument + // in the next call + int64_t Scan(const DataType& dtype, int64_t cursor, const std::string& pattern, int64_t count, + std::vector* keys); + + // Iterate over a collection of elements by specified range + // return a next_key that the user need to use as the key_start argument + // in the next call + Status PKScanRange(const DataType& data_type, const Slice& key_start, const Slice& key_end, const Slice& pattern, + int32_t limit, std::vector* keys, std::vector* kvs, std::string* next_key); + + // part from the reversed ordering, PKRSCANRANGE is similar to PKScanRange + Status PKRScanRange(const DataType& data_type, const Slice& key_start, const Slice& key_end, const Slice& pattern, + int32_t limit, std::vector* keys, std::vector* kvs, std::string* next_key); + + // Traverses the database of the specified type, removing the Key that matches + // the pattern + Status PKPatternMatchDel(const DataType& data_type, const std::string& pattern, int32_t* ret); + + // Iterate over a collection of elements + // return next_key that the user need to use as the start_key argument + // in the next call + Status Scanx(const DataType& data_type, const std::string& start_key, const std::string& pattern, int64_t count, + std::vector* keys, std::string* next_key); + + // Returns if key exists. + // return -1 operation exception errors happen in database + // return >=0 the number of keys existing + int64_t Exists(const std::vector& keys); + + // Return the key exists type count + // return param type_status: return every type status + int64_t IsExist(const Slice& key, std::map* type_status); + + // EXPIREAT has the same effect and semantic as EXPIRE, but instead of + // specifying the number of seconds representing the TTL (time to live), it + // takes an absolute Unix timestamp (seconds since January 1, 1970). A + // timestamp in the past will delete the key immediately. + // return -1 operation exception errors happen in database + // return 0 if key does not exist + // return >=1 if the timueout was set + int32_t Expireat(const Slice& key, uint64_t timestamp); + + // Remove the existing timeout on key, turning the key from volatile (a key + // with an expire set) to persistent (a key that will never expire as no + // timeout is associated). + // return -1 operation exception errors happen in database + // return 0 if key does not exist or does not have an associated timeout + // return >=1 if the timueout was set + int32_t Persist(const Slice& key, std::map* type_status); + + // Returns the remaining time to live of a key that has a timeout. + // return -3 operation exception errors happen in database + // return -2 if the key does not exist + // return -1 if the key exists but has not associated expire + // return > 0 TTL in seconds + std::map TTL(const Slice& key, std::map* type_status); + + // Reutrns the data all type of the key + // if single is true, the query will return the first one + Status GetType(const std::string& key, bool single, std::vector& types); + + // Reutrns the data all type of the key + Status Type(const std::string& key, std::vector& types); + + Status Keys(const DataType& data_type, const std::string& pattern, std::vector* keys); + + // Return OK if Rename successfully, + // NotFound if key doesn't exist, + // otherwise abort. + Status Rename(const std::string& key, const std::string& newkey); + + // Return OK if Renamenx successfully, + // NotFound if key doesn't exist, + // Corruption if newkey already exists, + // otherwise abort. + Status Renamenx(const std::string& key, const std::string& newkey); + + // Dynamic switch WAL + void DisableWal(const bool is_wal_disable); + + // Iterate through all the data in the database. + void ScanDatabase(const DataType& type); + + // HyperLogLog + enum { + kMaxKeys = 255, + kPrecision = 17, + }; + // Adds all the element arguments to the HyperLogLog data structure stored + // at the variable name specified as first argument. + Status PfAdd(const Slice& key, const std::vector& values, bool* update); + + // When called with a single key, returns the approximated cardinality + // computed by the HyperLogLog data structure stored at the specified + // variable, which is 0 if the variable does not exist. + Status PfCount(const std::vector& keys, int64_t* result); + + // Merge multiple HyperLogLog values into an unique value that will + // approximate the cardinality of the union of the observed Sets of the source + // HyperLogLog structures. + Status PfMerge(const std::vector& keys, std::string& value_to_dest); + + // Admin Commands + Status StartBGThread(); + Status RunBGTask(); + Status AddBGTask(const BGTask& bg_task); + + Status Compact(const DataType& type, bool sync = false); + Status CompactRange(const DataType& type, const std::string& start, const std::string& end, bool sync = false); + Status DoCompactRange(const DataType& type, const std::string& start, const std::string& end); + Status DoCompactSpecificKey(const DataType& type, const std::string& key); + + Status SetMaxCacheStatisticKeys(uint32_t max_cache_statistic_keys); + Status SetSmallCompactionThreshold(uint32_t small_compaction_threshold); + Status SetSmallCompactionDurationThreshold(uint32_t small_compaction_duration_threshold); + + std::string GetCurrentTaskType(); + Status GetUsage(const std::string& property, uint64_t* result); + Status GetUsage(const std::string& property, std::map* type_result); + uint64_t GetProperty(const std::string& property); + + Status GetKeyNum(std::vector* key_infos); + Status StopScanKeyNum(); + + rocksdb::DB* GetDBByIndex(int index); + + Status SetOptions(const OptionType& option_type, const std::unordered_map& options); + void GetRocksDBInfo(std::string& info); + Status OnBinlogWrite(const pikiwidb::Binlog& log, LogIndex log_idx); + + LogIndex GetSmallestFlushedLogIndex() const; + + private: + std::vector> insts_; + std::unique_ptr slot_indexer_; + std::atomic is_opened_ = false; + + std::unique_ptr> cursors_store_; + + // Storage start the background thread for compaction task + pthread_t bg_tasks_thread_id_ = 0; + pstd::Mutex bg_tasks_mutex_; + pstd::CondVar bg_tasks_cond_var_; + std::queue bg_tasks_queue_; + + std::atomic current_task_type_ = kNone; + std::atomic bg_tasks_should_exit_ = false; + + // For scan keys in data base + std::atomic scan_keynum_exit_ = false; + size_t db_instance_num_ = 3; + int db_id_ = 0; +}; + +} // namespace storage +#endif // INCLUDE_STORAGE_STORAGE_H_