KEP-596 Replace existing storage in memory database with RocksDB

bluzelle · Sep 25, 2018 · 91f9ab7 · 91f9ab7
1 parent ea7382b
commit 91f9ab7
Show file tree

Hide file tree

Showing 24 changed files with 414 additions and 275 deletions.
diff --git a/.travis.yml b/.travis.yml
@@ -41,7 +41,7 @@ matrix:
     - os: osx
       osx_image: xcode10
       env:
-        - MATRIX_EVAL="brew update && brew unlink python && brew install protobuf && brew install snappy && brew install openssl && brew install cmake"
+        - MATRIX_EVAL="brew update && brew unlink python && brew install protobuf && brew install snappy && brew install openssl && brew install cmake && brew install lz4"
         - CMAKE_URL="https://cmake.org/files/v3.12/cmake-3.12.1-Darwin-x86_64.tar.gz"
         - CMAKE_COMMAND="cmake -DOPENSSL_ROOT_DIR=/usr/local/opt/openssl"
 

diff --git a/cmake/rocksdb.cmake b/cmake/rocksdb.cmake
@@ -86,3 +86,7 @@ ExternalProject_Get_Property(rocksdb binary_dir)
 link_directories(${binary_dir}/)
 
 set(ROCKSDB_LIBRARIES ${binary_dir}/librocksdb.a ${SNAPPY_LIBRARIES} ${ZLIB_LIBRARIES} ${BZIP2_LIBRARIES})
+
+if (APPLE)
+    list(APPEND ROCKSDB_LIBRARIES "lz4")
+endif()
diff --git a/crud/crud.cpp b/crud/crud.cpp
@@ -14,7 +14,7 @@
 
 #include <crud/crud.hpp>
 #include <numeric>
-#include <storage/storage.hpp>
+#include <storage/mem_storage.hpp>
 
 #include <boost/beast/core/detail/base64.hpp>
 
@@ -154,7 +154,7 @@ crud::handle_read(const bzn::message& /*msg*/, const database_msg& request, data
     if (auto record = this->storage->read(request.header().db_uuid(), request.read().key()); record)
     {
         response.mutable_read()->set_key(request.read().key());
-        response.mutable_read()->set_value(record->value);
+        response.mutable_read()->set_value(*record);
         return;
     }
 

diff --git a/crud/test/crud_test.cpp b/crud/test/crud_test.cpp
@@ -315,11 +315,7 @@ TEST_F(crud_test, test_that_a_follower_can_read_an_existing_record)
     EXPECT_CALL(*this->mock_storage, read(USER_UUID, "key0")).WillOnce(Invoke(
         [](const bzn::uuid_t& /*uuid*/, const std::string& /*key*/)
         {
-            std::shared_ptr<bzn::storage_base::record> record = std::make_shared<bzn::storage_base::record>();
-            record->value = "skdif9ek34587fk30df6vm73==";
-            record->timestamp = std::chrono::seconds(0);
-            record->transaction_id = TEST_NODE_UUID;
-            return record;
+            return std::optional<bzn::value_t>("skdif9ek34587fk30df6vm73==");
         }));
 
     EXPECT_CALL(*this->mock_session, send_message(An<std::shared_ptr<std::string>>(),_)).WillOnce(Invoke(
@@ -341,7 +337,7 @@ TEST_F(crud_test, test_that_a_follower_apon_failing_to_read_suggests_leader)
 
     EXPECT_CALL(*this->mock_raft, get_state()).WillRepeatedly(Return(bzn::raft_state ::follower));
 
-    EXPECT_CALL(*this->mock_storage, read(USER_UUID, "key0")).WillOnce(Return(nullptr));
+    EXPECT_CALL(*this->mock_storage, read(USER_UUID, "key0")).WillOnce(Return(std::nullopt));
 
     EXPECT_CALL(*this->mock_raft, get_leader()).WillRepeatedly(Return(bzn::peer_address_t("127.0.0.1",49152,8080,"ozzy",LEADER_UUID)));
 
@@ -393,11 +389,7 @@ TEST_F(crud_test, test_that_a_leader_can_read_existing_record)
     EXPECT_CALL(*this->mock_storage, read(USER_UUID, "key0")).WillOnce(Invoke(
         [](const bzn::uuid_t& /*uuid*/, const std::string& /*key*/)
         {
-            auto record = std::make_shared<bzn::storage_base::record>();
-            record->value = "skdif9ek34587fk30df6vm73==";
-            record->timestamp = std::chrono::seconds(0);
-            record->transaction_id = TEST_NODE_UUID;
-            return record;
+            return std::optional<bzn::value_t>("skdif9ek34587fk30df6vm73==");
         }));
 
     EXPECT_CALL(*this->mock_session, send_message(An<std::shared_ptr<std::string>>(),_)).WillOnce(Invoke(

diff --git a/include/bluzelle.hpp b/include/bluzelle.hpp
@@ -28,6 +28,8 @@ namespace bzn
 
     using key_t = std::string;
 
+    using value_t = std::string;
+
     using session_id = uint64_t;
 
 } // bzn

diff --git a/mocks/mock_storage_base.hpp b/mocks/mock_storage_base.hpp
@@ -24,7 +24,7 @@ namespace bzn {
         MOCK_METHOD3(create,
                      storage_base::result(const bzn::uuid_t& uuid, const std::string& key, const std::string& value));
         MOCK_METHOD2(read,
-                     std::shared_ptr<bzn::storage_base::record>(const bzn::uuid_t& uuid, const std::string& key));
+                     std::optional<bzn::value_t> (const bzn::uuid_t& uuid, const std::string& key));
         MOCK_METHOD3(update,
                      storage_base::result(const bzn::uuid_t& uuid, const std::string& key, const std::string& value));
         MOCK_METHOD2(remove,

diff --git a/options/options.cpp b/options/options.cpp
@@ -17,6 +17,7 @@
 #include <regex>
 #include <cstdint>
 #include <utils/crypto.hpp>
+#include <chrono>
 
 using namespace bzn;
 using namespace bzn::option_names;
@@ -183,6 +184,13 @@ options::get_max_storage() const
 }
 
 
+bool
+options::get_mem_storage() const
+{
+    return this->raw_opts.get<bool>(MEM_STORAGE);
+}
+
+
 size_t
 options::get_logfile_rotation_size() const
 {

diff --git a/options/options.hpp b/options/options.hpp
@@ -55,6 +55,8 @@ namespace bzn
 
         size_t get_max_storage() const override;
 
+        bool get_mem_storage() const override;
+
         size_t get_logfile_rotation_size() const override ;
 
         size_t get_logfile_max_size() const override;

diff --git a/options/options_base.hpp b/options/options_base.hpp
@@ -22,6 +22,7 @@
 #include <string>
 #include <map>
 #include <optional>
+#include <chrono>
 
 
 namespace bzn
@@ -136,6 +137,13 @@ namespace bzn
         virtual size_t get_max_storage() const = 0;
 
 
+        /**
+         * Database to use
+         * @return true if we are using in memory data
+         */
+        virtual bool get_mem_storage() const = 0;
+
+
         /**
          * Get the size of a log file to rotate
          * @return size

diff --git a/options/simple_options.cpp b/options/simple_options.cpp
@@ -77,6 +77,9 @@ simple_options::build_options()
                 (MAX_STORAGE.c_str(),
                          po::value<std::string>()->default_value("2G"),
                         "maximum db storage on this node (bytes)")
+                (MEM_STORAGE.c_str(),
+                         po::value<bool>()->default_value(true),
+                         "enable in memory storage for debugging")
                 (NODE_UUID.c_str(),
                         po::value<std::string>(),
                         "uuid of this node")

diff --git a/options/simple_options.hpp b/options/simple_options.hpp
@@ -34,6 +34,7 @@ namespace bzn::option_names
     const std::string LOGFILE_MAX_SIZE = "logfile_max_size";
     const std::string LOGFILE_ROTATION_SIZE = "logfile_rotation_size";
     const std::string MAX_STORAGE = "max_storage";
+    const std::string MEM_STORAGE = "mem_storage";
     const std::string MONITOR_ADDRESS = "monitor_address";
     const std::string MONITOR_PORT = "monitor_port";
     const std::string NODE_UUID = "uuid";

diff --git a/options/test/options_test.cpp b/options/test/options_test.cpp
@@ -43,7 +43,8 @@ namespace
         "  \"logfile_max_size\" : \"1M\","
         "  \"logfile_rotation_size\" : \"2M\","
         "  \"logfile_dir\" : \".\","
-        "  \"http_port\" : 80";
+        "  \"http_port\" : 80,"
+        "  \"mem_storage\" : false";
 
     const std::string DEFAULT_CONFIG_DATA = "{" + DEFAULT_CONFIG_CONTENT + "}";
 
@@ -104,7 +105,7 @@ class options_file_test : public Test
             this->open_files.insert(filename);
         }
 
-        std::cout << filename;
+        //std::cout << filename;
 
         std::ofstream ofile(filename.c_str());
         ofile.exceptions(std::ios::failbit);
@@ -147,6 +148,7 @@ TEST_F(options_file_test, test_that_loading_of_default_config_file)
     EXPECT_EQ(".", options.get_logfile_dir());
     EXPECT_EQ(uint16_t(80), options.get_http_port());
     EXPECT_FALSE(options.peer_validation_enabled());
+    EXPECT_FALSE(options.get_mem_storage());
 
     // defaults..
     {
@@ -161,6 +163,7 @@ TEST_F(options_file_test, test_that_loading_of_default_config_file)
         EXPECT_EQ(size_t(65536), options.get_logfile_rotation_size());
         EXPECT_EQ("logs/", options.get_logfile_dir());
         EXPECT_EQ(uint16_t(8080), options.get_http_port());
+        EXPECT_TRUE(options.get_mem_storage());
     }
 }
 

diff --git a/raft/raft.hpp b/raft/raft.hpp
@@ -20,7 +20,8 @@
 #include <raft/raft_base.hpp>
 #include <raft/log_entry.hpp>
 #include <raft/raft_log.hpp>
-#include <storage/storage.hpp>
+#include <storage/mem_storage.hpp>
+#include <node/node_base.hpp>
 #include <gtest/gtest_prod.h>
 #include <fstream>
 #include <optional>
@@ -81,7 +82,6 @@ namespace bzn
         void set_audit_enabled(bool val);
 
     private:
-        friend class raft_log_base;
         friend class raft_log;
         FRIEND_TEST(raft, test_raft_timeout_scale_can_get_set);
         FRIEND_TEST(raft, test_that_raft_can_rehydrate_state_and_log_entries);

diff --git a/raft/test/raft_test.cpp b/raft/test/raft_test.cpp
@@ -18,7 +18,7 @@
 #include <raft/raft.hpp>
 #include <raft/log_entry.hpp>
 #include <raft/raft_log.hpp>
-#include <storage/storage.hpp>
+#include <storage/mem_storage.hpp>
 #include <boost/filesystem.hpp>
 #include <algorithm>
 #include <vector>
@@ -931,7 +931,7 @@ namespace bzn
         }
 
         // We've got a number of entries in the log, now we can load them into storage.
-        auto storage = std::make_shared<bzn::storage>();
+        auto storage = std::make_shared<bzn::mem_storage>();
         raft->initialize_storage_from_log(storage);
 
         // lets make sure we got them all
@@ -950,10 +950,10 @@ namespace bzn
         {
             if (std::find(updated_keys.begin(), updated_keys.end(), key) == updated_keys.end())
             {
-                EXPECT_EQ(I_AM_CREATED, storage->read(db_uuid, key)->value);
+                EXPECT_EQ(I_AM_CREATED, *storage->read(db_uuid, key));
                 continue;
             }
-            EXPECT_EQ(I_AM_UPDATED, storage->read(db_uuid, key)->value);
+            EXPECT_EQ(I_AM_UPDATED, *storage->read(db_uuid, key));
         }
     }
 

diff --git a/storage/CMakeLists.txt b/storage/CMakeLists.txt
@@ -1,11 +1,12 @@
 add_library(storage STATIC
-        storage.cpp
-        storage.hpp
-        storage_base.hpp
-        )
+    mem_storage.cpp
+    mem_storage.hpp
+    storage_base.hpp
+    rocksdb_storage.hpp
+    rocksdb_storage.cpp)
 
 target_link_libraries(storage)
 add_dependencies(storage jsoncpp)
-target_include_directories(storage PRIVATE ${JSONCPP_INCLUDE_DIRS})
+target_include_directories(storage PRIVATE ${JSONCPP_INCLUDE_DIRS} ${ROCKSDB_INCLUDE_DIRS})
 
 add_subdirectory(test)