Adds RocksDB support (#2197)

* Add RocksDB support

* Remove unused variable

* node.confirm_locked test is using the wrong store

* Fix Tests, ASAN issues and other improvements

* Delete column family after dropping it

* (Unrelated) Output to cerr if there is a problem reading genesis block

* Vacuum fails if the backup directory does not exist

* Copy WAL log files over when doing snapshot/vacuuming and open the database to explicitly flush them

* Don't atomic_flush yet since WAL are enabled by default

CMakeLists.txt

@@ -32,6 +32,9 @@ set (NANO_SECURE_RPC OFF CACHE BOOL "")
 set (NANO_ROCKSDB OFF CACHE BOOL "")
 set (NANO_WARN_TO_ERR OFF CACHE BOOL "")
 
+if (NANO_ROCKSDB)
+	add_definitions (-DNANO_ROCKSDB=1)
+endif ()
 option(NANO_ASAN_INT "Enable ASan+UBSan+Integer overflow" OFF)
 option(NANO_ASAN "Enable ASan+UBSan" OFF)
 option(NANO_TSAN "Enable TSan" OFF)

ci/build-travis.sh

@@ -27,6 +27,8 @@ else
     SANITIZERS=""
 fi
 
+ulimit -S -n 8192
+
 cmake \
     -G'Unix Makefiles' \
     -DACTIVE_NETWORK=nano_test_network \

ci/test.sh

@@ -1,7 +1,7 @@
 #!/usr/bin/env bash
 
 build_dir=${1-${PWD}}
-TIMEOUT_DEFAULT=180
+TIMEOUT_DEFAULT=360
 
 BUSYBOX_BASH=${BUSYBOX_BASH-0}
 

docker/ci/Dockerfile-clang

@@ -1,7 +1,12 @@
 FROM nanocurrency/nano-env:base
 
 RUN apt-get update -qq && apt-get install -yqq \
-    clang-3.9 lldb-3.9 librocksdb-dev
+    clang-3.9 lldb-3.9 git
+
+RUN git clone https://github.com/facebook/rocksdb.git && \
+    cd rocksdb && \
+    make static_lib && \
+    make install
 
 ENV CXX=/usr/bin/clang++
 ENV CC=/usr/bin/clang

docker/ci/Dockerfile-gcc

@@ -1,6 +1,11 @@
 FROM nanocurrency/nano-env:base
 
-RUN apt-get install -yqq librocksdb-dev
+RUN apt-get install -yqq git
+
+RUN git clone https://github.com/facebook/rocksdb.git && \
+    cd rocksdb && \
+    make static_lib && \
+    make install
 
 ENV BOOST_ROOT=/usr/local
 ADD util/build_prep/bootstrap_boost.sh bootstrap_boost.sh

nano/core_test/CMakeLists.txt

@@ -1,8 +1,3 @@
-if (NANO_ROCKSDB)
-  set (rocksdb_test rocksdb_test.cpp)
-  set (rocksdb_libs ${ROCKSDB_LIBRARIES} ${ZLIB_LIBRARIES})
-endif ()
-
 add_executable (core_test
 	core_test_main.cc
 	testutil.hpp
@@ -23,7 +18,6 @@ add_executable (core_test
 	memory_pool.cpp
 	processor_service.cpp
 	peer_container.cpp
-	${rocksdb_test}
 	signing.cpp
 	socket.cpp
 	toml.cpp
@@ -41,4 +35,4 @@ target_compile_definitions(core_test
 			-DTAG_VERSION_STRING=${TAG_VERSION_STRING}
 			-DGIT_COMMIT_HASH=${GIT_COMMIT_HASH}
 			-DBOOST_PROCESS_SUPPORTED=${BOOST_PROCESS_SUPPORTED})
-target_link_libraries (core_test node secure gtest libminiupnpc-static Boost::boost ${rocksdb_libs})
+target_link_libraries (core_test node secure gtest libminiupnpc-static Boost::boost)

nano/core_test/block_store.cpp

@@ -614,13 +614,15 @@ TEST (block_store, latest_find)
 	ASSERT_EQ (second, find3);
 }
 
+#if !NANO_ROCKSDB
 TEST (block_store, bad_path)
 {
 	nano::logger_mt logger;
 	bool init (false);
 	auto store = nano::make_store (init, logger, boost::filesystem::path ("///"));
 	ASSERT_TRUE (init);
 }
+#endif
 
 TEST (block_store, DISABLED_already_open) // File can be shared
 {
@@ -1849,14 +1851,18 @@ TEST (block_store, incompatible_version)
 
 		// Put version to an unreachable number so that it should always be incompatible
 		auto transaction (store->tx_begin_write ());
-		store->version_put (transaction, std::numeric_limits<unsigned>::max ());
+		store->version_put (transaction, std::numeric_limits<int>::max ());
 	}
 
 	// Now try and read it, should give an error
 	{
 		auto error (false);
-		auto store = nano::make_store (error, logger, path);
+		auto store = nano::make_store (error, logger, path, true);
 		ASSERT_TRUE (error);
+
+		auto transaction = store->tx_begin_read ();
+		auto version_l = store->version_get (transaction);
+		ASSERT_EQ (version_l, std::numeric_limits<int>::max ());
 	}
 }
 
@@ -1901,7 +1907,7 @@ void modify_account_info_to_v13 (nano::mdb_store & store, nano::transaction cons
 	nano::account_info info;
 	ASSERT_FALSE (store.account_get (transaction_a, account, info));
 	nano::account_info_v13 account_info_v13 (info.head, info.rep_block, info.open_block, info.balance, info.modified, info.block_count, info.epoch);
-	auto status (mdb_put (store.env.tx (transaction_a), store.get_account_db (info.epoch) == nano::block_store_partial<MDB_val, nano::mdb_store>::tables::accounts_v0 ? store.accounts_v0 : store.accounts_v1, nano::mdb_val (account), nano::mdb_val (account_info_v13), 0));
+	auto status (mdb_put (store.env.tx (transaction_a), store.get_account_db (info.epoch) == nano::tables::accounts_v0 ? store.accounts_v0 : store.accounts_v1, nano::mdb_val (account), nano::mdb_val (account_info_v13), 0));
 	(void)status;
 	assert (status == 0);
 }
@@ -1911,7 +1917,7 @@ void modify_account_info_to_v14 (nano::mdb_store & store, nano::transaction cons
 	nano::account_info info;
 	ASSERT_FALSE (store.account_get (transaction_a, account, info));
 	nano::account_info_v14 account_info_v14 (info.head, info.rep_block, info.open_block, info.balance, info.modified, info.block_count, confirmation_height, info.epoch);
-	auto status (mdb_put (store.env.tx (transaction_a), store.get_account_db (info.epoch) == nano::block_store_partial<MDB_val, nano::mdb_store>::tables::accounts_v0 ? store.accounts_v0 : store.accounts_v1, nano::mdb_val (account), nano::mdb_val (account_info_v14), 0));
+	auto status (mdb_put (store.env.tx (transaction_a), store.get_account_db (info.epoch) == nano::tables::accounts_v0 ? store.accounts_v0 : store.accounts_v1, nano::mdb_val (account), nano::mdb_val (account_info_v14), 0));
 	(void)status;
 	assert (status == 0);
 }

nano/core_test/ledger.cpp

@@ -8,16 +8,18 @@
 
 using namespace std::chrono_literals;
 
+#if !NANO_ROCKSDB
 // Init returns an error if it can't open files at the path
 TEST (ledger, store_error)
 {
 	nano::logger_mt logger;
 	bool init (false);
 	auto store = nano::make_store (init, logger, boost::filesystem::path ("///"));
-	ASSERT_FALSE (!init);
+	ASSERT_TRUE (init);
 	nano::stat stats;
 	nano::ledger ledger (*store, stats);
 }
+#endif
 
 // Ledger can be initialized and returns a basic query for an empty account
 TEST (ledger, empty)

nano/core_test/network.cpp

@@ -1544,7 +1544,7 @@ TEST (confirmation_height, single)
 			ASSERT_NO_ERROR (system.poll ());
 		}
 
-		auto transaction = node->store.tx_begin_read ();
+		auto transaction = node->store.tx_begin_write ();
 		ASSERT_FALSE (node->store.confirmation_height_get (transaction, nano::test_genesis_key.pub, confirmation_height));
 		ASSERT_EQ (2, confirmation_height);
 

nano/core_test/node.cpp

@@ -242,6 +242,7 @@ TEST (node, auto_bootstrap)
 	node1->network.send_keepalive (channel);
 	node1->start ();
 	system.nodes.push_back (node1);
+	system.deadline_set (10s);
 	while (!node1->bootstrap_initiator.in_progress ())
 	{
 		ASSERT_NO_ERROR (system.poll ());
@@ -497,7 +498,7 @@ TEST (node, confirm_locked)
 {
 	nano::system system (24000, 1);
 	system.wallet (0)->insert_adhoc (nano::test_genesis_key.prv);
-	auto transaction (system.nodes[0]->store.tx_begin_read ());
+	auto transaction (system.wallet (0)->wallets.tx_begin_read ());
 	system.wallet (0)->enter_password (transaction, "1");
 	auto block (std::make_shared<nano::send_block> (0, 0, 0, nano::keypair ().prv, 0, 0));
 	system.nodes[0]->network.flood_block (block);
@@ -3059,15 +3060,15 @@ TEST (node, dont_write_lock_node)
 	std::thread ([&path, &write_lock_held_promise, &finished_promise]() {
 		nano::logger_mt logger;
 		bool init (false);
-		nano::mdb_store store (init, logger, path / "data.ldb");
+		auto store = nano::make_store (init, logger, path, false, true);
 		nano::genesis genesis;
 		{
-			auto transaction (store.tx_begin_write ());
-			store.initialize (transaction, genesis);
+			auto transaction (store->tx_begin_write ());
+			store->initialize (transaction, genesis);
 		}
 
 		// Hold write lock open until main thread is done needing it
-		auto transaction (store.tx_begin_write ());
+		auto transaction (store->tx_begin_write ());
 		write_lock_held_promise.set_value ();
 		finished_promise.get_future ().wait ();
 	})

nano/core_test/socket.cpp

@@ -10,7 +10,7 @@ using namespace std::chrono_literals;
 
 TEST (socket, concurrent_writes)
 {
-	nano::inactive_node inactivenode;
+	nano::inactive_node inactivenode (nano::working_path (), 24000, false);
 	auto node = inactivenode.node;
 
 	// This gives more realistic execution than using system#poll, allowing writes to

nano/core_test/utility.cpp

@@ -32,3 +32,53 @@ TEST (thread, worker)
 	}
 	ASSERT_TRUE (passed_sleep);
 }
+
+TEST (filesystem, remove_all_files)
+{
+	auto path = nano::unique_path ();
+	auto dummy_directory = path / "tmp";
+	boost::filesystem::create_directories (dummy_directory);
+
+	auto dummy_file1 = path / "my_file1.txt";
+	auto dummy_file2 = path / "my_file2.txt";
+	std::ofstream (dummy_file1.string ());
+	std::ofstream (dummy_file2.string ());
+
+	// Check all exist
+	ASSERT_TRUE (boost::filesystem::exists (dummy_directory));
+	ASSERT_TRUE (boost::filesystem::exists (dummy_file1));
+	ASSERT_TRUE (boost::filesystem::exists (dummy_file2));
+
+	// Should remove only the files
+	nano::remove_all_files_in_dir (path);
+
+	ASSERT_TRUE (boost::filesystem::exists (dummy_directory));
+	ASSERT_FALSE (boost::filesystem::exists (dummy_file1));
+	ASSERT_FALSE (boost::filesystem::exists (dummy_file2));
+}
+
+TEST (filesystem, move_all_files)
+{
+	auto path = nano::unique_path ();
+	auto dummy_directory = path / "tmp";
+	boost::filesystem::create_directories (dummy_directory);
+
+	auto dummy_file1 = dummy_directory / "my_file1.txt";
+	auto dummy_file2 = dummy_directory / "my_file2.txt";
+	std::ofstream (dummy_file1.string ());
+	std::ofstream (dummy_file2.string ());
+
+	// Check all exist
+	ASSERT_TRUE (boost::filesystem::exists (dummy_directory));
+	ASSERT_TRUE (boost::filesystem::exists (dummy_file1));
+	ASSERT_TRUE (boost::filesystem::exists (dummy_file2));
+
+	// Should move only the files
+	nano::move_all_files_to_dir (dummy_directory, path);
+
+	ASSERT_TRUE (boost::filesystem::exists (dummy_directory));
+	ASSERT_TRUE (boost::filesystem::exists (path / "my_file1.txt"));
+	ASSERT_TRUE (boost::filesystem::exists (path / "my_file2.txt"));
+	ASSERT_FALSE (boost::filesystem::exists (dummy_file1));
+	ASSERT_FALSE (boost::filesystem::exists (dummy_file2));
+}

nano/core_test/wallets.cpp

@@ -78,6 +78,7 @@ TEST (wallets, remove)
 	}
 }
 
+#if !NANO_ROCKSDB
 TEST (wallets, upgrade)
 {
 	nano::system system (24000, 1);
@@ -101,7 +102,7 @@ TEST (wallets, upgrade)
 		nano::account_info info;
 		ASSERT_FALSE (mdb_store.account_get (transaction_destination, nano::genesis_account, info));
 		nano::account_info_v13 account_info_v13 (info.head, info.rep_block, info.open_block, info.balance, info.modified, info.block_count, info.epoch);
-		auto status (mdb_put (mdb_store.env.tx (transaction_destination), mdb_store.get_account_db (info.epoch) == nano::block_store_partial<MDB_val, nano::mdb_store>::tables::accounts_v0 ? mdb_store.accounts_v0 : mdb_store.accounts_v1, nano::mdb_val (nano::test_genesis_key.pub), nano::mdb_val (account_info_v13), 0));
+		auto status (mdb_put (mdb_store.env.tx (transaction_destination), mdb_store.get_account_db (info.epoch) == nano::tables::accounts_v0 ? mdb_store.accounts_v0 : mdb_store.accounts_v1, nano::mdb_val (nano::test_genesis_key.pub), nano::mdb_val (account_info_v13), 0));
 		(void)status;
 		assert (status == 0);
 	}
@@ -118,6 +119,7 @@ TEST (wallets, upgrade)
 	MDB_dbi new_handle;
 	ASSERT_EQ (0, mdb_dbi_open (tx_new, id.pub.to_string ().c_str (), 0, &new_handle));
 }
+#endif
 
 // Keeps breaking whenever we add new DBs
 TEST (wallets, DISABLED_wallet_create_max)

nano/lib/CMakeLists.txt

@@ -21,6 +21,8 @@ add_library (nano_lib
 	config.hpp
 	config.cpp
 	configbase.hpp
+	diagnosticsconfig.hpp
+	diagnosticsconfig.cpp
 	errors.hpp
 	errors.cpp
 	ipc.hpp

nano/node/diagnosticsconfig.cpp → nano/lib/diagnosticsconfig.cpp

@@ -1,6 +1,6 @@
+#include <nano/lib/diagnosticsconfig.hpp>
 #include <nano/lib/jsonconfig.hpp>
 #include <nano/lib/tomlconfig.hpp>
-#include <nano/node/diagnosticsconfig.hpp>
 
 nano::error nano::diagnostics_config::serialize_json (nano::jsonconfig & json) const
 {

nano/lib/utility.cpp

@@ -298,6 +298,30 @@ std::unique_ptr<nano::seq_con_info_component> nano::collect_seq_con_info (nano::
 	return composite;
 }
 
+void nano::remove_all_files_in_dir (boost::filesystem::path const & dir)
+{
+	for (auto & p : boost::filesystem::directory_iterator (dir))
+	{
+		auto path = p.path ();
+		if (boost::filesystem::is_regular_file (path))
+		{
+			boost::filesystem::remove (path);
+		}
+	}
+}
+
+void nano::move_all_files_to_dir (boost::filesystem::path const & from, boost::filesystem::path const & to)
+{
+	for (auto & p : boost::filesystem::directory_iterator (from))
+	{
+		auto path = p.path ();
+		if (boost::filesystem::is_regular_file (path))
+		{
+			boost::filesystem::rename (path, to / path.filename ());
+		}
+	}
+}
+
 /*
  * Backing code for "release_assert", which is itself a macro
  */

nano/lib/utility.hpp

@@ -217,6 +217,9 @@ inline std::unique_ptr<seq_con_info_component> collect_seq_con_info (observer_se
 	composite->add_component (std::make_unique<seq_con_info_leaf> (seq_con_info{ "observers", count, sizeof_element }));
 	return composite;
 }
+
+void remove_all_files_in_dir (boost::filesystem::path const & dir);
+void move_all_files_to_dir (boost::filesystem::path const & from, boost::filesystem::path const & to);
 }
 
 void release_assert_internal (bool check, const char * check_expr, const char * file, unsigned int line);