Create column families when opening the DB

BlueBrain · Feb 17, 2020 · 1fcedd9 · 1fcedd9
1 parent dbf1dc0
commit 1fcedd9
Show file tree

Hide file tree

Showing 4 changed files with 16 additions and 43 deletions.
diff --git a/src/basalt/config.cpp b/src/basalt/config.cpp
@@ -295,40 +295,6 @@ static rocksdb::ColumnFamilyOptions column_families_options(
     return result;
 }
 
-/**
- * Prepare the rocksdb column families on the filesystem if they do not already exist.
- */
-static void create_columns_families(const nlohmann::json& config,
-                                    const std::string& db_path,
-                                    rocksdb::Options& options,
-                                    std::shared_ptr<rocksdb::Cache> global_block_cache) {
-    std::unique_ptr<rocksdb::DB> db_ptr;
-    {  // open db
-        rocksdb::DB* db;
-        rocksdb::Status status = rocksdb::DB::Open(options, db_path, &db);
-        if (status.code() == rocksdb::Status::kInvalidArgument) {
-            return;
-        }
-        to_status(status).raise_on_error();
-        db_ptr.reset(db);
-    }
-
-    std::vector<std::string> column_family_names;
-    rocksdb::DB::ListColumnFamilies(options, db_ptr->GetName(), &column_family_names);
-
-    for (auto const& cf_config: config) {
-        auto const& name = column_family_name(cf_config["name"]);
-        if (std::find(column_family_names.begin(), column_family_names.end(), name) ==
-            column_family_names.end()) {
-            gsl::owner<rocksdb::ColumnFamilyHandle*> cf = nullptr;
-            auto const& cf_options = column_families_options(cf_config["config"],
-                                                             global_block_cache);
-            to_status(db_ptr->CreateColumnFamily(cf_options, name, &cf)).raise_on_error();
-            delete cf;
-        }
-    }
-}
-
 /**
  * Setup rocksdb statistics according to JSON config
  */
@@ -352,10 +318,20 @@ static void setup_max_open_files(const nlohmann::json& config, rocksdb::Options&
 }
 
 static void setup_create_if_missing(const nlohmann::json& config, rocksdb::Options& options) {
-    auto const& value = config.find("create_if_missing");
+  {
+    auto const &value = config.find("create_if_missing");
+    if (value != config.end()) {
+      options.create_if_missing =
+          value.value().get<decltype(options.create_if_missing)>();
+    }
+  }
+  {
+    auto const &value = config.find("create_missing_column_families");
     if (value != config.end()) {
-        options.create_if_missing = value.value().get<decltype(options.create_if_missing)>();
+      options.create_missing_column_families =
+          value.value().get<decltype(options.create_missing_column_families)>();
     }
+  }
 }
 
 static void setup_compression(const nlohmann::json& config, rocksdb::Options& options) {
@@ -375,6 +351,7 @@ static nlohmann::json default_json() {
     config["statistics"] = true;
     config["max_open_files"] = -1;
     config["create_if_missing"] = true;
+    config["create_missing_column_families"] = true;
     // clang-format off
     config["block_cache"] = {
         {"type", "lru"},
@@ -483,15 +460,14 @@ Config::Config()
 Config::Config(std::ifstream& istr)
     : config_(from_stream(istr)) {}
 
-void Config::configure(rocksdb::Options& options, const std::string& db_path) const {
+void Config::configure(rocksdb::Options& options) const {
     setup_statistics(config_, options);
     setup_max_open_files(config_, options);
     setup_create_if_missing(config_, options);
     setup_compression(config_, options);
 
     std::shared_ptr<rocksdb::Cache> empty;
     auto global_block_cache = block_cache_if_present(config_, empty);
-    create_columns_families(config_["column_families"], db_path, options, global_block_cache);
 }
 
 std::ostream& Config::write(std::ostream& ostr, std::streamsize indent) const {

diff --git a/src/basalt/config.hpp b/src/basalt/config.hpp
@@ -48,10 +48,9 @@ class Config {
 
     /**
      * Setup RocksDB configuration option
-     * \param options RocksDB configuration to update
      * \param db_path path to database
      */
-    void configure(rocksdb::Options& options, const std::string& db_path) const;
+    void configure(rocksdb::Options& options) const;
 
     /**
      * Write configuration to an output stream in JSON format

diff --git a/src/basalt/graph_impl.cpp b/src/basalt/graph_impl.cpp
@@ -71,7 +71,7 @@ GraphImpl<Orientation>::GraphImpl(const std::string& path, Config config, bool t
 
     rocksdb::DB* db;
 
-    this->config_.configure(*options_, path);
+    this->config_.configure(*options_);
     const auto& column_families = this->config_.column_families();
     std::vector<rocksdb::ColumnFamilyHandle*> handles;
     handles.reserve(column_families.size());

diff --git a/tests/py/topology.py b/tests/py/topology.py
@@ -85,8 +85,6 @@ def import_json_data(self):
             del language
             links.setdefault("la_de", set()).add((language_id, developer_id))
 
-        print(sorted(language_ids.keys()))
-
         # load language -> license data
         # dbpedia SPARQL query (http://dbpedia.org/sparql):
         #   SELECT *