Merge #2159 #2206

2159: feat: load db options from a file; support configuring column families r=driftluo,quake a=yangby-cryptape



2206: chore: tweak cargo dependencies version, remove some duplicate crates r=yangby-cryptape,driftluo,zhangsoledad a=quake



Co-authored-by: Boyu Yang <yangby@cryptape.com>
Co-authored-by: quake <quake.wang@gmail.com>

db/src/db.rs

@@ -5,8 +5,8 @@ use ckb_app_config::DBConfig;
 use ckb_logger::{info, warn};
 use rocksdb::ops::{GetColumnFamilys, GetPinned, GetPinnedCF, IterateCF, OpenCF, Put, SetOptions};
 use rocksdb::{
-    ffi, ColumnFamily, DBPinnableSlice, IteratorMode, OptimisticTransactionDB,
-    OptimisticTransactionOptions, Options, WriteOptions,
+    ffi, ColumnFamily, ColumnFamilyDescriptor, DBPinnableSlice, FullOptions, IteratorMode,
+    OptimisticTransactionDB, OptimisticTransactionOptions, Options, WriteOptions,
 };
 use std::sync::Arc;
 
@@ -19,50 +19,80 @@ pub struct RocksDB {
 
 impl RocksDB {
     pub(crate) fn open_with_check(config: &DBConfig, columns: u32) -> Result<Self> {
-        let mut opts = Options::default();
+        let cf_names: Vec<_> = (0..columns).map(|c| c.to_string()).collect();
+
+        let (mut opts, cf_descriptors) = if let Some(ref file) = config.options_file {
+            let mut full_opts = FullOptions::load_from_file(file, None, false).map_err(|err| {
+                internal_error(format!("failed to load the options file: {}", err))
+            })?;
+            let cf_names_str: Vec<&str> = cf_names.iter().map(|s| s.as_str()).collect();
+            full_opts
+                .complete_column_families(&cf_names_str, false)
+                .map_err(|err| {
+                    internal_error(format!("failed to check all column families: {}", err))
+                })?;
+            let FullOptions {
+                db_opts,
+                cf_descriptors,
+            } = full_opts;
+            (db_opts, cf_descriptors)
+        } else {
+            let opts = Options::default();
+            let cf_descriptors: Vec<_> = cf_names
+                .iter()
+                .map(|ref c| ColumnFamilyDescriptor::new(*c, Options::default()))
+                .collect();
+            (opts, cf_descriptors)
+        };
+
         opts.create_if_missing(false);
         opts.create_missing_column_families(true);
 
-        let cfnames: Vec<_> = (0..columns).map(|c| c.to_string()).collect();
-        let cf_options: Vec<&str> = cfnames.iter().map(|n| n as &str).collect();
-
-        let db =
-            OptimisticTransactionDB::open_cf(&opts, &config.path, &cf_options).or_else(|err| {
-                let err_str = err.as_ref();
-                if err_str.starts_with("Invalid argument:")
-                    && err_str.ends_with("does not exist (create_if_missing is false)")
-                {
-                    info!("Initialize a new database");
-                    opts.create_if_missing(true);
-                    let db = OptimisticTransactionDB::open_cf(&opts, &config.path, &cf_options)
-                        .map_err(|err| {
-                            internal_error(format!(
-                                "failed to open a new created database: {}",
-                                err
-                            ))
-                        })?;
-                    Ok(db)
-                } else if err.as_ref().starts_with("Corruption:") {
-                    warn!("Repairing the rocksdb since {} ...", err);
-                    let mut repair_opts = Options::default();
-                    repair_opts.create_if_missing(false);
-                    repair_opts.create_missing_column_families(false);
-                    OptimisticTransactionDB::repair(repair_opts, &config.path).map_err(|err| {
-                        internal_error(format!("failed to repair the database: {}", err))
-                    })?;
-                    warn!("Opening the repaired rocksdb ...");
-                    OptimisticTransactionDB::open_cf(&opts, &config.path, &cf_options).map_err(
-                        |err| {
-                            internal_error(format!("failed to open the repaired database: {}", err))
-                        },
-                    )
-                } else {
-                    Err(internal_error(format!(
-                        "failed to open the database: {}",
-                        err
-                    )))
-                }
-            })?;
+        let db = OptimisticTransactionDB::open_cf_descriptors(
+            &opts,
+            &config.path,
+            cf_descriptors.clone(),
+        )
+        .or_else(|err| {
+            let err_str = err.as_ref();
+            if err_str.starts_with("Invalid argument:")
+                && err_str.ends_with("does not exist (create_if_missing is false)")
+            {
+                info!("Initialize a new database");
+                opts.create_if_missing(true);
+                let db = OptimisticTransactionDB::open_cf_descriptors(
+                    &opts,
+                    &config.path,
+                    cf_descriptors.clone(),
+                )
+                .map_err(|err| {
+                    internal_error(format!("failed to open a new created database: {}", err))
+                })?;
+                Ok(db)
+            } else if err.as_ref().starts_with("Corruption:") {
+                warn!("Repairing the rocksdb since {} ...", err);
+                let mut repair_opts = Options::default();
+                repair_opts.create_if_missing(false);
+                repair_opts.create_missing_column_families(false);
+                OptimisticTransactionDB::repair(repair_opts, &config.path).map_err(|err| {
+                    internal_error(format!("failed to repair the database: {}", err))
+                })?;
+                warn!("Opening the repaired rocksdb ...");
+                OptimisticTransactionDB::open_cf_descriptors(
+                    &opts,
+                    &config.path,
+                    cf_descriptors.clone(),
+                )
+                .map_err(|err| {
+                    internal_error(format!("failed to open the repaired database: {}", err))
+                })
+            } else {
+                Err(internal_error(format!(
+                    "failed to open the database: {}",
+                    err
+                )))
+            }
+        })?;
 
         if !config.options.is_empty() {
             let rocksdb_options: Vec<(&str, &str)> = config
@@ -185,6 +215,7 @@ mod tests {
                 opts.insert("disable_auto_compactions".to_owned(), "true".to_owned());
                 opts
             },
+            options_file: None,
         };
         RocksDB::open(&config, 2); // no panic
     }
@@ -198,6 +229,7 @@ mod tests {
         let config = DBConfig {
             path: tmp_dir.as_ref().to_path_buf(),
             options: HashMap::new(),
+            options_file: None,
         };
         RocksDB::open(&config, 2); // no panic
     }
@@ -216,6 +248,7 @@ mod tests {
                 opts.insert("letsrock".to_owned(), "true".to_owned());
                 opts
             },
+            options_file: None,
         };
         RocksDB::open(&config, 2); // panic
     }

network/Cargo.toml

@@ -23,7 +23,7 @@ bs58 = "0.3.0"
 sentry = "0.16.0"
 faster-hex = "0.4"
 ckb-hash = {path = "../util/hash"}
-secp256k1 = {version = "0.15.0", features = ["recovery"] }
+secp256k1 = {version = "0.17", features = ["recovery"] }
 resolve = "0.2.0"
 num_cpus = "1.10"
 snap = "0.2"

script/Cargo.toml

@@ -25,7 +25,7 @@ serde = { version = "1.0", features = ["derive"] }
 ckb-error = { path = "../error" }
 failure = "0.1.5"
 ckb-chain-spec = { path = "../spec" }
-goblin = "0.1.3"
+goblin = "0.2"
 ckb-vm-definitions = "0.19.1"
 
 [dev-dependencies]

util/Cargo.toml

@@ -7,7 +7,7 @@ edition = "2018"
 
 [dependencies]
 parking_lot = "=0.7.1"
-linked-hash-map = { git = "https://github.com/nervosnetwork/linked-hash-map", rev = "df27f21" }
+linked-hash-map = "0.5"
 
 [dev-dependencies]
 ckb-fixed-hash = { path = "fixed-hash" }

util/app-config/src/app_config.rs

@@ -157,10 +157,10 @@ impl CKBAppConfig {
     fn derive_options(mut self, root_dir: &Path, subcommand_name: &str) -> Result<Self, ExitCode> {
         self.data_dir = canonicalize_data_dir(self.data_dir, root_dir)?;
 
-        self.db.update_path_if_not_set(self.data_dir.join("db"));
+        self.db.adjust(root_dir, &self.data_dir, "db");
         self.indexer
             .db
-            .update_path_if_not_set(self.data_dir.join("indexer_db"));
+            .adjust(root_dir, &self.data_dir, "indexer_db");
         self.network.path = self.data_dir.join("network");
         if self.tmp_dir.is_none() {
             self.tmp_dir = Some(self.data_dir.join("tmp"));

util/app-config/src/configs/db.rs

@@ -8,12 +8,24 @@ pub struct Config {
     pub path: PathBuf,
     #[serde(default)]
     pub options: HashMap<String, String>,
+    pub options_file: Option<PathBuf>,
 }
 
 impl Config {
-    pub fn update_path_if_not_set<P: AsRef<Path>>(&mut self, path: P) {
+    pub fn adjust<P: AsRef<Path>>(&mut self, root_dir: &Path, data_dir: P, name: &str) {
+        // If path is not set, use the default path
         if self.path.to_str().is_none() || self.path.to_str() == Some("") {
-            self.path = path.as_ref().to_path_buf();
+            self.path = data_dir.as_ref().to_path_buf().join(name);
+        } else if self.path.is_relative() {
+            // If the path is relative, set the base path to `ckb.toml`
+            self.path = root_dir.to_path_buf().join(&self.path)
+        }
+        // If options file is a relative path, set the base path to `ckb.toml`
+        if let Some(file) = self.options_file.iter_mut().next() {
+            if file.is_relative() {
+                let file_new = root_dir.to_path_buf().join(&file);
+                *file = file_new;
+            }
         }
     }
 }

util/crypto/Cargo.toml

@@ -8,7 +8,7 @@ edition = "2018"
 [dependencies]
 ckb-fixed-hash = { path = "../fixed-hash" }
 lazy_static = "1.3"
-secp256k1 = { version = "0.17.0", features = ["recovery"], optional = true }
+secp256k1 = { version = "0.17", features = ["recovery"], optional = true }
 failure = "0.1.5"
 rand = "0.6"
 faster-hex = "0.4"