Merge nervosnetwork#418

418: fix: try to repair a corrupted rocksdb automatically r=quake a=yangby-cryptape

I have test this patch with a corrupted database (from @TheWaWaR).

But I can not reproduce the bug by myself, so I did not write any test case.

The database works, but the data is still corrupted (**because `header` add new fields?**):

```
2019-04-09 07:39:24.154 +00:00 main WARN ckb_db::diskdb  Try repairing the rocksdb since Corruption: SST file is ahead of WALs ...
2019-04-09 07:39:24.384 +00:00 main WARN ckb_db::diskdb  Try opening the repaired rocksdb ...
thread 'main' panicked at 'header deserializing should be ok: Custom("invalid length 3, expected a 0x-prefixed hex string with 64 digits")', src/libcore/result.rs:997:5
stack backtrace:
```

Ref:
- [RocksDB: Repairer](https://github.com/facebook/rocksdb/wiki/RocksDB-Repairer)
- [RocksDB: WAL Recovery Modes](https://github.com/facebook/rocksdb/wiki/WAL-Recovery-Modes)
- [`pub fn set_wal_recovery_mode(&mut self, mode: DBRecoveryMode)`](https://github.com/rust-rocksdb/rust-rocksdb/blob/267d92cbf9fdcbce9052ef06d7cb90b3111c6c54/src/db_options.rs#L1042-L1046)

Co-authored-by: Boyu Yang <yangby@cryptape.com>

db/Cargo.toml

@@ -14,6 +14,7 @@ serde = "1.0"
 serde_derive = "1.0"
 serde_json = "1.0"
 failure = "0.1.5"
+log = "0.4"
 
 [dev-dependencies]
 tempfile = "3.0"

db/src/diskdb.rs

@@ -1,6 +1,7 @@
 use crate::batch::{Batch, Col, Operation};
 use crate::config::DBConfig;
 use crate::kvdb::{ErrorKind, KeyValueDB, Result};
+use log::warn;
 use rocksdb::{ColumnFamily, Options, WriteBatch, DB};
 use std::ops::Range;
 
@@ -21,7 +22,21 @@ impl RocksDB {
 
         let cfnames: Vec<_> = (0..columns).map(|c| format!("c{}", c)).collect();
         let cf_options: Vec<&str> = cfnames.iter().map(|n| n as &str).collect();
-        let db = DB::open_cf(&opts, &config.path, &cf_options).expect("Failed to open rocksdb");
+        let db = DB::open_cf(&opts, &config.path, &cf_options).unwrap_or_else(|err| {
+            if err.as_ref().starts_with("Corruption:") {
+                warn!("Try repairing the rocksdb since {} ...", err);
+                let mut repair_opts = Options::default();
+                repair_opts.create_if_missing(false);
+                repair_opts.create_missing_column_families(false);
+                DB::repair(repair_opts, &config.path)
+                    .unwrap_or_else(|err| panic!("Failed to repair the rocksdb: {}", err));
+                warn!("Try opening the repaired rocksdb ...");
+                DB::open_cf(&opts, &config.path, &cf_options)
+                    .unwrap_or_else(|err| panic!("Failed to open the repaired rocksdb: {}", err))
+            } else {
+                panic!("Failed to open rocksdb: {}", err);
+            }
+        });
 
         if let Some(db_opt) = config.options.as_ref() {
             let rocksdb_options: Vec<(&str, &str)> = db_opt