From 74bca78bfb52a3ded2b8ef107db4f8935bc651fc Mon Sep 17 00:00:00 2001 From: sun haoyu Date: Mon, 21 Jan 2019 12:17:28 +0800 Subject: [PATCH 01/13] rebase develop --- build.gradle | 2 + .../common/application/ApplicationImpl.java | 2 + .../common/storage/BatchSourceInterRocks.java | 23 + .../org/tron/common/storage/DBSettings.java | 40 ++ .../common/storage/DbSourceInterRocks.java | 25 + .../tron/common/storage/SourceInterRocks.java | 19 + .../leveldb/RocksDbDataSourceImpl.java | 469 +++++++++++++++++ .../java/org/tron/common/utils/FileUtil.java | 21 + .../java/org/tron/common/utils/PropUtil.java | 77 +++ src/main/java/org/tron/core/Constant.java | 1 + .../tron/core/capsule/utils/BlockUtil.java | 17 + .../org/tron/core/config/CommonConfig.java | 2 + .../org/tron/core/config/DefaultConfig.java | 30 +- .../java/org/tron/core/config/args/Args.java | 26 +- src/main/java/org/tron/core/db/Manager.java | 2 - .../java/org/tron/core/db/ManagerForTest.java | 81 +++ .../org/tron/core/db/RevokingDatabase.java | 1 + .../org/tron/core/db/RevokingStoreRocks.java | 477 ++++++++++++++++++ .../tron/core/db/TronStoreWithRevoking.java | 12 +- .../org/tron/core/db/backup/BackupDbUtil.java | 151 ++++++ .../core/db/backup/BackupRocksDBAspect.java | 36 ++ .../tron/core/db/backup/DbBackupConfig.java | 77 +++ .../core/db/backup/NeedBeanCondition.java | 15 + .../db/common/iterator/RockStoreIterator.java | 72 +++ .../RevokingRocksDBWithCachingOldValue.java | 163 ++++++ .../java/org/tron/core/net/node/NodeImpl.java | 37 +- src/main/resources/config-dbbackup.conf | 411 +++++++++++++++ src/main/resources/config.conf | 15 +- src/main/resources/logback.xml | 18 +- .../leveldb/RocksDbDataSourceImplTest.java | 253 ++++++++++ .../org/tron/common/utils/FileUtilTest.java | 49 ++ .../org/tron/common/utils/PropUtilTest.java | 40 ++ .../tron/core/db/backup/BackupDbUtilTest.java | 115 +++++ .../org/tron/core/net/node/BaseNetTest.java | 2 + .../org/tron/core/net/node/BroadTest.java | 3 + .../net/node/GetBlockChainSummaryTest.java | 3 + .../core/net/node/GetLostBlockIdsTest.java | 3 + .../org/tron/core/net/node/NodeImplTest.java | 20 +- .../net/node/StartFetchSyncBlockTest.java | 2 + .../org/tron/core/net/node/TcpNetTest.java | 1 + src/test/resources/config-test-dbbackup.conf | 411 +++++++++++++++ 41 files changed, 3180 insertions(+), 44 deletions(-) create mode 100644 src/main/java/org/tron/common/storage/BatchSourceInterRocks.java create mode 100644 src/main/java/org/tron/common/storage/DBSettings.java create mode 100644 src/main/java/org/tron/common/storage/DbSourceInterRocks.java create mode 100644 src/main/java/org/tron/common/storage/SourceInterRocks.java create mode 100644 src/main/java/org/tron/common/storage/leveldb/RocksDbDataSourceImpl.java create mode 100644 src/main/java/org/tron/common/utils/PropUtil.java create mode 100644 src/main/java/org/tron/core/db/ManagerForTest.java create mode 100644 src/main/java/org/tron/core/db/RevokingStoreRocks.java create mode 100644 src/main/java/org/tron/core/db/backup/BackupDbUtil.java create mode 100644 src/main/java/org/tron/core/db/backup/BackupRocksDBAspect.java create mode 100644 src/main/java/org/tron/core/db/backup/DbBackupConfig.java create mode 100644 src/main/java/org/tron/core/db/backup/NeedBeanCondition.java create mode 100644 src/main/java/org/tron/core/db/common/iterator/RockStoreIterator.java create mode 100644 src/main/java/org/tron/core/db2/core/RevokingRocksDBWithCachingOldValue.java create mode 100644 src/main/resources/config-dbbackup.conf create mode 100644 src/test/java/org/tron/common/storage/leveldb/RocksDbDataSourceImplTest.java create mode 100644 src/test/java/org/tron/common/utils/FileUtilTest.java create mode 100644 src/test/java/org/tron/common/utils/PropUtilTest.java create mode 100644 src/test/java/org/tron/core/db/backup/BackupDbUtilTest.java create mode 100644 src/test/resources/config-test-dbbackup.conf diff --git a/build.gradle b/build.gradle index c1e701f3174..9705cf60fd6 100755 --- a/build.gradle +++ b/build.gradle @@ -116,6 +116,8 @@ dependencies { compile "org.iq80.leveldb:leveldb:0.7" + compile group: 'org.rocksdb', name: 'rocksdbjni', version: '5.15.10' + compile group: leveldbGroup, name: leveldbName, version: leveldbVersion compile "org.apache.commons:commons-collections4:4.0" diff --git a/src/main/java/org/tron/common/application/ApplicationImpl.java b/src/main/java/org/tron/common/application/ApplicationImpl.java index ab4a484d29f..417b41a558b 100644 --- a/src/main/java/org/tron/common/application/ApplicationImpl.java +++ b/src/main/java/org/tron/common/application/ApplicationImpl.java @@ -68,6 +68,7 @@ public void startup() { @Override public void shutdown() { logger.info("******** begin to shutdown ********"); + //p2pNode.shutDown(); synchronized (dbManager.getRevokingStore()) { closeRevokingStore(); closeAllStore(); @@ -124,6 +125,7 @@ private void closeConnection() { } private void closeRevokingStore() { + logger.info("******** begin to closeRevokingStore ********"); dbManager.getRevokingStore().shutdown(); } diff --git a/src/main/java/org/tron/common/storage/BatchSourceInterRocks.java b/src/main/java/org/tron/common/storage/BatchSourceInterRocks.java new file mode 100644 index 00000000000..3a0eb814e9b --- /dev/null +++ b/src/main/java/org/tron/common/storage/BatchSourceInterRocks.java @@ -0,0 +1,23 @@ +package org.tron.common.storage; + +import java.util.Map; +import org.rocksdb.WriteOptions; + + +public interface BatchSourceInterRocks extends SourceInterRocks { + + + void updateByBatch(Map rows); + + void updateByBatch(Map rows, WriteOptions writeOptions); + + void putData(K key, V val); + + V getData(K key); + + + void deleteData(K key); + + boolean flush(); + +} diff --git a/src/main/java/org/tron/common/storage/DBSettings.java b/src/main/java/org/tron/common/storage/DBSettings.java new file mode 100644 index 00000000000..2c52c870b35 --- /dev/null +++ b/src/main/java/org/tron/common/storage/DBSettings.java @@ -0,0 +1,40 @@ +package org.tron.common.storage; + + +public class DBSettings { + + public static final DBSettings DEFAULT = new DBSettings() + .withMaxThreads(1) + .withMaxOpenFiles(32); + + int maxOpenFiles; + int maxThreads; + + private DBSettings() { + } + + public static DBSettings newInstance() { + DBSettings settings = new DBSettings(); + settings.maxOpenFiles = DEFAULT.maxOpenFiles; + settings.maxThreads = DEFAULT.maxThreads; + return settings; + } + + public int getMaxOpenFiles() { + return maxOpenFiles; + } + + public DBSettings withMaxOpenFiles(int maxOpenFiles) { + this.maxOpenFiles = maxOpenFiles; + return this; + } + + public int getMaxThreads() { + return maxThreads; + } + + public DBSettings withMaxThreads(int maxThreads) { + this.maxThreads = maxThreads; + return this; + } +} diff --git a/src/main/java/org/tron/common/storage/DbSourceInterRocks.java b/src/main/java/org/tron/common/storage/DbSourceInterRocks.java new file mode 100644 index 00000000000..d5907401da7 --- /dev/null +++ b/src/main/java/org/tron/common/storage/DbSourceInterRocks.java @@ -0,0 +1,25 @@ +package org.tron.common.storage; + +import java.util.Set; + +public interface DbSourceInterRocks extends BatchSourceInterRocks { + + String getDBName(); + + void setDBName(String name); + + void initDB(); + + boolean isAlive(); + + void closeDB(); + + void resetDb(); + + Set allKeys() throws RuntimeException; + + Set allValues() throws RuntimeException; + + long getTotal() throws RuntimeException; + +} diff --git a/src/main/java/org/tron/common/storage/SourceInterRocks.java b/src/main/java/org/tron/common/storage/SourceInterRocks.java new file mode 100644 index 00000000000..71096b9ec5f --- /dev/null +++ b/src/main/java/org/tron/common/storage/SourceInterRocks.java @@ -0,0 +1,19 @@ +package org.tron.common.storage; + + +import org.rocksdb.WriteOptions; + +public interface SourceInterRocks { + + void putData(K key, V val); + + void putData(K k, V v, WriteOptions options); + + V getData(K key); + + void deleteData(K key); + + void deleteData(K k, WriteOptions options); + + boolean flush(); +} diff --git a/src/main/java/org/tron/common/storage/leveldb/RocksDbDataSourceImpl.java b/src/main/java/org/tron/common/storage/leveldb/RocksDbDataSourceImpl.java new file mode 100644 index 00000000000..7fdc6c0a04e --- /dev/null +++ b/src/main/java/org/tron/common/storage/leveldb/RocksDbDataSourceImpl.java @@ -0,0 +1,469 @@ +package org.tron.common.storage.leveldb; + +import com.google.common.collect.Sets; +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.concurrent.locks.ReadWriteLock; +import java.util.concurrent.locks.ReentrantReadWriteLock; +import lombok.NoArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.rocksdb.BlockBasedTableConfig; +import org.rocksdb.BloomFilter; +import org.rocksdb.Checkpoint; +import org.rocksdb.CompressionType; +import org.rocksdb.Options; +import org.rocksdb.ReadOptions; +import org.rocksdb.RocksDB; +import org.rocksdb.RocksDBException; +import org.rocksdb.RocksIterator; +import org.rocksdb.WriteBatch; +import org.rocksdb.WriteOptions; +import org.tron.common.storage.DBSettings; +import org.tron.common.storage.DbSourceInterRocks; +import org.tron.common.utils.FileUtil; +import org.tron.core.config.args.Args; +import org.tron.core.db.common.iterator.RockStoreIterator; + +@Slf4j +@NoArgsConstructor +public class RocksDbDataSourceImpl implements DbSourceInterRocks, + Iterable> { + + private String dataBaseName; + private RocksDB database; + private boolean alive; + private String parentName; + ReadOptions readOpts; + + DBSettings settings = DBSettings.DEFAULT; + + private ReadWriteLock resetDbLock = new ReentrantReadWriteLock(); + + public RocksDbDataSourceImpl(String parentName, String name) { + this.dataBaseName = name; + this.parentName = Paths.get( + parentName, + Args.getInstance().getStorage().getDbDirectory() + ).toString(); + } + + public Path getDbPath() { + return Paths.get(parentName, dataBaseName); + } + + public RocksDB getDatabase() { + return database; + } + + public boolean isAlive() { + return alive; + } + + @Override + public void closeDB() { + resetDbLock.writeLock().lock(); + try { + if (!isAlive()) { + return; + } + database.close(); + alive = false; + } catch (Exception e) { + } finally { + resetDbLock.writeLock().unlock(); + } + } + + @Override + public void resetDb() { + closeDB(); + FileUtil.recursiveDelete(getDbPath().toString()); + initDB(); + } + + private boolean quitIfNotAlive() { + if (!isAlive()) { + logger.warn("db is not alive"); + } + return !isAlive(); + } + + @Override + public Set allKeys() throws RuntimeException { + if (quitIfNotAlive()) { + return null; + } + resetDbLock.readLock().lock(); + Set result = Sets.newHashSet(); + try (final RocksIterator iter = database.newIterator()) { + for (iter.seekToFirst(); iter.isValid(); iter.next()) { + result.add(iter.key()); + } + return result; + } finally { + resetDbLock.readLock().unlock(); + } + } + + @Override + public Set allValues() throws RuntimeException { + return null; + } + + @Override + public long getTotal() throws RuntimeException { + return 0; + } + + @Override + public String getDBName() { + return this.dataBaseName; + } + + @Override + public void setDBName(String name) { + } + + public void initDB() { + initDB(DBSettings.DEFAULT); + } + + public void initDB(DBSettings settings) { + this.settings = settings; + resetDbLock.writeLock().lock(); + try { + if (isAlive()) { + return; + } + if (dataBaseName == null) { + throw new NullPointerException("no name set to the dbStore"); + } + + try (Options options = new Options()) { + + // most of these options are suggested by https://github.com/facebook/rocksdb/wiki/Set-Up-Options + + // general options + options.setCreateIfMissing(true); + options.setCompressionType(CompressionType.LZ4_COMPRESSION); + options.setBottommostCompressionType(CompressionType.ZSTD_COMPRESSION); + options.setLevelCompactionDynamicLevelBytes(true); + options.setMaxOpenFiles(settings.getMaxOpenFiles()); + options.setIncreaseParallelism(settings.getMaxThreads()); + + // table options + final BlockBasedTableConfig tableCfg; + options.setTableFormatConfig(tableCfg = new BlockBasedTableConfig()); + tableCfg.setBlockSize(16 * 1024); + tableCfg.setBlockCacheSize(32 * 1024 * 1024); + tableCfg.setCacheIndexAndFilterBlocks(true); + tableCfg.setPinL0FilterAndIndexBlocksInCache(true); + tableCfg.setFilter(new BloomFilter(10, false)); + + // read options + readOpts = new ReadOptions(); + readOpts = readOpts.setPrefixSameAsStart(true) + .setVerifyChecksums(false); + + try { + logger.debug("Opening database"); + final Path dbPath = getDbPath(); + if (!Files.isSymbolicLink(dbPath.getParent())) { + Files.createDirectories(dbPath.getParent()); + } + + try { + database = RocksDB.open(options, dbPath.toString()); + } catch (RocksDBException e) { + logger.error(e.getMessage(), e); + throw new RuntimeException("Failed to initialize database", e); + } + + alive = true; + + } catch (IOException ioe) { + logger.error(ioe.getMessage(), ioe); + throw new RuntimeException("Failed to initialize database", ioe); + } + + logger.debug("<~ RocksDbDataSource.initDB(): " + dataBaseName); + } + } finally { + resetDbLock.writeLock().unlock(); + } + } + + @Override + public void putData(byte[] key, byte[] value) { + if (quitIfNotAlive()) { + return; + } + resetDbLock.readLock().lock(); + try { + database.put(key, value); + } catch (RocksDBException e) { + logger.error("RocksDBException:{}", e); + } finally { + resetDbLock.readLock().unlock(); + } + } + + @Override + public void putData(byte[] key, byte[] value, WriteOptions writeOpt) { + if (quitIfNotAlive()) { + return; + } + resetDbLock.readLock().lock(); + try { + database.put(writeOpt, key, value); + } catch (RocksDBException e) { + logger.error("RocksDBException:{}", e); + } finally { + resetDbLock.readLock().unlock(); + } + } + + @Override + public byte[] getData(byte[] key) { + if (quitIfNotAlive()) { + return null; + } + resetDbLock.readLock().lock(); + try { + return database.get(key); + } catch (RocksDBException e) { + logger.error("RocksDBException: {}", e); + } finally { + resetDbLock.readLock().unlock(); + } + return null; + } + + @Override + public void deleteData(byte[] key) { + if (quitIfNotAlive()) { + return; + } + resetDbLock.readLock().lock(); + try { + database.delete(key); + } catch (RocksDBException e) { + logger.error("RocksDBException:{}", e); + } finally { + resetDbLock.readLock().unlock(); + } + } + + @Override + public void deleteData(byte[] key, WriteOptions writeOpt) { + if (quitIfNotAlive()) { + return; + } + resetDbLock.readLock().lock(); + try { + database.delete(writeOpt, key); + } catch (RocksDBException e) { + logger.error("RocksDBException:{}", e); + } finally { + resetDbLock.readLock().unlock(); + } + } + + @Override + public boolean flush() { + return false; + } + + @Override + public org.tron.core.db.common.iterator.DBIterator iterator() { + return new RockStoreIterator(database.newIterator()); + } + + private void updateByBatchInner(Map rows) throws Exception { + if (quitIfNotAlive()) { + return; + } + try (WriteBatch batch = new WriteBatch()) { + for (Map.Entry entry : rows.entrySet()) { + if (entry.getValue() == null) { + batch.delete(entry.getKey()); + } else { + batch.put(entry.getKey(), entry.getValue()); + } + } + database.write(new WriteOptions(), batch); + } + } + + private void updateByBatchInner(Map rows, WriteOptions options) + throws Exception { + if (quitIfNotAlive()) { + return; + } + try (WriteBatch batch = new WriteBatch()) { + for (Map.Entry entry : rows.entrySet()) { + if (entry.getValue() == null) { + batch.delete(entry.getKey()); + } else { + batch.put(entry.getKey(), entry.getValue()); + } + } + database.write(new WriteOptions(), batch); + } + } + + @Override + public void updateByBatch(Map rows) { + if (quitIfNotAlive()) { + return; + } + resetDbLock.readLock().lock(); + try { + updateByBatchInner(rows); + } catch (Exception e) { + try { + updateByBatchInner(rows); + } catch (Exception e1) { + throw new RuntimeException(e); + } + } finally { + resetDbLock.readLock().unlock(); + } + } + + @Override + public void updateByBatch(Map rows, WriteOptions writeOptions) { + if (quitIfNotAlive()) { + return; + } + resetDbLock.readLock().lock(); + try { + updateByBatchInner(rows, writeOptions); + } catch (Exception e) { + try { + updateByBatchInner(rows); + } catch (Exception e1) { + throw new RuntimeException(e); + } + } finally { + resetDbLock.readLock().unlock(); + } + } + + public Map getNext(byte[] key, long limit) { + if (quitIfNotAlive()) { + return null; + } + if (limit <= 0) { + return Collections.emptyMap(); + } + resetDbLock.readLock().lock(); + try (RocksIterator iter = database.newIterator()) { + Map result = new HashMap<>(); + long i = 0; + for (iter.seek(key); iter.isValid() && i < limit; iter.next(), i++) { + result.put(iter.key(), iter.value()); + } + return result; + } finally { + resetDbLock.readLock().unlock(); + } + } + + public Set getlatestValues(long limit) { + if (quitIfNotAlive()) { + return null; + } + if (limit <= 0) { + return Sets.newHashSet(); + } + resetDbLock.readLock().lock(); + try (RocksIterator iter = database.newIterator()) { + Set result = Sets.newHashSet(); + long i = 0; + for (iter.seekToLast(); iter.isValid() && i < limit; iter.prev(), i++) { + result.add(iter.value()); + } + return result; + } finally { + resetDbLock.readLock().unlock(); + } + } + + public Set getValuesPrev(byte[] key, long limit) { + if (quitIfNotAlive()) { + return null; + } + if (limit <= 0) { + return Sets.newHashSet(); + } + resetDbLock.readLock().lock(); + try (RocksIterator iter = database.newIterator()) { + Set result = Sets.newHashSet(); + long i = 0; + byte[] data = getData(key); + if (Objects.nonNull(data)) { + result.add(data); + i++; + } + for (iter.seekForPrev(key); iter.isValid() && i < limit; iter.prev(), i++) { + result.add(iter.value()); + } + return result; + } finally { + resetDbLock.readLock().unlock(); + } + } + + public Set getValuesNext(byte[] key, long limit) { + if (quitIfNotAlive()) { + return null; + } + if (limit <= 0) { + return Sets.newHashSet(); + } + resetDbLock.readLock().lock(); + try (RocksIterator iter = database.newIterator()) { + Set result = Sets.newHashSet(); + long i = 0; + for (iter.seek(key); iter.isValid() && i < limit; iter.next(), i++) { + result.add(iter.value()); + } + return result; + } finally { + resetDbLock.readLock().unlock(); + } + } + + // rocksdb use + public void backup(int i) throws RocksDBException { + if (i == 1) { + Checkpoint.create(database) + .createCheckpoint(Args.getInstance().getDbBackupConfig().getBak1path() + + this.getDBName()); + } else { + Checkpoint.create(database) + .createCheckpoint(Args.getInstance().getDbBackupConfig().getBak2path() + + this.getDBName()); + } + } + + // rocksdb use + public boolean deleteDbBakPath(int i) { + if (i == 1) { + return FileUtil.deleteDir(new File(Args.getInstance().getDbBackupConfig().getBak1path() + + this.getDBName())); + } else { + return FileUtil.deleteDir((new File(Args.getInstance().getDbBackupConfig().getBak2path() + + this.getDBName()))); + } + } +} \ No newline at end of file diff --git a/src/main/java/org/tron/common/utils/FileUtil.java b/src/main/java/org/tron/common/utils/FileUtil.java index e7184cdd4df..f9d1f57a9aa 100755 --- a/src/main/java/org/tron/common/utils/FileUtil.java +++ b/src/main/java/org/tron/common/utils/FileUtil.java @@ -124,4 +124,25 @@ public static boolean deleteDir(File dir) { } return dir.delete(); } + + public static boolean createFileIfNotExists(String filepath) { + File file = new File(filepath); + if (!file.exists()) { + try { + file.createNewFile(); + } catch (IOException e) { + e.printStackTrace(); + return false; + } + } + return true; + } + + public static boolean createDirIfNotExists(String dirPath) { + File dir = new File(dirPath); + if (!dir.exists()) { + return dir.mkdirs(); + } + return true; + } } diff --git a/src/main/java/org/tron/common/utils/PropUtil.java b/src/main/java/org/tron/common/utils/PropUtil.java new file mode 100644 index 00000000000..30bc6ec89be --- /dev/null +++ b/src/main/java/org/tron/common/utils/PropUtil.java @@ -0,0 +1,77 @@ +package org.tron.common.utils; + +import java.io.BufferedInputStream; +import java.io.BufferedReader; +import java.io.BufferedWriter; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.OutputStream; +import java.io.OutputStreamWriter; +import java.net.URISyntaxException; +import java.util.ArrayList; +import java.util.List; +import java.util.Properties; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public class PropUtil { + + public static String readProperty(String file, String key) { + InputStream is = null; + FileInputStream fis = null; + Properties prop = null; + try { + prop = new Properties(); + fis = new FileInputStream(file); + is = new BufferedInputStream(fis); + prop.load(is); + String value = new String(prop.getProperty(key, "").getBytes("ISO-8859-1"), "UTF-8"); + return value; + } catch (Exception e) { + logger.warn("{}", e); + return ""; + } finally { + if (prop != null) { + prop = null; + } + //fis + try { + if (fis != null) { + fis.close(); + fis = null; + } + } catch (Exception e) { + logger.warn("{}", e); + } + //is + try { + if (is != null) { + is.close(); + is = null; + } + } catch (Exception e) { + logger.warn("{}", e); + } + } + } + + public static void writeProperty(String file, String key, String value) { + FileInputStream fis = null; + Properties properties = new Properties(); + try { + fis = new FileInputStream(file); + BufferedReader bf = new BufferedReader(new InputStreamReader(fis, "UTF-8")); + properties.load(bf); + OutputStream out = new FileOutputStream(file); + BufferedWriter bw = new BufferedWriter(new OutputStreamWriter(out, "UTF-8")); + properties.setProperty(key, value); + properties.store(bw, "Generated by the application. PLEASE DO NOT EDIT! "); + out.close(); + } catch (Exception e) { + logger.warn("{}", e); + } + } + +} \ No newline at end of file diff --git a/src/main/java/org/tron/core/Constant.java b/src/main/java/org/tron/core/Constant.java index bfc4fffff2a..c1247c0f52e 100644 --- a/src/main/java/org/tron/core/Constant.java +++ b/src/main/java/org/tron/core/Constant.java @@ -32,6 +32,7 @@ public class Constant { //config for junit test public static final String TEST_CONF = "config-test.conf"; + public static final String TESTBACKUP_CONF = "config-dbbackup.conf"; public static final String DATABASE_DIR = "storage.directory"; diff --git a/src/main/java/org/tron/core/capsule/utils/BlockUtil.java b/src/main/java/org/tron/core/capsule/utils/BlockUtil.java index ea422623db0..9420bcf53b8 100644 --- a/src/main/java/org/tron/core/capsule/utils/BlockUtil.java +++ b/src/main/java/org/tron/core/capsule/utils/BlockUtil.java @@ -17,11 +17,15 @@ import com.google.protobuf.ByteString; import java.util.List; +import java.util.Map; import java.util.stream.Collectors; import org.tron.common.utils.ByteArray; +import org.tron.common.utils.Sha256Hash; import org.tron.core.capsule.BlockCapsule; import org.tron.core.config.args.Args; import org.tron.core.config.args.GenesisBlock; +import org.tron.core.db.Manager; +import org.tron.core.witness.WitnessController; import org.tron.protos.Protocol.Transaction; public class BlockUtil { @@ -62,4 +66,17 @@ public static BlockCapsule newGenesisBlockCapsule() { public static boolean isParentOf(BlockCapsule blockCapsule1, BlockCapsule blockCapsule2) { return blockCapsule1.getBlockId().equals(blockCapsule2.getParentHash()); } + + public static BlockCapsule createTestBlockCapsule(Manager dbManager, long time, + long number, ByteString hash, Map addressToProvateKeys) { + WitnessController witnessController = dbManager.getWitnessController(); + ByteString witnessAddress = + witnessController.getScheduledWitness(witnessController.getSlotAtTime(time)); + BlockCapsule blockCapsule = new BlockCapsule(number, Sha256Hash.wrap(hash), time, + witnessAddress); + blockCapsule.generatedByMyself = true; + blockCapsule.setMerkleRoot(); + blockCapsule.sign(ByteArray.fromHexString(addressToProvateKeys.get(witnessAddress))); + return blockCapsule; + } } diff --git a/src/main/java/org/tron/core/config/CommonConfig.java b/src/main/java/org/tron/core/config/CommonConfig.java index 43f4a1efb22..509eca7c379 100644 --- a/src/main/java/org/tron/core/config/CommonConfig.java +++ b/src/main/java/org/tron/core/config/CommonConfig.java @@ -19,10 +19,12 @@ import org.springframework.context.annotation.ComponentScan; import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.EnableAspectJAutoProxy; import org.springframework.transaction.annotation.EnableTransactionManagement; @Configuration @EnableTransactionManagement +@EnableAspectJAutoProxy @ComponentScan(basePackages = "org.tron") public class CommonConfig { diff --git a/src/main/java/org/tron/core/config/DefaultConfig.java b/src/main/java/org/tron/core/config/DefaultConfig.java index 1a7f44d0f56..0bef8ae0d75 100755 --- a/src/main/java/org/tron/core/config/DefaultConfig.java +++ b/src/main/java/org/tron/core/config/DefaultConfig.java @@ -5,13 +5,17 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.ApplicationContext; import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Conditional; import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.Import; import org.tron.core.config.args.Args; import org.tron.core.db.RevokingDatabase; import org.tron.core.db.RevokingStore; import org.tron.core.db.TransactionCache; +import org.tron.core.db.RevokingStoreRocks; import org.tron.core.db.api.IndexHelper; +import org.tron.core.db.backup.BackupRocksDBAspect; +import org.tron.core.db.backup.NeedBeanCondition; import org.tron.core.db2.core.SnapshotManager; import org.tron.core.services.interfaceOnSolidity.RpcApiServiceOnSolidity; import org.tron.core.services.interfaceOnSolidity.http.solidity.HttpApiOnSolidityService; @@ -43,15 +47,24 @@ public IndexHelper indexHelper() { @Bean public RevokingDatabase revokingDatabase() { int dbVersion = Args.getInstance().getStorage().getDbVersion(); - if (dbVersion == 1) { - return RevokingStore.getInstance(); - } else if (dbVersion == 2) { - return new SnapshotManager(); - } else { - throw new RuntimeException("db version is error."); + RevokingDatabase revokingDatabase; + try { + if (dbVersion == 1) { + revokingDatabase = RevokingStore.getInstance(); + } else if (dbVersion == 2) { + revokingDatabase = new SnapshotManager(); + } else if (dbVersion == 3) { + revokingDatabase = RevokingStoreRocks.getInstance(); + } else { + throw new RuntimeException("db version is error."); + } + return revokingDatabase; + } finally { + logger.info("key-value data source created."); } } + @Bean public RpcApiServiceOnSolidity getRpcApiServiceOnSolidity() { boolean isSolidityNode = Args.getInstance().isSolidityNode(); @@ -84,4 +97,9 @@ public TransactionCache transactionCache() { return null; } + @Bean + @Conditional(NeedBeanCondition.class) + public BackupRocksDBAspect backupRocksDBAspect() { + return new BackupRocksDBAspect(); + } } diff --git a/src/main/java/org/tron/core/config/args/Args.java b/src/main/java/org/tron/core/config/args/Args.java index 926a2a977a6..5e91b1d1808 100644 --- a/src/main/java/org/tron/core/config/args/Args.java +++ b/src/main/java/org/tron/core/config/args/Args.java @@ -44,6 +44,7 @@ import org.tron.core.config.Parameter.ChainConstant; import org.tron.core.config.Parameter.NetConstants; import org.tron.core.db.AccountStore; +import org.tron.core.db.backup.DbBackupConfig; import org.tron.keystore.CipherException; import org.tron.keystore.Credentials; import org.tron.keystore.WalletUtils; @@ -396,7 +397,6 @@ public class Args { @Setter private String trxReferenceBlock; - @Getter @Setter private int minEffectiveConnection; @@ -415,6 +415,9 @@ public class Args { @Getter private FilterQuery eventFilter; + @Getter + private DbBackupConfig dbBackupConfig; + public static void clearParam() { INSTANCE.outputDirectory = "output-directory"; INSTANCE.help = false; @@ -819,7 +822,6 @@ public static void setParam(final String[] args, final String confFileName) { config.hasPath("storage.needToUpdateAsset") ? config .getBoolean("storage.needToUpdateAsset") : true; - INSTANCE.trxReferenceBlock = config.hasPath("trx.reference.block") ? config.getString("trx.reference.block") : "head"; @@ -843,6 +845,7 @@ public static void setParam(final String[] args, final String confFileName) { INSTANCE.eventFilter = config.hasPath("event.subscribe.filter") ? getEventFilter(config) : null; + initRocksDbBackupProperty(config); initBackupProperty(config); logConfig(); @@ -1123,6 +1126,25 @@ private static double calcMaxTimeRatio() { return 5.0; } + + private static void initRocksDbBackupProperty(Config config) { + boolean enable = false; + if (Args.getInstance().getStorage().getDbVersion() == 3) { + enable = config.hasPath("storage.backup.enable") ? config.getBoolean("storage.backup.enable") + : false; + } + String propPath = config.hasPath("storage.backup.propPath") + ? config.getString("storage.backup.propPath") : "prop.properties"; + String bak1path = config.hasPath("storage.backup.bak1path") + ? config.getString("storage.backup.bak1path") : "bak1/database/"; + String bak2path = config.hasPath("storage.backup.bak2path") + ? config.getString("storage.backup.bak2path") : "bak2/database/"; + int frequency = config.hasPath("storage.backup.frequency") + ? config.getInt("storage.backup.frequency") : 10000; + INSTANCE.dbBackupConfig = DbBackupConfig.getInstance() + .initArgs(enable, propPath, bak1path, bak2path, frequency); + } + private static void initBackupProperty(Config config) { INSTANCE.backupPriority = config.hasPath("node.backup.priority") ? config.getInt("node.backup.priority") : 0; diff --git a/src/main/java/org/tron/core/db/Manager.java b/src/main/java/org/tron/core/db/Manager.java index 802c9ec9726..d3dae5158ee 100644 --- a/src/main/java/org/tron/core/db/Manager.java +++ b/src/main/java/org/tron/core/db/Manager.java @@ -1679,7 +1679,6 @@ public void setAccountIndexStore(AccountIndexStore indexStore) { } public void closeAllStore() { - logger.info("******** begin to close db ********"); closeOneStore(accountStore); closeOneStore(blockStore); closeOneStore(blockIndexStore); @@ -1702,7 +1701,6 @@ public void closeAllStore() { closeOneStore(delegatedResourceStore); closeOneStore(assetIssueV2Store); closeOneStore(exchangeV2Store); - logger.info("******** end to close db ********"); } public void closeOneStore(ITronChainBase database) { diff --git a/src/main/java/org/tron/core/db/ManagerForTest.java b/src/main/java/org/tron/core/db/ManagerForTest.java new file mode 100644 index 00000000000..81ceef29bcd --- /dev/null +++ b/src/main/java/org/tron/core/db/ManagerForTest.java @@ -0,0 +1,81 @@ +package org.tron.core.db; + +import com.google.common.collect.Maps; +import com.google.protobuf.ByteString; +import java.util.Map; +import java.util.stream.Collectors; +import java.util.stream.IntStream; +import org.tron.common.crypto.ECKey; +import org.tron.common.utils.ByteArray; +import org.tron.common.utils.Sha256Hash; +import org.tron.common.utils.Utils; +import org.tron.core.capsule.AccountCapsule; +import org.tron.core.capsule.BlockCapsule; +import org.tron.core.capsule.WitnessCapsule; +import org.tron.core.witness.WitnessController; +import org.tron.protos.Protocol.Account; + +public class ManagerForTest { + + private Manager dbManager; + + public ManagerForTest(Manager dbManager) { + this.dbManager = dbManager; + } + + private Map addTestWitnessAndAccount() { + dbManager.getWitnesses().clear(); + return IntStream.range(0, 2) + .mapToObj( + i -> { + ECKey ecKey = new ECKey(Utils.getRandom()); + String privateKey = ByteArray.toHexString(ecKey.getPrivKey().toByteArray()); + ByteString address = ByteString.copyFrom(ecKey.getAddress()); + + WitnessCapsule witnessCapsule = new WitnessCapsule(address); + dbManager.getWitnessStore().put(address.toByteArray(), witnessCapsule); + dbManager.getWitnessController().addWitness(address); + + AccountCapsule accountCapsule = + new AccountCapsule(Account.newBuilder().setAddress(address).build()); + dbManager.getAccountStore().put(address.toByteArray(), accountCapsule); + + return Maps.immutableEntry(address, privateKey); + }) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + } + + private ByteString getWitnessAddress(long time) { + WitnessController witnessController = dbManager.getWitnessController(); + return witnessController.getScheduledWitness(witnessController.getSlotAtTime(time)); + } + + public BlockCapsule createTestBlockCapsule(long time, + long number, ByteString hash) { + + Map addressToProvateKeys = addTestWitnessAndAccount(); + ByteString witnessAddress = getWitnessAddress(time); + + BlockCapsule blockCapsule = new BlockCapsule(number, Sha256Hash.wrap(hash), time, + witnessAddress); + blockCapsule.generatedByMyself = true; + blockCapsule.setMerkleRoot(); + blockCapsule.sign(ByteArray.fromHexString(addressToProvateKeys.get(witnessAddress))); + return blockCapsule; + } + + public boolean pushNTestBlock(int count) { + try { + for (int i = 1; i <= count; i++) { + ByteString hash = dbManager.getDynamicPropertiesStore().getLatestBlockHeaderHash().getByteString(); + long time = dbManager.getDynamicPropertiesStore().getLatestBlockHeaderTimestamp() + 3000L; + long number = dbManager.getDynamicPropertiesStore().getLatestBlockHeaderNumber() + 1; + BlockCapsule blockCapsule = createTestBlockCapsule(time, number, hash); + dbManager.pushBlock(blockCapsule); + } + } catch (Exception ignore) { + return false; + } + return true; + } +} \ No newline at end of file diff --git a/src/main/java/org/tron/core/db/RevokingDatabase.java b/src/main/java/org/tron/core/db/RevokingDatabase.java index f81058a31e9..5bfebd932fa 100755 --- a/src/main/java/org/tron/core/db/RevokingDatabase.java +++ b/src/main/java/org/tron/core/db/RevokingDatabase.java @@ -37,4 +37,5 @@ public interface RevokingDatabase { void setMaxFlushCount(int maxFlushCount); void shutdown(); + } diff --git a/src/main/java/org/tron/core/db/RevokingStoreRocks.java b/src/main/java/org/tron/core/db/RevokingStoreRocks.java new file mode 100644 index 00000000000..65b6d53cc4b --- /dev/null +++ b/src/main/java/org/tron/core/db/RevokingStoreRocks.java @@ -0,0 +1,477 @@ +package org.tron.core.db; + +import static org.tron.core.db2.core.SnapshotManager.simpleDecode; + +import com.google.common.collect.Maps; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Deque; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.stream.Collectors; +import lombok.AllArgsConstructor; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.ToString; +import lombok.extern.slf4j.Slf4j; +import org.rocksdb.RocksDB; +import org.rocksdb.WriteOptions; +import org.tron.common.storage.SourceInterRocks; +import org.tron.common.storage.leveldb.RocksDbDataSourceImpl; +import org.tron.common.utils.FileUtil; +import org.tron.common.utils.Utils; +import org.tron.core.config.args.Args; +import org.tron.core.db2.common.IRevokingDB; +import org.tron.core.db2.core.ISession; +import org.tron.core.db2.core.RevokingRocksDBWithCachingOldValue; +import org.tron.core.exception.RevokingStoreIllegalStateException; + +@Slf4j +public class RevokingStoreRocks implements RevokingDatabase { + + static { + RocksDB.loadLibrary(); + } + + private static final int DEFAULT_STACK_MAX_SIZE = 256; + private Deque stack = new LinkedList<>(); + private boolean disabled = true; + private int activeDialog = 0; + private AtomicInteger maxSize = new AtomicInteger(DEFAULT_STACK_MAX_SIZE); + private WriteOptions writeOptions = new WriteOptions().setSync(true); + + @Getter + private List dbs = new ArrayList<>(); + + private static volatile RevokingStoreRocks instance; + + private RevokingStoreRocks() { + } + + public static void releaseInstance() { + instance = null; + } + + public static RevokingStoreRocks getInstance() { + if (instance == null) { + synchronized (RevokingStoreRocks.class) { + if (instance == null) { + instance = new RevokingStoreRocks(); + } + } + } + return instance; + } + + @Override + public ISession buildSession() { + return buildSession(false); + } + + @Override + public ISession buildSession(boolean forceEnable) { + if (disabled && !forceEnable) { + return new Dialog(this); + } + + boolean disableOnExit = disabled && forceEnable; + if (forceEnable) { + disabled = false; + } + + while (stack.size() > maxSize.get()) { + stack.poll(); + } + + stack.add(new RevokingState()); + ++activeDialog; + return new Dialog(this, disableOnExit); + } + + + @Override + public synchronized void check() { + RocksDbDataSourceImpl check = + new RocksDbDataSourceImpl(Args.getInstance().getOutputDirectoryByDbName("tmp"), "tmp"); + check.initDB(); + + if (!check.allKeys().isEmpty()) { + Map dbMap = dbs.stream() + .map(db -> Maps.immutableEntry(db.getDBName(), db)) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + + for (Map.Entry e : check) { + byte[] key = e.getKey(); + byte[] value = e.getValue(); + String db = simpleDecode(key); + byte[] realKey = Arrays.copyOfRange(key, db.getBytes().length + 4, key.length); + + byte[] realValue = value.length == 1 ? null : Arrays.copyOfRange(value, 1, value.length); + if (realValue != null) { + dbMap.get(db).putData(realKey, realValue, new WriteOptions().setSync(true)); + } else { + dbMap.get(db).deleteData(realKey, new WriteOptions().setSync(true)); + } + } + } + + check.closeDB(); + FileUtil.recursiveDelete(check.getDbPath().toString()); + } + + public synchronized void onCreate(RevokingTuple tuple, byte[] value) { + if (disabled) { + return; + } + + addIfEmpty(); + RevokingState state = stack.peekLast(); + state.newIds.add(tuple); + } + + public synchronized void onModify(RevokingTuple tuple, byte[] value) { + if (disabled) { + return; + } + + addIfEmpty(); + RevokingState state = stack.peekLast(); + if (state.newIds.contains(tuple) || state.oldValues.containsKey(tuple)) { + return; + } + + state.oldValues.put(tuple, Utils.clone(value)); + } + + public synchronized void onRemove(RevokingTuple tuple, byte[] value) { + if (disabled) { + return; + } + + addIfEmpty(); + RevokingState state = stack.peekLast(); + if (state.newIds.contains(tuple)) { + state.newIds.remove(tuple); + return; + } + + if (state.oldValues.containsKey(tuple)) { + state.removed.put(tuple, state.oldValues.get(tuple)); + state.oldValues.remove(tuple); + return; + } + + if (state.removed.containsKey(tuple)) { + return; + } + + state.removed.put(tuple, Utils.clone(value)); + } + + @Override + public void add(IRevokingDB revokingDB) { + dbs.add(((RevokingRocksDBWithCachingOldValue) revokingDB).getDbSource()); + + } + + @Override + public void merge() throws RevokingStoreIllegalStateException { + if (activeDialog <= 0) { + throw new RevokingStoreIllegalStateException("activeDialog has to be greater than 0"); + } + + if (activeDialog == 1 && stack.size() == 1) { + stack.pollLast(); + --activeDialog; + return; + } + + if (stack.size() < 2) { + return; + } + + RevokingState state = stack.peekLast(); + @SuppressWarnings("unchecked") + List list = (List) stack; + RevokingState prevState = list.get(stack.size() - 2); + + state.oldValues.entrySet().stream() + .filter(e -> !prevState.newIds.contains(e.getKey())) + .filter(e -> !prevState.oldValues.containsKey(e.getKey())) + .forEach(e -> prevState.oldValues.put(e.getKey(), e.getValue())); + + prevState.newIds.addAll(state.newIds); + + state.removed.entrySet().stream() + .filter(e -> { + boolean has = prevState.newIds.contains(e.getKey()); + if (has) { + prevState.newIds.remove(e.getKey()); + } + + return !has; + }) + .filter(e -> { + boolean has = prevState.oldValues.containsKey(e.getKey()); + if (has) { + prevState.removed.put(e.getKey(), prevState.oldValues.get(e.getKey())); + prevState.oldValues.remove(e.getKey()); + } + + return !has; + }) + .forEach(e -> prevState.removed.put(e.getKey(), e.getValue())); + + stack.pollLast(); + --activeDialog; + } + + @Override + public void revoke() throws RevokingStoreIllegalStateException { + if (disabled) { + return; + } + + if (activeDialog <= 0) { + throw new RevokingStoreIllegalStateException("activeDialog has to be greater than 0"); + } + + disabled = true; + + try { + RevokingState state = stack.peekLast(); + if (Objects.isNull(state)) { + return; + } + + state.oldValues.forEach((k, v) -> k.database.putData(k.key, v)); + state.newIds.forEach(e -> e.database.deleteData(e.key)); + state.removed.forEach((k, v) -> k.database.putData(k.key, v)); + stack.pollLast(); + } finally { + disabled = false; + } + --activeDialog; + } + + @Override + public void commit() throws RevokingStoreIllegalStateException { + if (activeDialog <= 0) { + throw new RevokingStoreIllegalStateException("activeDialog has to be greater than 0"); + } + + --activeDialog; + } + + @Override + public void pop() throws RevokingStoreIllegalStateException { + prune(writeOptions); + } + + @Override + public void fastPop() throws RevokingStoreIllegalStateException { + prune(new WriteOptions()); + } + + private synchronized void prune(WriteOptions options) { + if (activeDialog != 0) { + throw new RevokingStoreIllegalStateException("activeDialog has to be equal 0"); + } + + if (stack.isEmpty()) { + throw new RevokingStoreIllegalStateException("stack is empty"); + } + + disabled = true; + + try { + RevokingState state = stack.peekLast(); + state.oldValues.forEach((k, v) -> k.database.putData(k.key, v, options)); + state.newIds.forEach(e -> e.database.deleteData(e.key, options)); + state.removed.forEach((k, v) -> k.database.putData(k.key, v, options)); + stack.pollLast(); + } finally { + disabled = false; + } + } + + @Override + public synchronized void enable() { + disabled = false; + } + + @Override + public synchronized void disable() { + disabled = true; + } + + @Override + public void setMaxFlushCount(int maxFlushCount) { + + } + + private void addIfEmpty() { + if (stack.isEmpty()) { + stack.add(new RevokingStoreRocks.RevokingState()); + } + } + + @Override + public synchronized int size() { + return stack.size(); + } + + @Override + public void setMaxSize(int maxSize) { + this.maxSize.set(maxSize); + } + + public int getMaxSize() { + return maxSize.get(); + } + + public synchronized void shutdown() { + System.err.println("******** begin to pop revokingDb ********"); + System.err.println("******** before revokingDb size:" + size()); + try { + disable(); + boolean exit = false; + while (!exit) { + try { + commit(); + } catch (RevokingStoreIllegalStateException e) { + exit = true; + } + } + + while (true) { + try { + pop(); + } catch (RevokingStoreIllegalStateException e) { + break; + } + } + } catch (Exception e) { + System.err.println("******** failed to pop revokingStore. " + e); + } finally { + System.err.println("******** after revokingStore size:" + stack.size()); + System.err.println("******** after revokingStore contains:" + stack); + System.err.println("******** end to pop revokingStore ********"); + } + } + + @Slf4j + @Getter // only for unit test + public static class Dialog implements ISession { + + private RevokingDatabase revokingDatabase; + private boolean applyRevoking = true; + private boolean disableOnExit = false; + + public Dialog(RevokingStoreRocks.Dialog dialog) { + this.revokingDatabase = dialog.revokingDatabase; + this.applyRevoking = dialog.applyRevoking; + dialog.applyRevoking = false; + } + + public Dialog(RevokingDatabase revokingDatabase) { + this(revokingDatabase, false); + } + + public Dialog(RevokingDatabase revokingDatabase, boolean disableOnExit) { + this.revokingDatabase = revokingDatabase; + this.disableOnExit = disableOnExit; + } + + @Override + public void commit() { + applyRevoking = false; + revokingDatabase.commit(); + } + + @Override + public void revoke() { + if (applyRevoking) { + revokingDatabase.revoke(); + } + + applyRevoking = false; + } + + @Override + public void merge() { + if (applyRevoking) { + revokingDatabase.merge(); + } + + applyRevoking = false; + } + + void copy(RevokingStoreRocks.Dialog dialog) { + if (this.equals(dialog)) { + return; + } + + if (applyRevoking) { + revokingDatabase.revoke(); + } + applyRevoking = dialog.applyRevoking; + dialog.applyRevoking = false; + } + + @Override + public void destroy() { + try { + if (applyRevoking) { + revokingDatabase.revoke(); + } + } catch (Exception e) { + logger.error("revoke database error.", e); + } + if (disableOnExit) { + revokingDatabase.disable(); + } + } + + @Override + public void close() { + try { + if (applyRevoking) { + revokingDatabase.revoke(); + } + } catch (Exception e) { + logger.error("revoke database error.", e); + throw new RevokingStoreIllegalStateException(e); + } + if (disableOnExit) { + revokingDatabase.disable(); + } + } + } + + @ToString + @Getter // only for unit test + static class RevokingState { + + Map oldValues = new HashMap<>(); + Set newIds = new HashSet<>(); + Map removed = new HashMap<>(); + } + + @AllArgsConstructor + @EqualsAndHashCode + @Getter + @ToString + public static class RevokingTuple { + + private SourceInterRocks database; + private byte[] key; + } + +} \ No newline at end of file diff --git a/src/main/java/org/tron/core/db/TronStoreWithRevoking.java b/src/main/java/org/tron/core/db/TronStoreWithRevoking.java index 55259b732dc..d7b0a0dc218 100755 --- a/src/main/java/org/tron/core/db/TronStoreWithRevoking.java +++ b/src/main/java/org/tron/core/db/TronStoreWithRevoking.java @@ -21,14 +21,18 @@ import org.tron.core.db2.core.ITronChainBase; import org.tron.core.db2.core.RevokingDBWithCachingNewValue; import org.tron.core.db2.core.RevokingDBWithCachingOldValue; +import org.tron.core.db2.core.RevokingRocksDBWithCachingOldValue; import org.tron.core.exception.BadItemException; import org.tron.core.exception.ItemNotFoundException; @Slf4j(topic = "DB") public abstract class TronStoreWithRevoking implements ITronChainBase { + @Getter // only for unit test protected IRevokingDB revokingDB; - private TypeToken token = new TypeToken(getClass()) {}; + private TypeToken token = new TypeToken(getClass()) { + }; + @Autowired private RevokingDatabase revokingDatabase; @Autowired(required = false) @@ -43,6 +47,8 @@ protected TronStoreWithRevoking(String dbName) { this.revokingDB = new RevokingDBWithCachingOldValue(dbName); } else if (dbVersion == 2) { this.revokingDB = new RevokingDBWithCachingNewValue(dbName); + } else if (dbVersion == 3) { + this.revokingDB = new RevokingRocksDBWithCachingOldValue(dbName); } else { throw new RuntimeException("db version is error."); } @@ -65,7 +71,8 @@ private void init() { // only for test protected TronStoreWithRevoking(String dbName, RevokingDatabase revokingDatabase) { - this.revokingDB = new RevokingDBWithCachingOldValue(dbName, (AbstractRevokingStore) revokingDatabase); + this.revokingDB = new RevokingDBWithCachingOldValue(dbName, + (AbstractRevokingStore) revokingDatabase); } @Override @@ -147,5 +154,4 @@ public long size() { public void setMode(boolean mode) { revokingDB.setMode(mode); } - } diff --git a/src/main/java/org/tron/core/db/backup/BackupDbUtil.java b/src/main/java/org/tron/core/db/backup/BackupDbUtil.java new file mode 100644 index 00000000000..514b7ce158a --- /dev/null +++ b/src/main/java/org/tron/core/db/backup/BackupDbUtil.java @@ -0,0 +1,151 @@ +package org.tron.core.db.backup; + +import java.util.List; +import lombok.Getter; +import lombok.Setter; +import lombok.extern.slf4j.Slf4j; +import org.rocksdb.RocksDBException; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; +import org.tron.common.storage.leveldb.RocksDbDataSourceImpl; +import org.tron.common.utils.PropUtil; +import org.tron.core.capsule.BlockCapsule; +import org.tron.core.config.args.Args; +import org.tron.core.db.RevokingDatabase; +import org.tron.core.db.RevokingStoreRocks; + +@Slf4j +@Component +public class BackupDbUtil { + + @Getter + private static String DB_BACKUP_STATE = "DB"; + private static final int DB_BACKUP_INDEX1 = 1; + private static final int DB_BACKUP_INDEX2 = 2; + + @Getter + private static final int DB_BACKUP_STATE_DEFAULT = 11; + + public enum STATE { + BAKINGONE(1), BAKEDONE(11), BAKINGTWO(2), BAKEDTWO(22); + public int status; + + private STATE(int status) { + this.status = status; + } + + public int getStatus() { + return status; + } + + public static STATE valueOf(int value) { + switch (value) { + case 1: + return BAKINGONE; + case 11: + return BAKEDONE; + case 2: + return BAKINGTWO; + case 22: + return BAKEDTWO; + default: + return BAKEDONE; + } + } + } + + @Getter + @Autowired + private RevokingDatabase db; + + private Args args = Args.getInstance(); + + private int getBackupState() { + try { + return Integer.valueOf(PropUtil + .readProperty(args.getDbBackupConfig().getPropPath(), BackupDbUtil.DB_BACKUP_STATE) + ); + } catch (NumberFormatException ignore) { + return DB_BACKUP_STATE_DEFAULT; //get default state if prop file is newly created + } + } + + private void setBackupState(int status) { + PropUtil.writeProperty(args.getDbBackupConfig().getPropPath(), BackupDbUtil.DB_BACKUP_STATE, + String.valueOf(status)); + } + + private void switchBackupState() { + switch (STATE.valueOf(getBackupState())) { + case BAKINGONE: + setBackupState(STATE.BAKEDONE.getStatus()); + break; + case BAKEDONE: + setBackupState(STATE.BAKEDTWO.getStatus()); + break; + case BAKINGTWO: + setBackupState(STATE.BAKEDTWO.getStatus()); + break; + case BAKEDTWO: + setBackupState(STATE.BAKEDONE.getStatus()); + break; + default: + break; + } + } + + public void doBackup(BlockCapsule block) { + if (block.getNum() % args.getDbBackupConfig().getFrequency() != 0) { + return; + } + long t1 = System.currentTimeMillis(); + try { + switch (STATE.valueOf(getBackupState())) { + case BAKINGONE: + deleteBackup(DB_BACKUP_INDEX1); + backup(DB_BACKUP_INDEX1); + switchBackupState(); + deleteBackup(DB_BACKUP_INDEX2); + break; + case BAKEDONE: + deleteBackup(DB_BACKUP_INDEX2); + backup(DB_BACKUP_INDEX2); + switchBackupState(); + deleteBackup(DB_BACKUP_INDEX1); + break; + case BAKINGTWO: + deleteBackup(DB_BACKUP_INDEX2); + backup(DB_BACKUP_INDEX2); + switchBackupState(); + deleteBackup(DB_BACKUP_INDEX1); + break; + case BAKEDTWO: + deleteBackup(DB_BACKUP_INDEX1); + backup(DB_BACKUP_INDEX1); + switchBackupState(); + deleteBackup(DB_BACKUP_INDEX2); + break; + default: + logger.warn("invalid backup state"); + } + } catch (RocksDBException e) { + logger.warn("backup db error"); + } + logger.info("current block number is {}, backup all store use {} ms!", block.getNum(), + System.currentTimeMillis() - t1); + } + + private void backup(int i) throws RocksDBException { + List stores = ((RevokingStoreRocks) db).getDbs(); + for (RocksDbDataSourceImpl store : stores) { + store.backup(i); + } + } + + private void deleteBackup(int i) { + List stores = ((RevokingStoreRocks) db).getDbs(); + for (RocksDbDataSourceImpl store : stores) { + store.deleteDbBakPath(i); + } + } +} diff --git a/src/main/java/org/tron/core/db/backup/BackupRocksDBAspect.java b/src/main/java/org/tron/core/db/backup/BackupRocksDBAspect.java new file mode 100644 index 00000000000..64cfc4dd1f6 --- /dev/null +++ b/src/main/java/org/tron/core/db/backup/BackupRocksDBAspect.java @@ -0,0 +1,36 @@ +package org.tron.core.db.backup; + +import lombok.extern.slf4j.Slf4j; +import org.aspectj.lang.annotation.AfterThrowing; +import org.aspectj.lang.annotation.Aspect; +import org.aspectj.lang.annotation.Before; +import org.aspectj.lang.annotation.Pointcut; +import org.springframework.beans.factory.annotation.Autowired; +import org.tron.core.capsule.BlockCapsule; + +@Slf4j +@Aspect +public class BackupRocksDBAspect { + + @Autowired + private BackupDbUtil util; + + @Pointcut("execution(** org.tron.core.db.Manager.pushBlock(..)) && args(block)") + public void pointPushBlock(BlockCapsule block) { + + } + + @Before("pointPushBlock(block)") + public void backupDb(BlockCapsule block) { + try { + util.doBackup(block); + } catch (Exception e) { + logger.error("backup db failure: {}", e); + } + } + + @AfterThrowing("pointPushBlock(block)") + public void logErrorPushBlock(BlockCapsule block) { + logger.info("AfterThrowing pushBlock"); + } +} \ No newline at end of file diff --git a/src/main/java/org/tron/core/db/backup/DbBackupConfig.java b/src/main/java/org/tron/core/db/backup/DbBackupConfig.java new file mode 100644 index 00000000000..fd4ac4e3ed6 --- /dev/null +++ b/src/main/java/org/tron/core/db/backup/DbBackupConfig.java @@ -0,0 +1,77 @@ +package org.tron.core.db.backup; + +import java.io.File; +import lombok.Getter; +import lombok.Setter; +import lombok.extern.slf4j.Slf4j; +import org.tron.common.utils.FileUtil; + +@Slf4j +public class DbBackupConfig { + + @Getter + @Setter + private String propPath; + + @Getter + @Setter + private String bak1path; + + @Getter + @Setter + private String bak2path; + + @Getter + @Setter + private int frequency; + + @Getter + @Setter + private boolean enable = true; + + private static volatile DbBackupConfig instance; + + // singleton + public static DbBackupConfig getInstance() { + if (instance == null) { + synchronized (DbBackupConfig.class) { + if (instance == null) { + instance = new DbBackupConfig(); + } + } + } + return instance; + } + + public DbBackupConfig initArgs(boolean enable, String propPath, String bak1path, String bak2path, + int frequency) { + setEnable(enable); + if (enable) { + if (!bak1path.endsWith(File.separator)) { + bak1path = bak1path + File.separator; + } + + if (!bak2path.endsWith(File.separator)) { + bak2path = bak2path + File.separator; + } + + boolean flag = + FileUtil.createFileIfNotExists(propPath) && FileUtil.createDirIfNotExists(bak1path) + && FileUtil.createDirIfNotExists(bak2path); + + if (!flag) { + logger.warn("fail to enable the db backup plugin"); + } else { + setPropPath(propPath); + setBak1path(bak1path); + setBak2path(bak2path); + setFrequency(frequency); + logger.info( + "success to enable the db backup plugin. bak1path:{}, bak2path:{}, backup once every {} blocks handled", + bak1path, bak2path, frequency); + } + } + + return this; + } +} \ No newline at end of file diff --git a/src/main/java/org/tron/core/db/backup/NeedBeanCondition.java b/src/main/java/org/tron/core/db/backup/NeedBeanCondition.java new file mode 100644 index 00000000000..a9e222bf8b0 --- /dev/null +++ b/src/main/java/org/tron/core/db/backup/NeedBeanCondition.java @@ -0,0 +1,15 @@ +package org.tron.core.db.backup; + +import org.springframework.context.annotation.Condition; +import org.springframework.context.annotation.ConditionContext; +import org.springframework.core.type.AnnotatedTypeMetadata; +import org.tron.core.config.args.Args; + +public class NeedBeanCondition implements Condition { + + @Override + public boolean matches(ConditionContext context, AnnotatedTypeMetadata metadata) { + return (Args.getInstance().getStorage().getDbVersion() == 3) && Args.getInstance() + .getDbBackupConfig().isEnable(); + } +} \ No newline at end of file diff --git a/src/main/java/org/tron/core/db/common/iterator/RockStoreIterator.java b/src/main/java/org/tron/core/db/common/iterator/RockStoreIterator.java new file mode 100644 index 00000000000..dd417848597 --- /dev/null +++ b/src/main/java/org/tron/core/db/common/iterator/RockStoreIterator.java @@ -0,0 +1,72 @@ +package org.tron.core.db.common.iterator; + +import java.io.IOException; +import java.util.Map.Entry; +import java.util.NoSuchElementException; +import lombok.extern.slf4j.Slf4j; +import org.rocksdb.RocksIterator; + +@Slf4j +public final class RockStoreIterator implements DBIterator { + + private RocksIterator dbIterator; + private boolean first = true; + + public RockStoreIterator(RocksIterator dbIterator) { + this.dbIterator = dbIterator; + } + + @Override + public void close() throws IOException { + dbIterator.close(); + } + + @Override + public boolean hasNext() { + boolean hasNext = false; + // true is first item + try { + if (first) { + dbIterator.seekToFirst(); + first = false; + } + if (!(hasNext = dbIterator.isValid())) { // false is last item + dbIterator.close(); + } + } catch (Exception e) { + System.out.println("e:" + e); + try { + dbIterator.close(); + } catch (Exception e1) { + System.out.println("e1:" + e1); + } + } + return hasNext; + } + + @Override + public Entry next() { + if (!dbIterator.isValid()) { + throw new NoSuchElementException(); + } + byte[] key = dbIterator.key(); + byte[] value = dbIterator.value(); + dbIterator.next(); + return new Entry() { + @Override + public byte[] getKey() { + return key; + } + + @Override + public byte[] getValue() { + return value; + } + + @Override + public byte[] setValue(byte[] value) { + throw new UnsupportedOperationException(); + } + }; + } +} \ No newline at end of file diff --git a/src/main/java/org/tron/core/db2/core/RevokingRocksDBWithCachingOldValue.java b/src/main/java/org/tron/core/db2/core/RevokingRocksDBWithCachingOldValue.java new file mode 100644 index 00000000000..2d791e56f92 --- /dev/null +++ b/src/main/java/org/tron/core/db2/core/RevokingRocksDBWithCachingOldValue.java @@ -0,0 +1,163 @@ +package org.tron.core.db2.core; + +import java.util.Iterator; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Objects; +import java.util.Set; +import java.util.Spliterator; +import java.util.function.Consumer; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.ArrayUtils; +import org.tron.common.storage.leveldb.RocksDbDataSourceImpl; +import org.tron.core.config.args.Args; +import org.tron.core.db.AbstractRevokingStore; +import org.tron.core.db.RevokingStore; +import org.tron.core.db.RevokingStoreRocks; +import org.tron.core.db.RevokingStoreRocks.RevokingTuple; +import org.tron.core.db2.common.IRevokingDB; +import org.tron.core.exception.ItemNotFoundException; + +@Slf4j +@NoArgsConstructor +public class RevokingRocksDBWithCachingOldValue implements IRevokingDB { + + private RevokingStoreRocks revokingDatabase; + + @Getter + private RocksDbDataSourceImpl dbSource; + + public RevokingRocksDBWithCachingOldValue(String dbName) { + this(dbName, RevokingStoreRocks.getInstance()); + } + + // only for unit test + public RevokingRocksDBWithCachingOldValue(String dbName, RevokingStoreRocks revokingDatabase) { + dbSource = new RocksDbDataSourceImpl(Args.getInstance().getOutputDirectoryByDbName(dbName), + dbName); + dbSource.initDB(); + this.revokingDatabase = revokingDatabase; + } + + @Override + public void put(byte[] key, byte[] value) { + if (Objects.isNull(key) || Objects.isNull(value)) { + return; + } + byte[] oldValue = dbSource.getData(key); + if (ArrayUtils.isNotEmpty(oldValue)) { + onModify(key, oldValue); + } + + dbSource.putData(key, value); + + if (ArrayUtils.isEmpty(oldValue)) { + onCreate(key); + } + } + + @Override + public void delete(byte[] key) { + onDelete(key); + dbSource.deleteData(key); + } + + @Override + public boolean hasOnSolidity(byte[] key) { + return false; + } + + @Override + public boolean has(byte[] key) { + return dbSource.getData(key) != null; + } + + @Override + public byte[] get(byte[] key) throws ItemNotFoundException { + byte[] value = dbSource.getData(key); + if (ArrayUtils.isEmpty(value)) { + throw new ItemNotFoundException(); + } + return value; + } + + @Override + public byte[] getUnchecked(byte[] key) { + try { + return get(key); + } catch (ItemNotFoundException e) { + return null; + } + } + + @Override + public byte[] getOnSolidity(byte[] key) throws ItemNotFoundException { + return new byte[0]; + } + + @Override + public byte[] getUncheckedOnSolidity(byte[] key) { + return new byte[0]; + } + + @Override + public void close() { + dbSource.closeDB(); + } + + @Override + public void reset() { + dbSource.resetDb(); + } + + /** + * This should be called just after an object is created + */ + private void onCreate(byte[] key) { + revokingDatabase.onCreate(new RevokingTuple(dbSource, key), null); + } + + /** + * This should be called just before an object is modified + */ + private void onModify(byte[] key, byte[] value) { + revokingDatabase.onModify(new RevokingTuple(dbSource, key), value); + } + + /** + * This should be called just before an object is removed. + */ + private void onDelete(byte[] key) { + byte[] value; + if (Objects.nonNull(value = dbSource.getData(key))) { + revokingDatabase.onRemove(new RevokingTuple(dbSource, key), value); + } + } + + @Override + public Iterator> iterator() { + return dbSource.iterator(); + } + + @Override + public void forEach(Consumer> action) { + + } + + @Override + public Spliterator> spliterator() { + return null; + } + + @Override + public Set getlatestValues(long limit) { + return dbSource.getlatestValues(limit); + } + + @Override + public Set getValuesNext(byte[] key, long limit) { + return dbSource.getValuesNext(key, limit); + } +} diff --git a/src/main/java/org/tron/core/net/node/NodeImpl.java b/src/main/java/org/tron/core/net/node/NodeImpl.java index e76890b2f22..160d528328b 100644 --- a/src/main/java/org/tron/core/net/node/NodeImpl.java +++ b/src/main/java/org/tron/core/net/node/NodeImpl.java @@ -562,7 +562,8 @@ private synchronized void handleSyncBlock() { final boolean[] isFound = {false}; getActivePeer().stream() .filter( - peer -> !peer.getSyncBlockToFetch().isEmpty() && peer.getSyncBlockToFetch().peek() + peer -> !peer.getSyncBlockToFetch().isEmpty() && peer.getSyncBlockToFetch() + .peek() .equals(msg.getBlockId())) .forEach(peer -> { peer.getSyncBlockToFetch().pop(); @@ -579,7 +580,6 @@ private synchronized void handleSyncBlock() { } } }); - } } @@ -1349,19 +1349,38 @@ public void onDisconnectPeer(PeerConnection peer) { } } + private void shutdownExecutor(ExecutorService exec, String name) { + exec.shutdown(); + try { + if (!exec.awaitTermination(10, TimeUnit.SECONDS)) { + exec.shutdownNow(); + } + } catch (InterruptedException e) { + e.printStackTrace(); + } finally { + if (!exec.isTerminated()) { + logger.warn("fail to shutdown {} ", name); + } else { + logger.info("success to shutdown {} ", name); + } + } + } + public void shutDown() { - logExecutor.shutdown(); - trxsHandlePool.shutdown(); - disconnectInactiveExecutor.shutdown(); - cleanInventoryExecutor.shutdown(); - broadPool.shutdown(); + logger.info("begin shutdown nodeimpl"); + shutdownExecutor(logExecutor, "logExecutor"); + shutdownExecutor(trxsHandlePool, "trxsHandlePool"); + shutdownExecutor(disconnectInactiveExecutor, "disconnectInactiveExecutor"); + shutdownExecutor(cleanInventoryExecutor, "cleanInventoryExecutor"); + shutdownExecutor(broadPool, "broadPool"); + shutdownExecutor(fetchSyncBlocksExecutor, "fetchSyncBlocksExecutor"); + shutdownExecutor(handleSyncBlockExecutor, "handleSyncBlockExecutor"); loopSyncBlockChain.shutdown(); loopFetchBlocks.shutdown(); loopAdvertiseInv.shutdown(); - fetchSyncBlocksExecutor.shutdown(); - handleSyncBlockExecutor.shutdown(); } + private void disconnectPeer(PeerConnection peer, ReasonCode reason) { peer.setSyncFlag(false); peer.disconnect(reason); diff --git a/src/main/resources/config-dbbackup.conf b/src/main/resources/config-dbbackup.conf new file mode 100644 index 00000000000..3008892b54b --- /dev/null +++ b/src/main/resources/config-dbbackup.conf @@ -0,0 +1,411 @@ +net { + type = mainnet + # type = testnet +} + +storage { + # Directory for storing persistent data + db.version = 3, + db.directory = "database", + index.directory = "index", + + # You can custom these 14 databases' configs: + + # account, account-index, asset-issue, block, block-index, + # block_KDB, peers, properties, recent-block, trans, + # utxo, votes, witness, witness_schedule. + + # Otherwise, db configs will remain defualt and data will be stored in + # the path of "output-directory" or which is set by "-d" ("--output-directory"). + + # Attention: name is a required field that must be set !!! + properties = [ +// { +// name = "account", +// path = "storage_directory_test", +// createIfMissing = true, +// paranoidChecks = true, +// verifyChecksums = true, + // compressionType = 1, // compressed with snappy + // blockSize = 4096, // 4 KB = 4 * 1024 B + // writeBufferSize = 10485760, // 10 MB = 10 * 1024 * 1024 B + // cacheSize = 10485760, // 10 MB = 10 * 1024 * 1024 B + // maxOpenFiles = 100 + // }, +// { +// name = "account-index", +// path = "storage_directory_test", +// createIfMissing = true, +// paranoidChecks = true, +// verifyChecksums = true, + // compressionType = 1, // compressed with snappy + // blockSize = 4096, // 4 KB = 4 * 1024 B + // writeBufferSize = 10485760, // 10 MB = 10 * 1024 * 1024 B + // cacheSize = 10485760, // 10 MB = 10 * 1024 * 1024 B + // maxOpenFiles = 100 + // }, + ] + + needToUpdateAsset = true + + backup = { + enable = true + properties = "prop.properties" + bak1path = "bak1/database" + bak2path = "bak2/database" + frequency = 5000 // backup db every ? blocks processed. + } +} + +node.discovery = { + enable = true + persist = true + bind.ip = "" + external.ip = null +} + +node.backup { + port = 10001 + + # my priority, each member should use different priority + priority = 8 + + # peer's ip list, can't contain mine + members = [ + # "ip", + # "ip" + ] +} + +node { + # trust node for solidity node + # trustNode = "ip:port" + trustNode = "127.0.0.1:50051" + + # expose extension api to public or not + walletExtensionApi = true + + listen.port = 18888 + + connection.timeout = 2 + + tcpNettyWorkThreadNum = 0 + + udpNettyWorkThreadNum = 1 + + # Number of validate sign thread, default availableProcessors / 2 + # validateSignThreadNum = 16 + + connectFactor = 0.3 + activeConnectFactor = 0.1 + + maxActiveNodes = 30 + + maxActiveNodesWithSameIp = 2 + + minParticipationRate = 15 + + # check the peer data transfer ,disconnect factor + disconnectNumberFactor = 0.4 + maxConnectNumberFactor = 0.8 + receiveTcpMinDataLength = 2048 + isOpenFullTcpDisconnect = true + + p2p { + version = 11111 # 11111: mainnet; 20180622: testnet + } + + active = [ + # Active establish connection in any case + # Sample entries: + # "ip:port", + # "ip:port" + ] + + passive = [ + # Passive accept connection in any case + # Sample entries: + # "ip:port", + # "ip:port" + ] + + http { + fullNodePort = 8090 + solidityPort = 8091 + } + + rpc { + port = 50051 + #solidityPort = 50061 + # Number of gRPC thread, default availableProcessors / 2 + # thread = 16 + + # The maximum number of concurrent calls permitted for each incoming connection + # maxConcurrentCallsPerConnection = + + # The HTTP/2 flow control window, default 1MB + # flowControlWindow = + + # Connection being idle for longer than which will be gracefully terminated + maxConnectionIdleInMillis = 60000 + + # Connection lasting longer than which will be gracefully terminated + # maxConnectionAgeInMillis = + + # The maximum message size allowed to be received on the server, default 4MB + # maxMessageSize = + + # The maximum size of header list allowed to be received, default 8192 + # maxHeaderListSize = + } + + # Limits the maximum percentage (default 75%) of producing block interval + # to provide sufficient time to perform other operations e.g. broadcast block + # blockProducedTimeOut = 75 + + # Limits the maximum number (default 700) of transaction from network layer + # netMaxTrxPerSecond = 700 +} + + + +seed.node = { + # List of the seed nodes + # Seed nodes are stable full nodes + # example: + # ip.list = [ + # "ip:port", + # "ip:port" + # ] + ip.list = [ + "54.236.37.243:18888", + "52.53.189.99:18888", + "18.196.99.16:18888", + "34.253.187.192:18888", + "52.56.56.149:18888", + "35.180.51.163:18888", + "54.252.224.209:18888", + "18.228.15.36:18888", + "52.15.93.92:18888", + "34.220.77.106:18888", + "13.127.47.162:18888", + "13.124.62.58:18888", + "13.229.128.108:18888", + "35.182.37.246:18888", + "34.200.228.125:18888", + "18.220.232.201:18888", + "13.57.30.186:18888", + "35.165.103.105:18888", + "18.184.238.21:18888", + "34.250.140.143:18888", + "35.176.192.130:18888", + "52.47.197.188:18888", + "52.62.210.100:18888", + "13.231.4.243:18888", + "18.231.76.29:18888", + "35.154.90.144:18888", + "13.125.210.234:18888", + "13.250.40.82:18888", + "35.183.101.48:18888" + ] +} + +genesis.block = { + # Reserve balance + assets = [ + { + accountName = "Zion" + accountType = "AssetIssue" + address = "TLLM21wteSPs4hKjbxgmH1L6poyMjeTbHm" + balance = "99000000000000000" + }, + { + accountName = "Sun" + accountType = "AssetIssue" + address = "TXmVpin5vq5gdZsciyyjdZgKRUju4st1wM" + balance = "0" + }, + { + accountName = "Blackhole" + accountType = "AssetIssue" + address = "TLsV52sRDL79HXGGm9yzwKibb6BeruhUzy" + balance = "-9223372036854775808" + } + ] + + witnesses = [ + { + address: THKJYuUmMKKARNf7s2VT51g5uPY6KEqnat, + url = "http://GR1.com", + voteCount = 100000026 + }, + { + address: TVDmPWGYxgi5DNeW8hXrzrhY8Y6zgxPNg4, + url = "http://GR2.com", + voteCount = 100000025 + }, + { + address: TWKZN1JJPFydd5rMgMCV5aZTSiwmoksSZv, + url = "http://GR3.com", + voteCount = 100000024 + }, + { + address: TDarXEG2rAD57oa7JTK785Yb2Et32UzY32, + url = "http://GR4.com", + voteCount = 100000023 + }, + { + address: TAmFfS4Tmm8yKeoqZN8x51ASwdQBdnVizt, + url = "http://GR5.com", + voteCount = 100000022 + }, + { + address: TK6V5Pw2UWQWpySnZyCDZaAvu1y48oRgXN, + url = "http://GR6.com", + voteCount = 100000021 + }, + { + address: TGqFJPFiEqdZx52ZR4QcKHz4Zr3QXA24VL, + url = "http://GR7.com", + voteCount = 100000020 + }, + { + address: TC1ZCj9Ne3j5v3TLx5ZCDLD55MU9g3XqQW, + url = "http://GR8.com", + voteCount = 100000019 + }, + { + address: TWm3id3mrQ42guf7c4oVpYExyTYnEGy3JL, + url = "http://GR9.com", + voteCount = 100000018 + }, + { + address: TCvwc3FV3ssq2rD82rMmjhT4PVXYTsFcKV, + url = "http://GR10.com", + voteCount = 100000017 + }, + { + address: TFuC2Qge4GxA2U9abKxk1pw3YZvGM5XRir, + url = "http://GR11.com", + voteCount = 100000016 + }, + { + address: TNGoca1VHC6Y5Jd2B1VFpFEhizVk92Rz85, + url = "http://GR12.com", + voteCount = 100000015 + }, + { + address: TLCjmH6SqGK8twZ9XrBDWpBbfyvEXihhNS, + url = "http://GR13.com", + voteCount = 100000014 + }, + { + address: TEEzguTtCihbRPfjf1CvW8Euxz1kKuvtR9, + url = "http://GR14.com", + voteCount = 100000013 + }, + { + address: TZHvwiw9cehbMxrtTbmAexm9oPo4eFFvLS, + url = "http://GR15.com", + voteCount = 100000012 + }, + { + address: TGK6iAKgBmHeQyp5hn3imB71EDnFPkXiPR, + url = "http://GR16.com", + voteCount = 100000011 + }, + { + address: TLaqfGrxZ3dykAFps7M2B4gETTX1yixPgN, + url = "http://GR17.com", + voteCount = 100000010 + }, + { + address: TX3ZceVew6yLC5hWTXnjrUFtiFfUDGKGty, + url = "http://GR18.com", + voteCount = 100000009 + }, + { + address: TYednHaV9zXpnPchSywVpnseQxY9Pxw4do, + url = "http://GR19.com", + voteCount = 100000008 + }, + { + address: TCf5cqLffPccEY7hcsabiFnMfdipfyryvr, + url = "http://GR20.com", + voteCount = 100000007 + }, + { + address: TAa14iLEKPAetX49mzaxZmH6saRxcX7dT5, + url = "http://GR21.com", + voteCount = 100000006 + }, + { + address: TBYsHxDmFaRmfCF3jZNmgeJE8sDnTNKHbz, + url = "http://GR22.com", + voteCount = 100000005 + }, + { + address: TEVAq8dmSQyTYK7uP1ZnZpa6MBVR83GsV6, + url = "http://GR23.com", + voteCount = 100000004 + }, + { + address: TRKJzrZxN34YyB8aBqqPDt7g4fv6sieemz, + url = "http://GR24.com", + voteCount = 100000003 + }, + { + address: TRMP6SKeFUt5NtMLzJv8kdpYuHRnEGjGfe, + url = "http://GR25.com", + voteCount = 100000002 + }, + { + address: TDbNE1VajxjpgM5p7FyGNDASt3UVoFbiD3, + url = "http://GR26.com", + voteCount = 100000001 + }, + { + address: TLTDZBcPoJ8tZ6TTEeEqEvwYFk2wgotSfD, + url = "http://GR27.com", + voteCount = 100000000 + } + ] + + timestamp = "0" #2017-8-26 12:00:00 + + parentHash = "0xe58f33f9baf9305dc6f82b9f1934ea8f0ade2defb951258d50167028c780351f" +} + +localwitness = [ +] + +#localwitnesskeystore = [ +# "localwitnesskeystore.json" +#] + +block = { + needSyncCheck = true + maintenanceTimeInterval = 21600000 + proposalExpireTime = 259200000 // 3 day: 259200000(ms) +} + +# Transaction reference block, default is "head", configure to "solid" can avoid TaPos error +# trx.reference.block = "head" // head;solid; + +vm = { + supportConstant = false + minTimeRatio = 0.0 + maxTimeRatio = 5.0 + + # In rare cases, transactions that will be within the specified maximum execution time (default 10(ms)) are re-executed and packaged + # longRunningTime = 10 +} + +committee = { + allowCreationOfContracts = 0 //mainnet:0 (reset by committee),test:1 + allowAdaptiveEnergy = 0 //mainnet:0 (reset by committee),test:1 +} + +log.level = { + root = "INFO" // TRACE;DEBUG;INFO;WARN;ERROR +} diff --git a/src/main/resources/config.conf b/src/main/resources/config.conf index 9771a36ab05..fbd9e25d5cd 100644 --- a/src/main/resources/config.conf +++ b/src/main/resources/config.conf @@ -5,8 +5,7 @@ net { storage { # Directory for storing persistent data - db.version = 2, - db.sync = false, + db.version = 3, db.directory = "database", index.directory = "index", transHistory.switch = "on", @@ -48,9 +47,17 @@ storage { // }, ] + needToUpdateAsset = true - needToUpdateAsset = true - + //backup settings when use rocks db as the storage implement (db.version=3). + //if you want to use the backup plugin, please confirm set the db.version=3 above. + backup = { + enable = true // indicate whether enable the backup plugin + properties = "prop.properties" // record which bak directory is valid + bak1path = "bak1/database" // you must set two backup directories to prevent application halt unexpected(e.g. kill -9). + bak2path = "bak2/database" + frequency = 10000 // indicate backup db once every 10000 blocks processed. + } } node.discovery = { diff --git a/src/main/resources/logback.xml b/src/main/resources/logback.xml index ee4a685e636..8969464a6a2 100644 --- a/src/main/resources/logback.xml +++ b/src/main/resources/logback.xml @@ -3,16 +3,16 @@ - + - - - - - - - - + + + %d{HH:mm:ss.SSS} %-5level [%t] [%c{1}]\(%F:%L\) %m%n + + + INFO + + diff --git a/src/test/java/org/tron/common/storage/leveldb/RocksDbDataSourceImplTest.java b/src/test/java/org/tron/common/storage/leveldb/RocksDbDataSourceImplTest.java new file mode 100644 index 00000000000..ae36773adf3 --- /dev/null +++ b/src/test/java/org/tron/common/storage/leveldb/RocksDbDataSourceImplTest.java @@ -0,0 +1,253 @@ +package org.tron.common.storage.leveldb; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; + +import com.google.common.collect.Sets; +import java.io.File; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; +import lombok.extern.slf4j.Slf4j; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.tron.common.utils.ByteArray; +import org.tron.common.utils.FileUtil; +import org.tron.core.Constant; +import org.tron.core.config.args.Args; + +@Slf4j +public class RocksDbDataSourceImplTest { + + private static final String dbPath = "output-Rocks-test"; + private static RocksDbDataSourceImpl dataSourceTest; + + private byte[] value1 = "10000".getBytes(); + private byte[] value2 = "20000".getBytes(); + private byte[] value3 = "30000".getBytes(); + private byte[] value4 = "40000".getBytes(); + private byte[] value5 = "50000".getBytes(); + private byte[] value6 = "60000".getBytes(); + private byte[] key1 = "00000001aa".getBytes(); + private byte[] key2 = "00000002aa".getBytes(); + private byte[] key3 = "00000003aa".getBytes(); + private byte[] key4 = "00000004aa".getBytes(); + private byte[] key5 = "00000005aa".getBytes(); + private byte[] key6 = "00000006aa".getBytes(); + + @Before + public void initDb() { + Args.setParam(new String[]{"--output-directory", dbPath}, Constant.TEST_CONF); + dataSourceTest = new RocksDbDataSourceImpl(dbPath + File.separator, "test_rocksDb"); + } + + /** + * Release resources. + */ + @AfterClass + public static void destroy() { + Args.clearParam(); + if (FileUtil.deleteDir(new File(dbPath))) { + logger.info("Release resources successful."); + } else { + logger.info("Release resources failure."); + } + } + + @Test + public void testPutGet() { + dataSourceTest.resetDb(); + String key1 = "2c0937534dd1b3832d05d865e8e6f2bf23218300b33a992740d45ccab7d4f519"; + byte[] key = key1.getBytes(); + dataSourceTest.initDB(); + String value1 = "50000"; + byte[] value = value1.getBytes(); + + dataSourceTest.putData(key, value); + + assertNotNull(dataSourceTest.getData(key)); + assertEquals(1, dataSourceTest.allKeys().size()); + assertEquals("50000", ByteArray.toStr(dataSourceTest.getData(key1.getBytes()))); + dataSourceTest.closeDB(); + } + + @Test + public void testReset() { + RocksDbDataSourceImpl dataSource = new RocksDbDataSourceImpl( + Args.getInstance().getOutputDirectory(), "test_reset"); + dataSource.resetDb(); + assertEquals(0, dataSource.allKeys().size()); + dataSource.closeDB(); + } + + @Test + public void testupdateByBatchInner() { + RocksDbDataSourceImpl dataSource = new RocksDbDataSourceImpl( + Args.getInstance().getOutputDirectory(), "test_updateByBatch"); + dataSource.initDB(); + dataSource.resetDb(); + String key1 = "431cd8c8d5abe5cb5944b0889b32482d85772fbb98987b10fbb7f17110757350"; + String value1 = "50000"; + String key2 = "431cd8c8d5abe5cb5944b0889b32482d85772fbb98987b10fbb7f17110757351"; + String value2 = "10000"; + + Map rows = new HashMap<>(); + rows.put(key1.getBytes(), value1.getBytes()); + rows.put(key2.getBytes(), value2.getBytes()); + + dataSource.updateByBatch(rows); + + assertEquals("50000", ByteArray.toStr(dataSource.getData(key1.getBytes()))); + assertEquals("10000", ByteArray.toStr(dataSource.getData(key2.getBytes()))); + assertEquals(2, dataSource.allKeys().size()); + dataSource.closeDB(); + } + + @Test + public void testdeleteData() { + RocksDbDataSourceImpl dataSource = new RocksDbDataSourceImpl( + Args.getInstance().getOutputDirectory(), "test_delete"); + dataSource.initDB(); + String key1 = "431cd8c8d5abe5cb5944b0889b32482d85772fbb98987b10fbb7f17110757350"; + byte[] key = key1.getBytes(); + dataSource.deleteData(key); + byte[] value = dataSource.getData(key); + String s = ByteArray.toStr(value); + assertNull(s); + dataSource.closeDB(); + } + + @Test + public void testallKeys() { + RocksDbDataSourceImpl dataSource = new RocksDbDataSourceImpl( + Args.getInstance().getOutputDirectory(), "test_find_key"); + dataSource.initDB(); + dataSource.resetDb(); + + String key1 = "431cd8c8d5abe5cb5944b0889b32482d85772fbb98987b10fbb7f17110757321"; + byte[] key = key1.getBytes(); + + String value1 = "50000"; + byte[] value = value1.getBytes(); + + dataSource.putData(key, value); + String key3 = "431cd8c8d5abe5cb5944b0889b32482d85772fbb98987b10fbb7f17110757091"; + byte[] key2 = key3.getBytes(); + + String value3 = "30000"; + byte[] value2 = value3.getBytes(); + + dataSource.putData(key2, value2); + assertEquals(2, dataSource.allKeys().size()); + dataSource.resetDb(); + dataSource.closeDB(); + } + + @Test(timeout = 1000) + public void testLockReleased() { + dataSourceTest.initDB(); + // normal close + dataSourceTest.closeDB(); + // closing already closed db. + dataSourceTest.closeDB(); + // closing again to make sure the lock is free. If not test will hang. + dataSourceTest.closeDB(); + + assertFalse("Database is still alive after closing.", dataSourceTest.isAlive()); + } + + @Test + public void allKeysTest() { + RocksDbDataSourceImpl dataSource = new RocksDbDataSourceImpl( + Args.getInstance().getOutputDirectory(), "test_allKeysTest_key"); + dataSource.initDB(); + dataSource.resetDb(); + + byte[] key = "0000000987b10fbb7f17110757321".getBytes(); + byte[] value = "50000".getBytes(); + byte[] key2 = "000000431cd8c8d5a".getBytes(); + byte[] value2 = "30000".getBytes(); + + dataSource.putData(key, value); + dataSource.putData(key2, value2); + dataSource.allKeys().forEach(keyOne -> { + logger.info(ByteArray.toStr(keyOne)); + }); + assertEquals(2, dataSource.allKeys().size()); + dataSource.resetDb(); + dataSource.closeDB(); + } + + private void putSomeKeyValue(RocksDbDataSourceImpl dataSource) { + value1 = "10000".getBytes(); + value2 = "20000".getBytes(); + value3 = "30000".getBytes(); + value4 = "40000".getBytes(); + value5 = "50000".getBytes(); + value6 = "60000".getBytes(); + key1 = "00000001aa".getBytes(); + key2 = "00000002aa".getBytes(); + key3 = "00000003aa".getBytes(); + key4 = "00000004aa".getBytes(); + key5 = "00000005aa".getBytes(); + key6 = "00000006aa".getBytes(); + + dataSource.putData(key1, value1); + dataSource.putData(key6, value6); + dataSource.putData(key2, value2); + dataSource.putData(key5, value5); + dataSource.putData(key3, value3); + dataSource.putData(key4, value4); + } + + @Test + public void seekTest() { + RocksDbDataSourceImpl dataSource = new RocksDbDataSourceImpl( + Args.getInstance().getOutputDirectory(), "test_seek_key"); + dataSource.initDB(); + dataSource.resetDb(); + + putSomeKeyValue(dataSource); + dataSource.resetDb(); + dataSource.closeDB(); + } + + @Test + public void getValuesNext() { + RocksDbDataSourceImpl dataSource = new RocksDbDataSourceImpl( + Args.getInstance().getOutputDirectory(), "test_getValuesNext_key"); + dataSource.initDB(); + dataSource.resetDb(); + putSomeKeyValue(dataSource); + Set seekKeyLimitNext = dataSource.getValuesNext("0000000300".getBytes(), 2); + HashSet hashSet = Sets.newHashSet(ByteArray.toStr(value3), ByteArray.toStr(value4)); + seekKeyLimitNext.forEach( + value -> Assert.assertTrue("getValuesNext", hashSet.contains(ByteArray.toStr(value)))); + dataSource.resetDb(); + dataSource.closeDB(); + } + + @Test + public void getValuesPrev() { + RocksDbDataSourceImpl dataSource = new RocksDbDataSourceImpl( + Args.getInstance().getOutputDirectory(), "test_getValuesPrev_key"); + dataSource.initDB(); + dataSource.resetDb(); + + putSomeKeyValue(dataSource); + Set seekKeyLimitNext = dataSource.getValuesPrev("0000000300".getBytes(), 2); + HashSet hashSet = Sets.newHashSet(ByteArray.toStr(value1), ByteArray.toStr(value2)); + seekKeyLimitNext.forEach(value -> { + Assert.assertTrue("getValuesPrev1", hashSet.contains(ByteArray.toStr(value))); + }); + seekKeyLimitNext = dataSource.getValuesPrev("0000000100".getBytes(), 2); + Assert.assertEquals("getValuesPrev2", 0, seekKeyLimitNext.size()); + dataSource.resetDb(); + dataSource.closeDB(); + } +} diff --git a/src/test/java/org/tron/common/utils/FileUtilTest.java b/src/test/java/org/tron/common/utils/FileUtilTest.java new file mode 100644 index 00000000000..33d284fcdad --- /dev/null +++ b/src/test/java/org/tron/common/utils/FileUtilTest.java @@ -0,0 +1,49 @@ +package org.tron.common.utils; + +import java.io.File; +import java.io.IOException; +import org.junit.Assert; +import org.junit.Test; + +public class FileUtilTest { + + @Test + public void testCreateFileIfNotExists() { + String existFile = "existsfile.txt"; + File file1 = new File(existFile); + try { + file1.createNewFile(); + } catch (IOException e) { + + } + Assert.assertTrue(file1.exists()); + Assert.assertTrue(FileUtil.createDirIfNotExists(existFile)); + Assert.assertTrue(file1.exists()); + + String notExistFile = "notexistsfile.txt"; + File file2 = new File(notExistFile); + Assert.assertTrue(!file2.exists()); + Assert.assertTrue(FileUtil.createDirIfNotExists(notExistFile)); + Assert.assertTrue(file2.exists()); + file1.delete(); + file2.delete(); + } + + @Test + public void testCreateDirIfNotExists() { + String existDir = "existsdir"; + File fileDir1 = new File(existDir); + fileDir1.mkdir(); + Assert.assertTrue(fileDir1.exists()); + Assert.assertTrue(FileUtil.createDirIfNotExists(existDir)); + Assert.assertTrue(fileDir1.exists()); + + String notExistDir = "notexistsdir"; + File fileDir2 = new File(notExistDir); + Assert.assertTrue(!fileDir2.exists()); + Assert.assertTrue(FileUtil.createDirIfNotExists(notExistDir)); + Assert.assertTrue(fileDir2.exists()); + fileDir1.delete(); + fileDir2.delete(); + } +} \ No newline at end of file diff --git a/src/test/java/org/tron/common/utils/PropUtilTest.java b/src/test/java/org/tron/common/utils/PropUtilTest.java new file mode 100644 index 00000000000..2df5bd8effd --- /dev/null +++ b/src/test/java/org/tron/common/utils/PropUtilTest.java @@ -0,0 +1,40 @@ +package org.tron.common.utils; + +import java.io.File; +import java.io.IOException; +import org.junit.Assert; +import org.junit.Test; + +public class PropUtilTest { + + @Test + public void testWriteProperty() { + String filename = "test_prop.properties"; + File file = new File(filename); + try { + file.createNewFile(); + } catch (IOException e) { + e.printStackTrace(); + } + PropUtil.writeProperty(filename, "key", "value"); + Assert.assertTrue("value".equals(PropUtil.readProperty(filename, "key"))); + PropUtil.writeProperty(filename, "key", "value2"); + Assert.assertTrue("value2".equals(PropUtil.readProperty(filename, "key"))); + file.delete(); + } + + @Test + public void testReadProperty() { + String filename = "test_prop.properties"; + File file = new File(filename); + try { + file.createNewFile(); + } catch (IOException e) { + e.printStackTrace(); + } + PropUtil.writeProperty(filename, "key", "value"); + Assert.assertTrue("value".equals(PropUtil.readProperty(filename, "key"))); + file.delete(); + Assert.assertTrue("".equals(PropUtil.readProperty(filename, "key"))); + } +} \ No newline at end of file diff --git a/src/test/java/org/tron/core/db/backup/BackupDbUtilTest.java b/src/test/java/org/tron/core/db/backup/BackupDbUtilTest.java new file mode 100644 index 00000000000..97408bc9bb0 --- /dev/null +++ b/src/test/java/org/tron/core/db/backup/BackupDbUtilTest.java @@ -0,0 +1,115 @@ +package org.tron.core.db.backup; + + +import java.io.File; +import java.util.List; +import lombok.extern.slf4j.Slf4j; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.rocksdb.RocksDB; +import org.springframework.beans.factory.annotation.Autowired; +import org.tron.common.application.TronApplicationContext; +import org.tron.common.storage.leveldb.RocksDbDataSourceImpl; +import org.tron.common.utils.FileUtil; +import org.tron.common.utils.PropUtil; +import org.tron.core.config.DefaultConfig; +import org.tron.core.config.args.Args; +import org.tron.core.db.Manager; +import org.tron.core.db.ManagerForTest; +import org.tron.core.db.RevokingStoreRocks; + +@Slf4j +public class BackupDbUtilTest { + + static { + RocksDB.loadLibrary(); + } + + public TronApplicationContext context; + public BackupDbUtil dbBackupUtil; + public Manager dbManager; + public ManagerForTest mng_test; + public String dbPath = "output-BackupDbUtilTest"; + + String prop_path; + String bak1_path; + String bak2_path; + int frequency; + + @Before + public void before() { + Args.setParam( + new String[]{ + "--output-directory", dbPath, + "--storage-db-directory", "database", + "--storage-index-directory", "index" + }, + "config-test-dbbackup.conf" + ); + + context = new TronApplicationContext(DefaultConfig.class); + dbManager = context.getBean(Manager.class); + dbBackupUtil = context.getBean(BackupDbUtil.class); + mng_test = new ManagerForTest(dbManager); + + //prepare prop.properties + prop_path = dbPath + File.separator + "test_prop.properties"; + bak1_path = dbPath + File.separator + "bak1/database"; + bak2_path = dbPath + File.separator + "bak2/database"; + frequency = 50; + Args cfgArgs = Args.getInstance(); + cfgArgs.getDbBackupConfig() + .initArgs(true, prop_path, bak1_path, bak2_path, frequency); + FileUtil.createFileIfNotExists(prop_path); + } + + @After + public void after() { + FileUtil.deleteDir(new File(dbPath)); + RevokingStoreRocks.releaseInstance(); + } + + @Test + public void testDoBackup() { + PropUtil.writeProperty(prop_path, BackupDbUtil.getDB_BACKUP_STATE(), + String.valueOf("11")); + mng_test.pushNTestBlock(50); + List alist = ((RevokingStoreRocks)dbBackupUtil.getDb()).getDbs(); + logger.info("alist.size():" + alist.size()); + for(RocksDbDataSourceImpl rocks :alist) { + logger.info("sss:" + rocks.getDatabase().toString()); + logger.info("aaa:" + rocks.getDBName()); + logger.info("bbb:" + rocks.getDbPath()); + } + + Assert.assertTrue(dbManager.getDynamicPropertiesStore().getLatestBlockHeaderNumber() == 50); + Assert.assertTrue("22".equals( + PropUtil.readProperty(prop_path, BackupDbUtil.getDB_BACKUP_STATE()))); + + mng_test.pushNTestBlock(50); + Assert.assertTrue(dbManager.getDynamicPropertiesStore().getLatestBlockHeaderNumber() == 100); + Assert.assertTrue("11".equals( + PropUtil.readProperty(prop_path, BackupDbUtil.getDB_BACKUP_STATE()))); + + mng_test.pushNTestBlock(50); + Assert.assertTrue(dbManager.getDynamicPropertiesStore().getLatestBlockHeaderNumber() == 150); + Assert.assertTrue("22".equals( + PropUtil.readProperty(prop_path, BackupDbUtil.getDB_BACKUP_STATE()))); + + PropUtil.writeProperty(prop_path, BackupDbUtil.getDB_BACKUP_STATE(), + String.valueOf("1")); + mng_test.pushNTestBlock(50); + Assert.assertTrue(dbManager.getDynamicPropertiesStore().getLatestBlockHeaderNumber() == 200); + Assert.assertTrue("11".equals( + PropUtil.readProperty(prop_path, BackupDbUtil.getDB_BACKUP_STATE()))); + + PropUtil.writeProperty(prop_path, BackupDbUtil.getDB_BACKUP_STATE(), + String.valueOf("2")); + mng_test.pushNTestBlock(50); + Assert.assertTrue(dbManager.getDynamicPropertiesStore().getLatestBlockHeaderNumber() == 250); + Assert.assertTrue("22".equals( + PropUtil.readProperty(prop_path, BackupDbUtil.getDB_BACKUP_STATE()))); + } +} \ No newline at end of file diff --git a/src/test/java/org/tron/core/net/node/BaseNetTest.java b/src/test/java/org/tron/core/net/node/BaseNetTest.java index 4bf680d22ff..00ea0041618 100644 --- a/src/test/java/org/tron/core/net/node/BaseNetTest.java +++ b/src/test/java/org/tron/core/net/node/BaseNetTest.java @@ -32,6 +32,7 @@ import org.tron.core.config.DefaultConfig; import org.tron.core.config.args.Args; import org.tron.core.db.Manager; +import org.tron.core.db.RevokingStoreRocks; import org.tron.core.net.peer.PeerConnection; import org.tron.core.services.RpcApiService; import org.tron.core.services.WitnessService; @@ -162,6 +163,7 @@ public void destroy() { for (PeerConnection peer : peerConnections) { peer.close(); } + RevokingStoreRocks.releaseInstance(); context.destroy(); node.shutDown(); appT.shutdownServices(); diff --git a/src/test/java/org/tron/core/net/node/BroadTest.java b/src/test/java/org/tron/core/net/node/BroadTest.java index fb7910c9a92..12c53de1240 100644 --- a/src/test/java/org/tron/core/net/node/BroadTest.java +++ b/src/test/java/org/tron/core/net/node/BroadTest.java @@ -12,6 +12,7 @@ import org.junit.After; import org.junit.Assert; import org.junit.Before; +import org.junit.Ignore; import org.junit.Test; import org.tron.common.application.Application; import org.tron.common.application.ApplicationFactory; @@ -30,6 +31,7 @@ import org.tron.core.config.args.Args; import org.tron.core.db.ByteArrayWrapper; import org.tron.core.db.Manager; +import org.tron.core.db.RevokingStoreRocks; import org.tron.core.net.message.BlockMessage; import org.tron.core.net.message.MessageTypes; import org.tron.core.net.message.TransactionMessage; @@ -294,6 +296,7 @@ public void destroy() { for (PeerConnection peer : peerConnections) { peer.close(); } + RevokingStoreRocks.releaseInstance(); context.destroy(); handshakeHandlerTest.close(); appT.shutdownServices(); diff --git a/src/test/java/org/tron/core/net/node/GetBlockChainSummaryTest.java b/src/test/java/org/tron/core/net/node/GetBlockChainSummaryTest.java index 74e53bed873..54f32288ce3 100644 --- a/src/test/java/org/tron/core/net/node/GetBlockChainSummaryTest.java +++ b/src/test/java/org/tron/core/net/node/GetBlockChainSummaryTest.java @@ -16,6 +16,7 @@ import org.junit.AfterClass; import org.junit.Assert; import org.junit.Before; +import org.junit.Ignore; import org.junit.Test; import org.tron.common.application.Application; import org.tron.common.application.ApplicationFactory; @@ -38,6 +39,7 @@ import org.tron.core.db.BlockStore; import org.tron.core.db.ByteArrayWrapper; import org.tron.core.db.Manager; +import org.tron.core.db.RevokingStoreRocks; import org.tron.core.net.node.override.HandshakeHandlerTest; import org.tron.core.net.node.override.PeerClientTest; import org.tron.core.net.node.override.TronChannelInitializerTest; @@ -344,6 +346,7 @@ public static void destroy() { handshakeHandlerTest.close(); appT.shutdownServices(); appT.shutdown(); + RevokingStoreRocks.releaseInstance(); context.destroy(); FileUtil.deleteDir(new File(dbPath)); } diff --git a/src/test/java/org/tron/core/net/node/GetLostBlockIdsTest.java b/src/test/java/org/tron/core/net/node/GetLostBlockIdsTest.java index 94dd6bdb186..c8eb1a0aaef 100644 --- a/src/test/java/org/tron/core/net/node/GetLostBlockIdsTest.java +++ b/src/test/java/org/tron/core/net/node/GetLostBlockIdsTest.java @@ -18,6 +18,7 @@ import org.junit.AfterClass; import org.junit.Assert; import org.junit.Before; +import org.junit.Ignore; import org.junit.Test; import org.tron.common.application.Application; import org.tron.common.application.ApplicationFactory; @@ -39,6 +40,7 @@ import org.tron.core.config.args.Args; import org.tron.core.db.ByteArrayWrapper; import org.tron.core.db.Manager; +import org.tron.core.db.RevokingStoreRocks; import org.tron.core.exception.StoreException; import org.tron.core.net.node.override.HandshakeHandlerTest; import org.tron.core.net.node.override.PeerClientTest; @@ -324,6 +326,7 @@ public static void destroy() { peer.close(); } handshakeHandlerTest.close(); + RevokingStoreRocks.releaseInstance(); appT.shutdownServices(); appT.shutdown(); context.destroy(); diff --git a/src/test/java/org/tron/core/net/node/NodeImplTest.java b/src/test/java/org/tron/core/net/node/NodeImplTest.java index 738a0f06154..037ce1e2ebf 100644 --- a/src/test/java/org/tron/core/net/node/NodeImplTest.java +++ b/src/test/java/org/tron/core/net/node/NodeImplTest.java @@ -1,6 +1,11 @@ package org.tron.core.net.node; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + import com.google.protobuf.ByteString; +import java.io.File; +import java.util.concurrent.ConcurrentHashMap; import lombok.extern.slf4j.Slf4j; import org.junit.AfterClass; import org.junit.Assert; @@ -22,18 +27,13 @@ import org.tron.core.config.Parameter.NetConstants; import org.tron.core.config.args.Args; import org.tron.core.db.Manager; +import org.tron.core.db.RevokingStoreRocks; import org.tron.core.net.message.BlockMessage; import org.tron.core.net.peer.PeerConnection; import org.tron.protos.Protocol.Block; import org.tron.protos.Protocol.BlockHeader; import org.tron.protos.Protocol.Inventory.InventoryType; -import java.io.File; -import java.util.concurrent.ConcurrentHashMap; - -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; - @Slf4j public class NodeImplTest { @@ -86,7 +86,7 @@ public void testSyncBlockMessage() throws Exception { } @Test - public void testAdvBlockMessage() throws Exception{ + public void testAdvBlockMessage() throws Exception { PeerConnection peer = new PeerConnection(); BlockCapsule genesisBlockCapsule = BlockUtil.newGenesisBlockCapsule(); @@ -112,7 +112,8 @@ public void testAdvBlockMessage() throws Exception{ blockCapsule.sign( ByteArray.fromHexString(Args.getInstance().getLocalWitnesses().getPrivateKey())); BlockMessage blockMessage = new BlockMessage(blockCapsule); - peer.getAdvObjWeRequested().put(new Item(blockMessage.getBlockId(), InventoryType.BLOCK), System.currentTimeMillis()); + peer.getAdvObjWeRequested() + .put(new Item(blockMessage.getBlockId(), InventoryType.BLOCK), System.currentTimeMillis()); nodeImpl.onMessage(peer, blockMessage); Assert.assertEquals(peer.getAdvObjWeRequested().size(), 0); } @@ -194,8 +195,9 @@ public static void init() { } @AfterClass - public static void destroy(){ + public static void destroy() { Args.clearParam(); + RevokingStoreRocks.releaseInstance(); context.destroy(); appT.shutdownServices(); appT.shutdown(); diff --git a/src/test/java/org/tron/core/net/node/StartFetchSyncBlockTest.java b/src/test/java/org/tron/core/net/node/StartFetchSyncBlockTest.java index 672b0fde837..4ebaa9a9d79 100644 --- a/src/test/java/org/tron/core/net/node/StartFetchSyncBlockTest.java +++ b/src/test/java/org/tron/core/net/node/StartFetchSyncBlockTest.java @@ -27,6 +27,7 @@ import org.tron.core.config.args.Args; import org.tron.core.db.ByteArrayWrapper; import org.tron.core.db.Manager; +import org.tron.core.db.RevokingStoreRocks; import org.tron.core.net.message.BlockMessage; import org.tron.core.net.node.override.HandshakeHandlerTest; import org.tron.core.net.node.override.PeerClientTest; @@ -249,6 +250,7 @@ public void run() { public static void destroy() { Args.clearParam(); handshakeHandlerTest.close(); + RevokingStoreRocks.releaseInstance(); context.destroy(); appT.shutdownServices(); appT.shutdown(); diff --git a/src/test/java/org/tron/core/net/node/TcpNetTest.java b/src/test/java/org/tron/core/net/node/TcpNetTest.java index b4dcbf7d9fb..75344c44bd2 100644 --- a/src/test/java/org/tron/core/net/node/TcpNetTest.java +++ b/src/test/java/org/tron/core/net/node/TcpNetTest.java @@ -26,6 +26,7 @@ import org.junit.After; import org.junit.AfterClass; import org.junit.Assert; +import org.junit.Ignore; import org.junit.Test; import org.tron.common.overlay.discover.node.Node; import org.tron.common.overlay.message.DisconnectMessage; diff --git a/src/test/resources/config-test-dbbackup.conf b/src/test/resources/config-test-dbbackup.conf new file mode 100644 index 00000000000..d7cf2e1faaf --- /dev/null +++ b/src/test/resources/config-test-dbbackup.conf @@ -0,0 +1,411 @@ +net { + type = mainnet + # type = testnet +} + +storage { + # Directory for storing persistent data + db.version = 3, + db.directory = "database", + index.directory = "index", + + # You can custom these 14 databases' configs: + + # account, account-index, asset-issue, block, block-index, + # block_KDB, peers, properties, recent-block, trans, + # utxo, votes, witness, witness_schedule. + + # Otherwise, db configs will remain defualt and data will be stored in + # the path of "output-directory" or which is set by "-d" ("--output-directory"). + + # Attention: name is a required field that must be set !!! + properties = [ +// { +// name = "account", +// path = "storage_directory_test", +// createIfMissing = true, +// paranoidChecks = true, +// verifyChecksums = true, + // compressionType = 1, // compressed with snappy + // blockSize = 4096, // 4 KB = 4 * 1024 B + // writeBufferSize = 10485760, // 10 MB = 10 * 1024 * 1024 B + // cacheSize = 10485760, // 10 MB = 10 * 1024 * 1024 B + // maxOpenFiles = 100 + // }, +// { +// name = "account-index", +// path = "storage_directory_test", +// createIfMissing = true, +// paranoidChecks = true, +// verifyChecksums = true, + // compressionType = 1, // compressed with snappy + // blockSize = 4096, // 4 KB = 4 * 1024 B + // writeBufferSize = 10485760, // 10 MB = 10 * 1024 * 1024 B + // cacheSize = 10485760, // 10 MB = 10 * 1024 * 1024 B + // maxOpenFiles = 100 + // }, + ] + + needToUpdateAsset = true + + backup = { + enable = true + properties = "" + bak1path = "" + bak2path = "" + frequency = 5000 // backup db every ? blocks processed. + } +} + +node.discovery = { + enable = true + persist = true + bind.ip = "" + external.ip = null +} + +node.backup { + port = 10001 + + # my priority, each member should use different priority + priority = 8 + + # peer's ip list, can't contain mine + members = [ + # "ip", + # "ip" + ] +} + +node { + # trust node for solidity node + # trustNode = "ip:port" + trustNode = "127.0.0.1:50051" + + # expose extension api to public or not + walletExtensionApi = true + + listen.port = 18888 + + connection.timeout = 2 + + tcpNettyWorkThreadNum = 0 + + udpNettyWorkThreadNum = 1 + + # Number of validate sign thread, default availableProcessors / 2 + # validateSignThreadNum = 16 + + connectFactor = 0.3 + activeConnectFactor = 0.1 + + maxActiveNodes = 30 + + maxActiveNodesWithSameIp = 2 + + minParticipationRate = 15 + + # check the peer data transfer ,disconnect factor + disconnectNumberFactor = 0.4 + maxConnectNumberFactor = 0.8 + receiveTcpMinDataLength = 2048 + isOpenFullTcpDisconnect = true + + p2p { + version = 11111 # 11111: mainnet; 20180622: testnet + } + + active = [ + # Active establish connection in any case + # Sample entries: + # "ip:port", + # "ip:port" + ] + + passive = [ + # Passive accept connection in any case + # Sample entries: + # "ip:port", + # "ip:port" + ] + + http { + fullNodePort = 8090 + solidityPort = 8091 + } + + rpc { + port = 50051 + #solidityPort = 50061 + # Number of gRPC thread, default availableProcessors / 2 + # thread = 16 + + # The maximum number of concurrent calls permitted for each incoming connection + # maxConcurrentCallsPerConnection = + + # The HTTP/2 flow control window, default 1MB + # flowControlWindow = + + # Connection being idle for longer than which will be gracefully terminated + maxConnectionIdleInMillis = 60000 + + # Connection lasting longer than which will be gracefully terminated + # maxConnectionAgeInMillis = + + # The maximum message size allowed to be received on the server, default 4MB + # maxMessageSize = + + # The maximum size of header list allowed to be received, default 8192 + # maxHeaderListSize = + } + + # Limits the maximum percentage (default 75%) of producing block interval + # to provide sufficient time to perform other operations e.g. broadcast block + # blockProducedTimeOut = 75 + + # Limits the maximum number (default 700) of transaction from network layer + # netMaxTrxPerSecond = 700 +} + + + +seed.node = { + # List of the seed nodes + # Seed nodes are stable full nodes + # example: + # ip.list = [ + # "ip:port", + # "ip:port" + # ] + ip.list = [ + "54.236.37.243:18888", + "52.53.189.99:18888", + "18.196.99.16:18888", + "34.253.187.192:18888", + "52.56.56.149:18888", + "35.180.51.163:18888", + "54.252.224.209:18888", + "18.228.15.36:18888", + "52.15.93.92:18888", + "34.220.77.106:18888", + "13.127.47.162:18888", + "13.124.62.58:18888", + "13.229.128.108:18888", + "35.182.37.246:18888", + "34.200.228.125:18888", + "18.220.232.201:18888", + "13.57.30.186:18888", + "35.165.103.105:18888", + "18.184.238.21:18888", + "34.250.140.143:18888", + "35.176.192.130:18888", + "52.47.197.188:18888", + "52.62.210.100:18888", + "13.231.4.243:18888", + "18.231.76.29:18888", + "35.154.90.144:18888", + "13.125.210.234:18888", + "13.250.40.82:18888", + "35.183.101.48:18888" + ] +} + +genesis.block = { + # Reserve balance + assets = [ + { + accountName = "Zion" + accountType = "AssetIssue" + address = "TLLM21wteSPs4hKjbxgmH1L6poyMjeTbHm" + balance = "99000000000000000" + }, + { + accountName = "Sun" + accountType = "AssetIssue" + address = "TXmVpin5vq5gdZsciyyjdZgKRUju4st1wM" + balance = "0" + }, + { + accountName = "Blackhole" + accountType = "AssetIssue" + address = "TLsV52sRDL79HXGGm9yzwKibb6BeruhUzy" + balance = "-9223372036854775808" + } + ] + + witnesses = [ + { + address: THKJYuUmMKKARNf7s2VT51g5uPY6KEqnat, + url = "http://GR1.com", + voteCount = 100000026 + }, + { + address: TVDmPWGYxgi5DNeW8hXrzrhY8Y6zgxPNg4, + url = "http://GR2.com", + voteCount = 100000025 + }, + { + address: TWKZN1JJPFydd5rMgMCV5aZTSiwmoksSZv, + url = "http://GR3.com", + voteCount = 100000024 + }, + { + address: TDarXEG2rAD57oa7JTK785Yb2Et32UzY32, + url = "http://GR4.com", + voteCount = 100000023 + }, + { + address: TAmFfS4Tmm8yKeoqZN8x51ASwdQBdnVizt, + url = "http://GR5.com", + voteCount = 100000022 + }, + { + address: TK6V5Pw2UWQWpySnZyCDZaAvu1y48oRgXN, + url = "http://GR6.com", + voteCount = 100000021 + }, + { + address: TGqFJPFiEqdZx52ZR4QcKHz4Zr3QXA24VL, + url = "http://GR7.com", + voteCount = 100000020 + }, + { + address: TC1ZCj9Ne3j5v3TLx5ZCDLD55MU9g3XqQW, + url = "http://GR8.com", + voteCount = 100000019 + }, + { + address: TWm3id3mrQ42guf7c4oVpYExyTYnEGy3JL, + url = "http://GR9.com", + voteCount = 100000018 + }, + { + address: TCvwc3FV3ssq2rD82rMmjhT4PVXYTsFcKV, + url = "http://GR10.com", + voteCount = 100000017 + }, + { + address: TFuC2Qge4GxA2U9abKxk1pw3YZvGM5XRir, + url = "http://GR11.com", + voteCount = 100000016 + }, + { + address: TNGoca1VHC6Y5Jd2B1VFpFEhizVk92Rz85, + url = "http://GR12.com", + voteCount = 100000015 + }, + { + address: TLCjmH6SqGK8twZ9XrBDWpBbfyvEXihhNS, + url = "http://GR13.com", + voteCount = 100000014 + }, + { + address: TEEzguTtCihbRPfjf1CvW8Euxz1kKuvtR9, + url = "http://GR14.com", + voteCount = 100000013 + }, + { + address: TZHvwiw9cehbMxrtTbmAexm9oPo4eFFvLS, + url = "http://GR15.com", + voteCount = 100000012 + }, + { + address: TGK6iAKgBmHeQyp5hn3imB71EDnFPkXiPR, + url = "http://GR16.com", + voteCount = 100000011 + }, + { + address: TLaqfGrxZ3dykAFps7M2B4gETTX1yixPgN, + url = "http://GR17.com", + voteCount = 100000010 + }, + { + address: TX3ZceVew6yLC5hWTXnjrUFtiFfUDGKGty, + url = "http://GR18.com", + voteCount = 100000009 + }, + { + address: TYednHaV9zXpnPchSywVpnseQxY9Pxw4do, + url = "http://GR19.com", + voteCount = 100000008 + }, + { + address: TCf5cqLffPccEY7hcsabiFnMfdipfyryvr, + url = "http://GR20.com", + voteCount = 100000007 + }, + { + address: TAa14iLEKPAetX49mzaxZmH6saRxcX7dT5, + url = "http://GR21.com", + voteCount = 100000006 + }, + { + address: TBYsHxDmFaRmfCF3jZNmgeJE8sDnTNKHbz, + url = "http://GR22.com", + voteCount = 100000005 + }, + { + address: TEVAq8dmSQyTYK7uP1ZnZpa6MBVR83GsV6, + url = "http://GR23.com", + voteCount = 100000004 + }, + { + address: TRKJzrZxN34YyB8aBqqPDt7g4fv6sieemz, + url = "http://GR24.com", + voteCount = 100000003 + }, + { + address: TRMP6SKeFUt5NtMLzJv8kdpYuHRnEGjGfe, + url = "http://GR25.com", + voteCount = 100000002 + }, + { + address: TDbNE1VajxjpgM5p7FyGNDASt3UVoFbiD3, + url = "http://GR26.com", + voteCount = 100000001 + }, + { + address: TLTDZBcPoJ8tZ6TTEeEqEvwYFk2wgotSfD, + url = "http://GR27.com", + voteCount = 100000000 + } + ] + + timestamp = "0" #2017-8-26 12:00:00 + + parentHash = "0xe58f33f9baf9305dc6f82b9f1934ea8f0ade2defb951258d50167028c780351f" +} + +localwitness = [ +] + +#localwitnesskeystore = [ +# "localwitnesskeystore.json" +#] + +block = { + needSyncCheck = true + maintenanceTimeInterval = 21600000 + proposalExpireTime = 259200000 // 3 day: 259200000(ms) +} + +# Transaction reference block, default is "head", configure to "solid" can avoid TaPos error +# trx.reference.block = "head" // head;solid; + +vm = { + supportConstant = false + minTimeRatio = 0.0 + maxTimeRatio = 5.0 + + # In rare cases, transactions that will be within the specified maximum execution time (default 10(ms)) are re-executed and packaged + # longRunningTime = 10 +} + +committee = { + allowCreationOfContracts = 0 //mainnet:0 (reset by committee),test:1 + allowAdaptiveEnergy = 0 //mainnet:0 (reset by committee),test:1 +} + +log.level = { + root = "INFO" // TRACE;DEBUG;INFO;WARN;ERROR +} From 41ac169c823a2426eef90773fb2f6265657587a5 Mon Sep 17 00:00:00 2001 From: sun haoyu Date: Sat, 22 Dec 2018 22:17:27 +0800 Subject: [PATCH 02/13] fix some conflict --- .../java/org/tron/core/config/args/Args.java | 1 + .../org/tron/core/db/RevokingStoreRocks.java | 5 +++++ .../RevokingRocksDBWithCachingOldValue.java | 20 +++++-------------- 3 files changed, 11 insertions(+), 15 deletions(-) diff --git a/src/main/java/org/tron/core/config/args/Args.java b/src/main/java/org/tron/core/config/args/Args.java index 5e91b1d1808..2dac7b22d13 100644 --- a/src/main/java/org/tron/core/config/args/Args.java +++ b/src/main/java/org/tron/core/config/args/Args.java @@ -397,6 +397,7 @@ public class Args { @Setter private String trxReferenceBlock; + @Getter @Setter private int minEffectiveConnection; diff --git a/src/main/java/org/tron/core/db/RevokingStoreRocks.java b/src/main/java/org/tron/core/db/RevokingStoreRocks.java index 65b6d53cc4b..21688993992 100644 --- a/src/main/java/org/tron/core/db/RevokingStoreRocks.java +++ b/src/main/java/org/tron/core/db/RevokingStoreRocks.java @@ -94,6 +94,11 @@ public ISession buildSession(boolean forceEnable) { return new Dialog(this, disableOnExit); } + @Override + public void setMode(boolean mode) { + + } + @Override public synchronized void check() { diff --git a/src/main/java/org/tron/core/db2/core/RevokingRocksDBWithCachingOldValue.java b/src/main/java/org/tron/core/db2/core/RevokingRocksDBWithCachingOldValue.java index 2d791e56f92..25c8684b32f 100644 --- a/src/main/java/org/tron/core/db2/core/RevokingRocksDBWithCachingOldValue.java +++ b/src/main/java/org/tron/core/db2/core/RevokingRocksDBWithCachingOldValue.java @@ -64,11 +64,6 @@ public void delete(byte[] key) { dbSource.deleteData(key); } - @Override - public boolean hasOnSolidity(byte[] key) { - return false; - } - @Override public boolean has(byte[] key) { return dbSource.getData(key) != null; @@ -92,16 +87,6 @@ public byte[] getUnchecked(byte[] key) { } } - @Override - public byte[] getOnSolidity(byte[] key) throws ItemNotFoundException { - return new byte[0]; - } - - @Override - public byte[] getUncheckedOnSolidity(byte[] key) { - return new byte[0]; - } - @Override public void close() { dbSource.closeDB(); @@ -112,6 +97,11 @@ public void reset() { dbSource.resetDb(); } + @Override + public void setMode(boolean mode) { + + } + /** * This should be called just after an object is created */ From e098d27e068163898560a5314abcccd62026a42d Mon Sep 17 00:00:00 2001 From: sun haoyu Date: Fri, 4 Jan 2019 15:17:18 +0800 Subject: [PATCH 03/13] tune rocksdb --- src/main/java/org/tron/common/storage/DBSettings.java | 2 +- .../tron/common/storage/leveldb/RocksDbDataSourceImpl.java | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/tron/common/storage/DBSettings.java b/src/main/java/org/tron/common/storage/DBSettings.java index 2c52c870b35..0874717e2d9 100644 --- a/src/main/java/org/tron/common/storage/DBSettings.java +++ b/src/main/java/org/tron/common/storage/DBSettings.java @@ -5,7 +5,7 @@ public class DBSettings { public static final DBSettings DEFAULT = new DBSettings() .withMaxThreads(1) - .withMaxOpenFiles(32); + .withMaxOpenFiles(-1); int maxOpenFiles; int maxThreads; diff --git a/src/main/java/org/tron/common/storage/leveldb/RocksDbDataSourceImpl.java b/src/main/java/org/tron/common/storage/leveldb/RocksDbDataSourceImpl.java index 7fdc6c0a04e..ce24a38e0b0 100644 --- a/src/main/java/org/tron/common/storage/leveldb/RocksDbDataSourceImpl.java +++ b/src/main/java/org/tron/common/storage/leveldb/RocksDbDataSourceImpl.java @@ -24,6 +24,7 @@ import org.rocksdb.RocksDB; import org.rocksdb.RocksDBException; import org.rocksdb.RocksIterator; +import org.rocksdb.Statistics; import org.rocksdb.WriteBatch; import org.rocksdb.WriteOptions; import org.tron.common.storage.DBSettings; @@ -152,13 +153,15 @@ public void initDB(DBSettings settings) { // most of these options are suggested by https://github.com/facebook/rocksdb/wiki/Set-Up-Options // general options + options.setStatistics(new Statistics()); + options.setStatsDumpPeriodSec(60); options.setCreateIfMissing(true); options.setCompressionType(CompressionType.LZ4_COMPRESSION); options.setBottommostCompressionType(CompressionType.ZSTD_COMPRESSION); options.setLevelCompactionDynamicLevelBytes(true); options.setMaxOpenFiles(settings.getMaxOpenFiles()); options.setIncreaseParallelism(settings.getMaxThreads()); - + options.setMaxBackgroundCompactions(8); // table options final BlockBasedTableConfig tableCfg; options.setTableFormatConfig(tableCfg = new BlockBasedTableConfig()); From 49dd625ce67137778d6636133fe5cfe461ca7596 Mon Sep 17 00:00:00 2001 From: sun haoyu Date: Mon, 21 Jan 2019 12:18:44 +0800 Subject: [PATCH 04/13] rebase develop --- .../org/tron/common/storage/DBSettings.java | 134 ++++++++++++++++-- .../leveldb/RocksDbDataSourceImpl.java | 30 ++-- .../java/org/tron/core/config/args/Args.java | 62 ++++++-- .../core/db/backup/BackupRocksDBAspect.java | 2 +- src/main/resources/config.conf | 16 ++- 5 files changed, 204 insertions(+), 40 deletions(-) diff --git a/src/main/java/org/tron/common/storage/DBSettings.java b/src/main/java/org/tron/common/storage/DBSettings.java index 0874717e2d9..11fe1a4283a 100644 --- a/src/main/java/org/tron/common/storage/DBSettings.java +++ b/src/main/java/org/tron/common/storage/DBSettings.java @@ -1,40 +1,144 @@ package org.tron.common.storage; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import lombok.Getter; +import lombok.extern.slf4j.Slf4j; +import org.rocksdb.CompressionType; + +@Slf4j public class DBSettings { - public static final DBSettings DEFAULT = new DBSettings() - .withMaxThreads(1) - .withMaxOpenFiles(-1); + private static DBSettings settings; - int maxOpenFiles; - int maxThreads; + @Getter + private int levelNumber; + @Getter + private int maxOpenFiles; + @Getter + private int compactThreads; + @Getter + private long blockSize; + @Getter + private long maxBytesForLevelBase; + @Getter + private double maxBytesForLevelMultiplier; + @Getter + private int level0FileNumCompactionTrigger; + @Getter + private List compressionTypeList; + @Getter + private long targetFileSizeBase; + @Getter + private int targetFileSizeMultiplier; + @Getter + private boolean enableStatistics; private DBSettings() { + } - public static DBSettings newInstance() { - DBSettings settings = new DBSettings(); - settings.maxOpenFiles = DEFAULT.maxOpenFiles; - settings.maxThreads = DEFAULT.maxThreads; + public static DBSettings getSettings() { return settings; } - public int getMaxOpenFiles() { - return maxOpenFiles; + public static DBSettings initCustomSettings(int levelNumber, int compactThreads, int blocksize, + long maxBytesForLevelBase, + double maxBytesForLevelMultiplier, String compressionStr, + int level0FileNumCompactionTrigger, long targetFileSizeBase, int targetFileSizeMultiplier) { + settings = new DBSettings() + .withMaxOpenFiles(-1) + .withEnableStatistics(false) + .withLevelNumber(levelNumber) + .withCompactThreads(compactThreads) + .withBlockSize(blocksize) + .withMaxBytesForLevelBase(maxBytesForLevelBase) + .withMaxBytesForLevelMultiplier(maxBytesForLevelMultiplier) + .withCompressionTypeList(compressionStr) + .withLevel0FileNumCompactionTrigger(level0FileNumCompactionTrigger) + .withTargetFileSizeBase(targetFileSizeBase) + .withTargetFileSizeMultiplier(targetFileSizeMultiplier); + return settings; } + public DBSettings withMaxOpenFiles(int maxOpenFiles) { this.maxOpenFiles = maxOpenFiles; return this; } - public int getMaxThreads() { - return maxThreads; + public DBSettings withCompactThreads(int compactThreads) { + this.compactThreads = compactThreads; + return this; + } + + public DBSettings withBlockSize(long blockSize) { + this.blockSize = blockSize * 1024; + return this; + } + + public DBSettings withMaxBytesForLevelBase(long maxBytesForLevelBase) { + this.maxBytesForLevelBase = maxBytesForLevelBase * 1024 * 1024; + return this; + } + + public DBSettings withMaxBytesForLevelMultiplier(double maxBytesForLevelMultiplier) { + this.maxBytesForLevelMultiplier = maxBytesForLevelMultiplier; + return this; + } + + public DBSettings withLevel0FileNumCompactionTrigger(int level0FileNumCompactionTrigger) { + this.level0FileNumCompactionTrigger = level0FileNumCompactionTrigger; + return this; + } + + public DBSettings withCompressionTypeList(String compressionTypeListString) { + List compressionTypeStringList = Arrays.asList(compressionTypeListString.split(":")); + List compressionTypeList = new ArrayList<>(); + for (String str : compressionTypeStringList) { + if (str.equals("no")) { + compressionTypeList.add(CompressionType.NO_COMPRESSION); + } else if (str.equals("lz4")) { + compressionTypeList.add(CompressionType.LZ4_COMPRESSION); + } else if (str.equals("zstd")) { + compressionTypeList.add(CompressionType.ZSTD_COMPRESSION); + } else if (str.equals("zlib")) { + compressionTypeList.add(CompressionType.ZLIB_COMPRESSION); + } + } + this.compressionTypeList = compressionTypeList; + return this; + } + + public DBSettings withEnableStatistics(boolean enable) { + this.enableStatistics = enable; + return this; + } + + public DBSettings withLevelNumber(int levelNumber) { + this.levelNumber = levelNumber; + return this; + } + + + public DBSettings withTargetFileSizeBase(long targetFileSizeBase) { + this.targetFileSizeBase = targetFileSizeBase * 1024 * 1024; + return this; } - public DBSettings withMaxThreads(int maxThreads) { - this.maxThreads = maxThreads; + public DBSettings withTargetFileSizeMultiplier(int targetFileSizeMultiplier) { + this.targetFileSizeMultiplier = targetFileSizeMultiplier; return this; } + + public static void loggingSettings() { + logger.info(String.format( + "level number: %d, CompactThreads: %d, Blocksize: %d, maxBytesForLevelBase: %d, withMaxBytesForLevelMultiplier: %f, level0FileNumCompactionTrigger: %d, withTargetFileSizeBase: %d, withTargetFileSizeMultiplier: %d", + settings.getLevelNumber(), + settings.getCompactThreads(), settings.getBlockSize(), settings.getMaxBytesForLevelBase(), + settings.getMaxBytesForLevelMultiplier(), settings.getLevel0FileNumCompactionTrigger(), + settings.getTargetFileSizeBase(), settings.getTargetFileSizeMultiplier())); + } } diff --git a/src/main/java/org/tron/common/storage/leveldb/RocksDbDataSourceImpl.java b/src/main/java/org/tron/common/storage/leveldb/RocksDbDataSourceImpl.java index ce24a38e0b0..43fee75dc55 100644 --- a/src/main/java/org/tron/common/storage/leveldb/RocksDbDataSourceImpl.java +++ b/src/main/java/org/tron/common/storage/leveldb/RocksDbDataSourceImpl.java @@ -18,7 +18,6 @@ import org.rocksdb.BlockBasedTableConfig; import org.rocksdb.BloomFilter; import org.rocksdb.Checkpoint; -import org.rocksdb.CompressionType; import org.rocksdb.Options; import org.rocksdb.ReadOptions; import org.rocksdb.RocksDB; @@ -44,8 +43,6 @@ public class RocksDbDataSourceImpl implements DbSourceInterRocks, private String parentName; ReadOptions readOpts; - DBSettings settings = DBSettings.DEFAULT; - private ReadWriteLock resetDbLock = new ReentrantReadWriteLock(); public RocksDbDataSourceImpl(String parentName, String name) { @@ -134,11 +131,10 @@ public void setDBName(String name) { } public void initDB() { - initDB(DBSettings.DEFAULT); + initDB(DBSettings.getSettings()); } public void initDB(DBSettings settings) { - this.settings = settings; resetDbLock.writeLock().lock(); try { if (isAlive()) { @@ -153,19 +149,29 @@ public void initDB(DBSettings settings) { // most of these options are suggested by https://github.com/facebook/rocksdb/wiki/Set-Up-Options // general options - options.setStatistics(new Statistics()); - options.setStatsDumpPeriodSec(60); + if (settings.isEnableStatistics()) { + options.setStatistics(new Statistics()); + options.setStatsDumpPeriodSec(60); + } options.setCreateIfMissing(true); - options.setCompressionType(CompressionType.LZ4_COMPRESSION); - options.setBottommostCompressionType(CompressionType.ZSTD_COMPRESSION); + options.setIncreaseParallelism(1); options.setLevelCompactionDynamicLevelBytes(true); options.setMaxOpenFiles(settings.getMaxOpenFiles()); - options.setIncreaseParallelism(settings.getMaxThreads()); - options.setMaxBackgroundCompactions(8); + + // general options supported user config + options.setNumLevels(settings.getLevelNumber()); + options.setMaxBytesForLevelMultiplier(settings.getMaxBytesForLevelMultiplier()); + options.setMaxBytesForLevelBase(settings.getMaxBytesForLevelBase()); + options.setMaxBackgroundCompactions(settings.getCompactThreads()); + options.setCompressionPerLevel(settings.getCompressionTypeList()); + options.setLevel0FileNumCompactionTrigger(settings.getLevel0FileNumCompactionTrigger()); + options.setTargetFileSizeMultiplier(settings.getTargetFileSizeMultiplier()); + options.setTargetFileSizeBase(settings.getTargetFileSizeBase()); + // table options final BlockBasedTableConfig tableCfg; options.setTableFormatConfig(tableCfg = new BlockBasedTableConfig()); - tableCfg.setBlockSize(16 * 1024); + tableCfg.setBlockSize(settings.getBlockSize()); tableCfg.setBlockCacheSize(32 * 1024 * 1024); tableCfg.setCacheIndexAndFilterBlocks(true); tableCfg.setPinL0FilterAndIndexBlocksInCache(true); diff --git a/src/main/java/org/tron/core/config/args/Args.java b/src/main/java/org/tron/core/config/args/Args.java index 2dac7b22d13..50115436b33 100644 --- a/src/main/java/org/tron/core/config/args/Args.java +++ b/src/main/java/org/tron/core/config/args/Args.java @@ -37,6 +37,7 @@ import org.tron.common.logsfilter.FilterQuery; import org.tron.common.logsfilter.TriggerConfig; import org.tron.common.overlay.discover.node.Node; +import org.tron.common.storage.DBSettings; import org.tron.common.utils.ByteArray; import org.tron.core.Constant; import org.tron.core.Wallet; @@ -120,7 +121,8 @@ public class Args { @Parameter(names = {"--storage-db-version"}, description = "Storage db version.(1 or 2)") private String storageDbVersion = ""; - @Parameter(names = {"--storage-db-synchronous"}, description = "Storage db is synchronous or not.(true or flase)") + @Parameter(names = { + "--storage-db-synchronous"}, description = "Storage db is synchronous or not.(true or flase)") private String storageDbSynchronous = ""; @Parameter(names = {"--storage-index-directory"}, description = "Storage index directory") @@ -129,7 +131,8 @@ public class Args { @Parameter(names = {"--storage-index-switch"}, description = "Storage index switch.(on or off)") private String storageIndexSwitch = ""; - @Parameter(names = {"--storage-transactionHistory-switch"}, description = "Storage transaction history switch.(on or off)") + @Parameter(names = { + "--storage-transactionHistory-switch"}, description = "Storage transaction history switch.(on or off)") private String storageTransactionHistoreSwitch = ""; @Getter @@ -212,7 +215,7 @@ public class Args { @Setter private long nodeP2pPingInterval; -// @Getter + // @Getter // @Setter // private long syncNodeCount; @Getter @@ -419,6 +422,9 @@ public class Args { @Getter private DbBackupConfig dbBackupConfig; + @Getter + private DBSettings rocksDBCustomSettings; + public static void clearParam() { INSTANCE.outputDirectory = "output-directory"; INSTANCE.help = false; @@ -605,9 +611,9 @@ public static void setParam(final String[] args, final String confFileName) { .orElse(Storage.getDbVersionFromConfig(config))); INSTANCE.storage.setDbSync(Optional.ofNullable(INSTANCE.storageDbSynchronous) - .filter(StringUtils::isNotEmpty) - .map(Boolean::valueOf) - .orElse(Storage.getDbVersionSyncFromConfig(config))); + .filter(StringUtils::isNotEmpty) + .map(Boolean::valueOf) + .orElse(Storage.getDbVersionSyncFromConfig(config))); INSTANCE.storage.setDbDirectory(Optional.ofNullable(INSTANCE.storageDbDirectory) .filter(StringUtils::isNotEmpty) @@ -621,10 +627,10 @@ public static void setParam(final String[] args, final String confFileName) { .filter(StringUtils::isNotEmpty) .orElse(Storage.getIndexSwitchFromConfig(config))); - INSTANCE.storage.setTransactionHistoreSwitch(Optional.ofNullable(INSTANCE.storageTransactionHistoreSwitch) - .filter(StringUtils::isNotEmpty) - .orElse(Storage.getTransactionHistoreSwitchFromConfig(config))); - + INSTANCE.storage + .setTransactionHistoreSwitch(Optional.ofNullable(INSTANCE.storageTransactionHistoreSwitch) + .filter(StringUtils::isNotEmpty) + .orElse(Storage.getTransactionHistoreSwitchFromConfig(config))); INSTANCE.storage.setPropertyMapFromConfig(config); @@ -846,8 +852,9 @@ public static void setParam(final String[] args, final String confFileName) { INSTANCE.eventFilter = config.hasPath("event.subscribe.filter") ? getEventFilter(config) : null; - initRocksDbBackupProperty(config); initBackupProperty(config); + initRocksDbBackupProperty(config); + initRocksDbSettings(config); logConfig(); } @@ -1128,6 +1135,38 @@ private static double calcMaxTimeRatio() { } + private static void initRocksDbSettings(Config config) { + String prefix = "storage.dbSettings."; + + if (Args.getInstance().getStorage().getDbVersion() == 3) { + int levelNumber = config.hasPath(prefix + "levelNumber") + ? config.getInt(prefix + "levelNumber") : 6; + int compactThreads = config.hasPath(prefix + "compactThreads") + ? config.getInt(prefix + "compactThreads") : 8; + int blocksize = config.hasPath(prefix + "blocksize") + ? config.getInt(prefix + "blocksize") : 16; + long maxBytesForLevelBase = config.hasPath(prefix + "maxBytesForLevelBase") + ? config.getInt(prefix + "maxBytesForLevelBase") : 256; + double maxBytesForLevelMultiplier = config.hasPath(prefix + "maxBytesForLevelMultiplier") + ? config.getDouble(prefix + "maxBytesForLevelMultiplier") : 10; + String compressionStr = config.hasPath(prefix + "compressionTypeListStr") + ? config.getString(prefix + "compressionTypeListStr") : "no:no:lz4:lz4:zstd:zstd"; + int level0FileNumCompactionTrigger = + config.hasPath(prefix + "level0FileNumCompactionTrigger") ? config + .getInt(prefix + "level0FileNumCompactionTrigger") : 2; + long targetFileSizeBase = config.hasPath(prefix + "targetFileSizeBase") ? config + .getLong(prefix + "targetFileSizeBase") : 64; + int targetFileSizeMultiplier = config.hasPath(prefix + "targetFileSizeMultiplier") ? config + .getInt(prefix + "targetFileSizeMultiplier") : 1; + + INSTANCE.rocksDBCustomSettings = DBSettings + .initCustomSettings(levelNumber, compactThreads, blocksize, maxBytesForLevelBase, + maxBytesForLevelMultiplier, compressionStr, level0FileNumCompactionTrigger, + targetFileSizeBase, targetFileSizeMultiplier); + DBSettings.loggingSettings(); + } + } + private static void initRocksDbBackupProperty(Config config) { boolean enable = false; if (Args.getInstance().getStorage().getDbVersion() == 3) { @@ -1146,6 +1185,7 @@ private static void initRocksDbBackupProperty(Config config) { .initArgs(enable, propPath, bak1path, bak2path, frequency); } + private static void initBackupProperty(Config config) { INSTANCE.backupPriority = config.hasPath("node.backup.priority") ? config.getInt("node.backup.priority") : 0; diff --git a/src/main/java/org/tron/core/db/backup/BackupRocksDBAspect.java b/src/main/java/org/tron/core/db/backup/BackupRocksDBAspect.java index 64cfc4dd1f6..e591311d599 100644 --- a/src/main/java/org/tron/core/db/backup/BackupRocksDBAspect.java +++ b/src/main/java/org/tron/core/db/backup/BackupRocksDBAspect.java @@ -11,7 +11,7 @@ @Slf4j @Aspect public class BackupRocksDBAspect { - + @Autowired private BackupDbUtil util; diff --git a/src/main/resources/config.conf b/src/main/resources/config.conf index fbd9e25d5cd..98b18937741 100644 --- a/src/main/resources/config.conf +++ b/src/main/resources/config.conf @@ -49,7 +49,21 @@ storage { needToUpdateAsset = true - //backup settings when use rocks db as the storage implement (db.version=3). + //dbsettings is needed when using rocksdb as the storage implement (db.version=3). + //we'd strongly recommend that do not modify it unless you know every item's meaning clearly. + dbSettings = { + levelNumber = 7 + compactThreads = 32 + blocksize = 64 // n * KB + maxBytesForLevelBase = 256 // n * MB + maxBytesForLevelMultiplier = 10 + level0FileNumCompactionTrigger = 4 + compressionTypeListStr = "no:no:lz4:lz4:lz4:zstd:zstd" + targetFileSizeBase = 256 // n * MB + targetFileSizeMultiplier = 1 + } + + //backup settings when using rocks db as the storage implement (db.version=3). //if you want to use the backup plugin, please confirm set the db.version=3 above. backup = { enable = true // indicate whether enable the backup plugin From a93959289ed4b60147d90dc313ba1e59ea73ff81 Mon Sep 17 00:00:00 2001 From: sun haoyu Date: Mon, 21 Jan 2019 13:50:44 +0800 Subject: [PATCH 05/13] optimise test code --- .../tron/core/capsule/utils/BlockUtil.java | 17 ----- .../java/org/tron/core/config/args/Args.java | 57 ++++++++-------- src/main/java/org/tron/core/db/Manager.java | 2 + src/main/resources/config-dbbackup.conf | 24 +++++-- src/main/resources/config.conf | 3 +- src/main/resources/logback.xml | 18 ++--- .../java/org/tron/core/BlockUtilTest.java | 1 - .../tron/core/capsule/utils/BlockUtil.java | 65 +++++++++++++++++++ .../java/org/tron/core/db/ManagerForTest.java | 0 .../net/node/GetBlockChainSummaryTest.java | 1 - .../core/net/node/GetLostBlockIdsTest.java | 1 - .../org/tron/core/net/node/TcpNetTest.java | 1 - 12 files changed, 126 insertions(+), 64 deletions(-) create mode 100644 src/test/java/org/tron/core/capsule/utils/BlockUtil.java rename src/{main => test}/java/org/tron/core/db/ManagerForTest.java (100%) diff --git a/src/main/java/org/tron/core/capsule/utils/BlockUtil.java b/src/main/java/org/tron/core/capsule/utils/BlockUtil.java index 9420bcf53b8..ea422623db0 100644 --- a/src/main/java/org/tron/core/capsule/utils/BlockUtil.java +++ b/src/main/java/org/tron/core/capsule/utils/BlockUtil.java @@ -17,15 +17,11 @@ import com.google.protobuf.ByteString; import java.util.List; -import java.util.Map; import java.util.stream.Collectors; import org.tron.common.utils.ByteArray; -import org.tron.common.utils.Sha256Hash; import org.tron.core.capsule.BlockCapsule; import org.tron.core.config.args.Args; import org.tron.core.config.args.GenesisBlock; -import org.tron.core.db.Manager; -import org.tron.core.witness.WitnessController; import org.tron.protos.Protocol.Transaction; public class BlockUtil { @@ -66,17 +62,4 @@ public static BlockCapsule newGenesisBlockCapsule() { public static boolean isParentOf(BlockCapsule blockCapsule1, BlockCapsule blockCapsule2) { return blockCapsule1.getBlockId().equals(blockCapsule2.getParentHash()); } - - public static BlockCapsule createTestBlockCapsule(Manager dbManager, long time, - long number, ByteString hash, Map addressToProvateKeys) { - WitnessController witnessController = dbManager.getWitnessController(); - ByteString witnessAddress = - witnessController.getScheduledWitness(witnessController.getSlotAtTime(time)); - BlockCapsule blockCapsule = new BlockCapsule(number, Sha256Hash.wrap(hash), time, - witnessAddress); - blockCapsule.generatedByMyself = true; - blockCapsule.setMerkleRoot(); - blockCapsule.sign(ByteArray.fromHexString(addressToProvateKeys.get(witnessAddress))); - return blockCapsule; - } } diff --git a/src/main/java/org/tron/core/config/args/Args.java b/src/main/java/org/tron/core/config/args/Args.java index 50115436b33..75e91de9517 100644 --- a/src/main/java/org/tron/core/config/args/Args.java +++ b/src/main/java/org/tron/core/config/args/Args.java @@ -627,10 +627,9 @@ public static void setParam(final String[] args, final String confFileName) { .filter(StringUtils::isNotEmpty) .orElse(Storage.getIndexSwitchFromConfig(config))); - INSTANCE.storage - .setTransactionHistoreSwitch(Optional.ofNullable(INSTANCE.storageTransactionHistoreSwitch) - .filter(StringUtils::isNotEmpty) - .orElse(Storage.getTransactionHistoreSwitchFromConfig(config))); + INSTANCE.storage.setTransactionHistoreSwitch(Optional.ofNullable(INSTANCE.storageTransactionHistoreSwitch) + .filter(StringUtils::isNotEmpty) + .orElse(Storage.getTransactionHistoreSwitchFromConfig(config))); INSTANCE.storage.setPropertyMapFromConfig(config); @@ -846,11 +845,11 @@ public static void setParam(final String[] args, final String confFileName) { config.hasPath("vm.saveInternalTx") && config.getBoolean("vm.saveInternalTx"); INSTANCE.eventPluginConfig = - config.hasPath("event.subscribe")? - getEventPluginConfig(config) : null; + config.hasPath("event.subscribe") ? + getEventPluginConfig(config) : null; INSTANCE.eventFilter = - config.hasPath("event.subscribe.filter") ? getEventFilter(config) : null; + config.hasPath("event.subscribe.filter") ? getEventFilter(config) : null; initBackupProperty(config); initRocksDbBackupProperty(config); @@ -942,35 +941,35 @@ private static void privateKey(final com.typesafe.config.Config config) { } } - private static EventPluginConfig getEventPluginConfig(final com.typesafe.config.Config config){ + private static EventPluginConfig getEventPluginConfig(final com.typesafe.config.Config config) { EventPluginConfig eventPluginConfig = new EventPluginConfig(); - if (config.hasPath("event.subscribe.path")){ + if (config.hasPath("event.subscribe.path")) { String pluginPath = config.getString("event.subscribe.path"); - if (StringUtils.isNotEmpty(pluginPath)){ + if (StringUtils.isNotEmpty(pluginPath)) { eventPluginConfig.setPluginPath(pluginPath.trim()); } } - - if (config.hasPath("event.subscribe.server")){ + if (config.hasPath("event.subscribe.server")) { String serverAddress = config.getString("event.subscribe.server"); - if (StringUtils.isNotEmpty(serverAddress)){ + if (StringUtils.isNotEmpty(serverAddress)) { eventPluginConfig.setServerAddress(serverAddress.trim()); } } - if (config.hasPath("event.subscribe.dbconfig")){ + if (config.hasPath("event.subscribe.dbconfig")) { String dbConfig = config.getString("event.subscribe.dbconfig"); - if (StringUtils.isNotEmpty(dbConfig)){ + if (StringUtils.isNotEmpty(dbConfig)) { eventPluginConfig.setDbConfig(dbConfig.trim()); } } - if (config.hasPath("event.subscribe.topics")){ - List triggerConfigList = config.getObjectList("event.subscribe.topics").stream() - .map(Args::createTriggerConfig) - .collect(Collectors.toCollection(ArrayList::new)); + if (config.hasPath("event.subscribe.topics")) { + List triggerConfigList = config.getObjectList("event.subscribe.topics") + .stream() + .map(Args::createTriggerConfig) + .collect(Collectors.toCollection(ArrayList::new)); eventPluginConfig.setTriggerConfigList(triggerConfigList); } @@ -978,8 +977,8 @@ private static EventPluginConfig getEventPluginConfig(final com.typesafe.config. return eventPluginConfig; } - private static TriggerConfig createTriggerConfig(ConfigObject triggerObject){ - if (Objects.isNull(triggerObject)){ + private static TriggerConfig createTriggerConfig(ConfigObject triggerObject) { + if (Objects.isNull(triggerObject)) { return null; } @@ -989,7 +988,7 @@ private static TriggerConfig createTriggerConfig(ConfigObject triggerObject){ triggerConfig.setTriggerName(triggerName); String enabled = triggerObject.get("enable").unwrapped().toString(); - triggerConfig.setEnabled("true".equalsIgnoreCase(enabled) ? true: false); + triggerConfig.setEnabled("true".equalsIgnoreCase(enabled) ? true : false); String topic = triggerObject.get("topic").unwrapped().toString(); triggerConfig.setTopic(topic); @@ -997,14 +996,14 @@ private static TriggerConfig createTriggerConfig(ConfigObject triggerObject){ return triggerConfig; } - private static FilterQuery getEventFilter(final com.typesafe.config.Config config){ + private static FilterQuery getEventFilter(final com.typesafe.config.Config config) { FilterQuery filter = new FilterQuery(); - long fromBlockLong = 0, toBlockLong = 0; + long fromBlockLong = 0, toBlockLong = 0; String fromBlock = config.getString("event.subscribe.filter.fromblock").trim(); try { - fromBlockLong = FilterQuery.parseFromBlockNumber(fromBlock); - } catch (Exception e){ + fromBlockLong = FilterQuery.parseFromBlockNumber(fromBlock); + } catch (Exception e) { logger.error("{}", e); return null; } @@ -1013,7 +1012,7 @@ private static FilterQuery getEventFilter(final com.typesafe.config.Config confi String toBlock = config.getString("event.subscribe.filter.toblock").trim(); try { toBlockLong = FilterQuery.parseToBlockNumber(toBlock); - } catch (Exception e){ + } catch (Exception e) { logger.error("{}", e); return null; } @@ -1021,12 +1020,12 @@ private static FilterQuery getEventFilter(final com.typesafe.config.Config confi List addressList = config.getStringList("event.subscribe.filter.contractAddress"); addressList = addressList.stream().filter(address -> StringUtils.isNotEmpty(address)).collect( - Collectors.toList()); + Collectors.toList()); filter.setContractAddressList(addressList); List topicList = config.getStringList("event.subscribe.filter.contractTopic"); topicList = topicList.stream().filter(top -> StringUtils.isNotEmpty(top)).collect( - Collectors.toList()); + Collectors.toList()); filter.setContractTopicList(topicList); return filter; diff --git a/src/main/java/org/tron/core/db/Manager.java b/src/main/java/org/tron/core/db/Manager.java index d3dae5158ee..802c9ec9726 100644 --- a/src/main/java/org/tron/core/db/Manager.java +++ b/src/main/java/org/tron/core/db/Manager.java @@ -1679,6 +1679,7 @@ public void setAccountIndexStore(AccountIndexStore indexStore) { } public void closeAllStore() { + logger.info("******** begin to close db ********"); closeOneStore(accountStore); closeOneStore(blockStore); closeOneStore(blockIndexStore); @@ -1701,6 +1702,7 @@ public void closeAllStore() { closeOneStore(delegatedResourceStore); closeOneStore(assetIssueV2Store); closeOneStore(exchangeV2Store); + logger.info("******** end to close db ********"); } public void closeOneStore(ITronChainBase database) { diff --git a/src/main/resources/config-dbbackup.conf b/src/main/resources/config-dbbackup.conf index 3008892b54b..cb7cb6ab46d 100644 --- a/src/main/resources/config-dbbackup.conf +++ b/src/main/resources/config-dbbackup.conf @@ -48,12 +48,28 @@ storage { needToUpdateAsset = true + //dbsettings is needed when using rocksdb as the storage implement (db.version=3). + //we'd strongly recommend that do not modify it unless you know every item's meaning clearly. + dbSettings = { + levelNumber = 7 + compactThreads = 32 + blocksize = 64 // n * KB + maxBytesForLevelBase = 256 // n * MB + maxBytesForLevelMultiplier = 10 + level0FileNumCompactionTrigger = 4 + compressionTypeListStr = "no:no:lz4:lz4:lz4:zstd:zstd" + targetFileSizeBase = 256 // n * MB + targetFileSizeMultiplier = 1 + } + + //backup settings when using rocks db as the storage implement (db.version=3). + //if you want to use the backup plugin, please confirm set the db.version=3 above. backup = { - enable = true - properties = "prop.properties" - bak1path = "bak1/database" + enable = true // indicate whether enable the backup plugin + properties = "prop.properties" // record which bak directory is valid + bak1path = "bak1/database" // you must set two backup directories to prevent application halt unexpected(e.g. kill -9). bak2path = "bak2/database" - frequency = 5000 // backup db every ? blocks processed. + frequency = 10000 // indicate backup db once every 10000 blocks processed. } } diff --git a/src/main/resources/config.conf b/src/main/resources/config.conf index 98b18937741..454212c00f4 100644 --- a/src/main/resources/config.conf +++ b/src/main/resources/config.conf @@ -5,7 +5,8 @@ net { storage { # Directory for storing persistent data - db.version = 3, + db.version = 2, + db.sync = false, db.directory = "database", index.directory = "index", transHistory.switch = "on", diff --git a/src/main/resources/logback.xml b/src/main/resources/logback.xml index 8969464a6a2..ee4a685e636 100644 --- a/src/main/resources/logback.xml +++ b/src/main/resources/logback.xml @@ -3,16 +3,16 @@ - + - - - %d{HH:mm:ss.SSS} %-5level [%t] [%c{1}]\(%F:%L\) %m%n - - - INFO - - + + + + + + + + diff --git a/src/test/java/org/tron/core/BlockUtilTest.java b/src/test/java/org/tron/core/BlockUtilTest.java index 50fe53cc223..25e998d7a70 100644 --- a/src/test/java/org/tron/core/BlockUtilTest.java +++ b/src/test/java/org/tron/core/BlockUtilTest.java @@ -73,6 +73,5 @@ public void testBlockUtil() { Assert.assertFalse(BlockUtil.isParentOf(blockCapsule1, blockCapsule2)); Assert.assertEquals(true, BlockUtil.isParentOf(blockCapsule2, blockCapsule3)); Assert.assertTrue(BlockUtil.isParentOf(blockCapsule2, blockCapsule3)); - } } diff --git a/src/test/java/org/tron/core/capsule/utils/BlockUtil.java b/src/test/java/org/tron/core/capsule/utils/BlockUtil.java new file mode 100644 index 00000000000..4e5fae474d7 --- /dev/null +++ b/src/test/java/org/tron/core/capsule/utils/BlockUtil.java @@ -0,0 +1,65 @@ +package org.tron.core.capsule.utils; + + +import com.google.protobuf.ByteString; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; +import org.tron.common.utils.ByteArray; +import org.tron.common.utils.Sha256Hash; +import org.tron.core.capsule.BlockCapsule; +import org.tron.core.config.args.Args; +import org.tron.core.config.args.GenesisBlock; +import org.tron.core.db.Manager; +import org.tron.core.witness.WitnessController; +import org.tron.protos.Protocol.Transaction; + +public class BlockUtil { + + /** + * create genesis block from transactions. + */ + public static BlockCapsule newGenesisBlockCapsule() { + + Args args = Args.getInstance(); + GenesisBlock genesisBlockArg = args.getGenesisBlock(); + List transactionList = + genesisBlockArg.getAssets().stream() + .map(key -> { + byte[] address = key.getAddress(); + long balance = key.getBalance(); + return TransactionUtil.newGenesisTransaction(address, balance); + }) + .collect(Collectors.toList()); + + long timestamp = Long.parseLong(genesisBlockArg.getTimestamp()); + ByteString parentHash = + ByteString.copyFrom(ByteArray.fromHexString(genesisBlockArg.getParentHash())); + long number = Long.parseLong(genesisBlockArg.getNumber()); + + BlockCapsule blockCapsule = new BlockCapsule(timestamp, parentHash, number, transactionList); + + blockCapsule.setMerkleRoot(); + blockCapsule.setWitness("A new system must allow existing systems to be linked together without requiring any central control or coordination"); + blockCapsule.generatedByMyself = true; + + return blockCapsule; + } + + public static boolean isParentOf(BlockCapsule blockCapsule1, BlockCapsule blockCapsule2) { + return blockCapsule1.getBlockId().equals(blockCapsule2.getParentHash()); + } + + public static BlockCapsule createTestBlockCapsule(Manager dbManager, long time, + long number, ByteString hash, Map addressToProvateKeys) { + WitnessController witnessController = dbManager.getWitnessController(); + ByteString witnessAddress = + witnessController.getScheduledWitness(witnessController.getSlotAtTime(time)); + BlockCapsule blockCapsule = new BlockCapsule(number, Sha256Hash.wrap(hash), time, + witnessAddress); + blockCapsule.generatedByMyself = true; + blockCapsule.setMerkleRoot(); + blockCapsule.sign(ByteArray.fromHexString(addressToProvateKeys.get(witnessAddress))); + return blockCapsule; + } +} \ No newline at end of file diff --git a/src/main/java/org/tron/core/db/ManagerForTest.java b/src/test/java/org/tron/core/db/ManagerForTest.java similarity index 100% rename from src/main/java/org/tron/core/db/ManagerForTest.java rename to src/test/java/org/tron/core/db/ManagerForTest.java diff --git a/src/test/java/org/tron/core/net/node/GetBlockChainSummaryTest.java b/src/test/java/org/tron/core/net/node/GetBlockChainSummaryTest.java index 54f32288ce3..dea1da0e7fa 100644 --- a/src/test/java/org/tron/core/net/node/GetBlockChainSummaryTest.java +++ b/src/test/java/org/tron/core/net/node/GetBlockChainSummaryTest.java @@ -16,7 +16,6 @@ import org.junit.AfterClass; import org.junit.Assert; import org.junit.Before; -import org.junit.Ignore; import org.junit.Test; import org.tron.common.application.Application; import org.tron.common.application.ApplicationFactory; diff --git a/src/test/java/org/tron/core/net/node/GetLostBlockIdsTest.java b/src/test/java/org/tron/core/net/node/GetLostBlockIdsTest.java index c8eb1a0aaef..1b09c04cd34 100644 --- a/src/test/java/org/tron/core/net/node/GetLostBlockIdsTest.java +++ b/src/test/java/org/tron/core/net/node/GetLostBlockIdsTest.java @@ -18,7 +18,6 @@ import org.junit.AfterClass; import org.junit.Assert; import org.junit.Before; -import org.junit.Ignore; import org.junit.Test; import org.tron.common.application.Application; import org.tron.common.application.ApplicationFactory; diff --git a/src/test/java/org/tron/core/net/node/TcpNetTest.java b/src/test/java/org/tron/core/net/node/TcpNetTest.java index 75344c44bd2..b4dcbf7d9fb 100644 --- a/src/test/java/org/tron/core/net/node/TcpNetTest.java +++ b/src/test/java/org/tron/core/net/node/TcpNetTest.java @@ -26,7 +26,6 @@ import org.junit.After; import org.junit.AfterClass; import org.junit.Assert; -import org.junit.Ignore; import org.junit.Test; import org.tron.common.overlay.discover.node.Node; import org.tron.common.overlay.message.DisconnectMessage; From d5f50d7a6850a669f65bd855de6cb7477c684000 Mon Sep 17 00:00:00 2001 From: sun haoyu Date: Tue, 22 Jan 2019 15:58:57 +0800 Subject: [PATCH 06/13] rocksdb2 --- .../tron/common/storage/BatchSourceInter.java | 4 +- .../common/storage/BatchSourceInterRocks.java | 23 ------ .../common/storage/DbSourceInterRocks.java | 25 ------- .../org/tron/common/storage/SourceInter.java | 6 +- .../tron/common/storage/SourceInterRocks.java | 19 ----- .../common/storage/WriteOptionsWrapper.java | 25 +++++++ .../leveldb/LevelDbDataSourceImpl.java | 15 ++-- .../leveldb/RocksDbDataSourceImpl.java | 17 +++-- .../org/tron/core/config/DefaultConfig.java | 9 ++- .../java/org/tron/core/config/args/Args.java | 26 ++++--- .../tron/core/db/AbstractRevokingStore.java | 19 ++--- .../org/tron/core/db/RevokingStoreRocks.java | 28 +++---- .../tron/core/db/TronStoreWithRevoking.java | 6 +- .../org/tron/core/db/backup/BackupDbUtil.java | 32 ++++++-- .../core/db/backup/NeedBeanCondition.java | 2 +- .../org/tron/core/db2/common/LevelDB.java | 3 +- .../org/tron/core/db2/common/RocksDB.java | 73 +++++++++++++++++++ .../core/RevokingDBWithCachingNewValue.java | 13 +--- .../tron/core/db2/core/SnapshotManager.java | 5 +- .../org/tron/core/db2/core/SnapshotRoot.java | 15 ++-- src/main/resources/config.conf | 4 +- 21 files changed, 209 insertions(+), 160 deletions(-) delete mode 100644 src/main/java/org/tron/common/storage/BatchSourceInterRocks.java delete mode 100644 src/main/java/org/tron/common/storage/DbSourceInterRocks.java delete mode 100644 src/main/java/org/tron/common/storage/SourceInterRocks.java create mode 100644 src/main/java/org/tron/common/storage/WriteOptionsWrapper.java create mode 100644 src/main/java/org/tron/core/db2/common/RocksDB.java diff --git a/src/main/java/org/tron/common/storage/BatchSourceInter.java b/src/main/java/org/tron/common/storage/BatchSourceInter.java index 1ed8a40f970..3782c20f2aa 100644 --- a/src/main/java/org/tron/common/storage/BatchSourceInter.java +++ b/src/main/java/org/tron/common/storage/BatchSourceInter.java @@ -18,8 +18,6 @@ package org.tron.common.storage; -import org.iq80.leveldb.WriteOptions; - import java.util.Map; @@ -28,5 +26,5 @@ public interface BatchSourceInter extends SourceInter { void updateByBatch(Map rows); - void updateByBatch(Map rows, WriteOptions writeOptions); + void updateByBatch(Map rows, WriteOptionsWrapper writeOptions); } diff --git a/src/main/java/org/tron/common/storage/BatchSourceInterRocks.java b/src/main/java/org/tron/common/storage/BatchSourceInterRocks.java deleted file mode 100644 index 3a0eb814e9b..00000000000 --- a/src/main/java/org/tron/common/storage/BatchSourceInterRocks.java +++ /dev/null @@ -1,23 +0,0 @@ -package org.tron.common.storage; - -import java.util.Map; -import org.rocksdb.WriteOptions; - - -public interface BatchSourceInterRocks extends SourceInterRocks { - - - void updateByBatch(Map rows); - - void updateByBatch(Map rows, WriteOptions writeOptions); - - void putData(K key, V val); - - V getData(K key); - - - void deleteData(K key); - - boolean flush(); - -} diff --git a/src/main/java/org/tron/common/storage/DbSourceInterRocks.java b/src/main/java/org/tron/common/storage/DbSourceInterRocks.java deleted file mode 100644 index d5907401da7..00000000000 --- a/src/main/java/org/tron/common/storage/DbSourceInterRocks.java +++ /dev/null @@ -1,25 +0,0 @@ -package org.tron.common.storage; - -import java.util.Set; - -public interface DbSourceInterRocks extends BatchSourceInterRocks { - - String getDBName(); - - void setDBName(String name); - - void initDB(); - - boolean isAlive(); - - void closeDB(); - - void resetDb(); - - Set allKeys() throws RuntimeException; - - Set allValues() throws RuntimeException; - - long getTotal() throws RuntimeException; - -} diff --git a/src/main/java/org/tron/common/storage/SourceInter.java b/src/main/java/org/tron/common/storage/SourceInter.java index bba9fd5e04c..1bfea4a6302 100644 --- a/src/main/java/org/tron/common/storage/SourceInter.java +++ b/src/main/java/org/tron/common/storage/SourceInter.java @@ -18,21 +18,19 @@ package org.tron.common.storage; -import org.iq80.leveldb.WriteOptions; - public interface SourceInter { void putData(K key, V val); - void putData(K k, V v, WriteOptions options); + void putData(K k, V v, WriteOptionsWrapper options); V getData(K key); void deleteData(K key); - void deleteData(K k, WriteOptions options); + void deleteData(K k, WriteOptionsWrapper options); boolean flush(); diff --git a/src/main/java/org/tron/common/storage/SourceInterRocks.java b/src/main/java/org/tron/common/storage/SourceInterRocks.java deleted file mode 100644 index 71096b9ec5f..00000000000 --- a/src/main/java/org/tron/common/storage/SourceInterRocks.java +++ /dev/null @@ -1,19 +0,0 @@ -package org.tron.common.storage; - - -import org.rocksdb.WriteOptions; - -public interface SourceInterRocks { - - void putData(K key, V val); - - void putData(K k, V v, WriteOptions options); - - V getData(K key); - - void deleteData(K key); - - void deleteData(K k, WriteOptions options); - - boolean flush(); -} diff --git a/src/main/java/org/tron/common/storage/WriteOptionsWrapper.java b/src/main/java/org/tron/common/storage/WriteOptionsWrapper.java new file mode 100644 index 00000000000..85d463940df --- /dev/null +++ b/src/main/java/org/tron/common/storage/WriteOptionsWrapper.java @@ -0,0 +1,25 @@ +package org.tron.common.storage; + +public class WriteOptionsWrapper { + + public org.rocksdb.WriteOptions rocks = null; + public org.iq80.leveldb.WriteOptions level = null; + + private WriteOptionsWrapper() { + + } + + public static WriteOptionsWrapper getInstance() { + WriteOptionsWrapper wapper = new WriteOptionsWrapper(); + wapper.level = new org.iq80.leveldb.WriteOptions(); + wapper.rocks = new org.rocksdb.WriteOptions(); + return wapper; + } + + + public WriteOptionsWrapper sync(boolean bool) { + this.level.sync(bool); + this.rocks.setSync(bool); + return this; + } +} \ No newline at end of file diff --git a/src/main/java/org/tron/common/storage/leveldb/LevelDbDataSourceImpl.java b/src/main/java/org/tron/common/storage/leveldb/LevelDbDataSourceImpl.java index dfab5a43d27..b375a817004 100644 --- a/src/main/java/org/tron/common/storage/leveldb/LevelDbDataSourceImpl.java +++ b/src/main/java/org/tron/common/storage/leveldb/LevelDbDataSourceImpl.java @@ -43,6 +43,7 @@ import org.iq80.leveldb.WriteBatch; import org.iq80.leveldb.WriteOptions; import org.tron.common.storage.DbSourceInter; +import org.tron.common.storage.WriteOptionsWrapper; import org.tron.common.utils.FileUtil; import org.tron.core.config.args.Args; import org.tron.core.db.common.iterator.StoreIterator; @@ -207,10 +208,10 @@ public void putData(byte[] key, byte[] value) { } @Override - public void putData(byte[] key, byte[] value, WriteOptions options) { + public void putData(byte[] key, byte[] value, WriteOptionsWrapper options) { resetDbLock.readLock().lock(); try { - database.put(key, value, options); + database.put(key, value, options.level); } finally { resetDbLock.readLock().unlock(); } @@ -227,10 +228,10 @@ public void deleteData(byte[] key) { } @Override - public void deleteData(byte[] key, WriteOptions options) { + public void deleteData(byte[] key, WriteOptionsWrapper options) { resetDbLock.readLock().lock(); try { - database.delete(key, options); + database.delete(key, options.level); } finally { resetDbLock.readLock().unlock(); } @@ -416,13 +417,13 @@ public void updateByBatch(Map rows) { } @Override - public void updateByBatch(Map rows, WriteOptions options) { + public void updateByBatch(Map rows, WriteOptionsWrapper options) { resetDbLock.readLock().lock(); try { - updateByBatchInner(rows, options); + updateByBatchInner(rows, options.level); } catch (Exception e) { try { - updateByBatchInner(rows, options); + updateByBatchInner(rows, options.level); } catch (Exception e1) { throw new RuntimeException(e); } diff --git a/src/main/java/org/tron/common/storage/leveldb/RocksDbDataSourceImpl.java b/src/main/java/org/tron/common/storage/leveldb/RocksDbDataSourceImpl.java index 43fee75dc55..1c5ba55bc81 100644 --- a/src/main/java/org/tron/common/storage/leveldb/RocksDbDataSourceImpl.java +++ b/src/main/java/org/tron/common/storage/leveldb/RocksDbDataSourceImpl.java @@ -27,14 +27,15 @@ import org.rocksdb.WriteBatch; import org.rocksdb.WriteOptions; import org.tron.common.storage.DBSettings; -import org.tron.common.storage.DbSourceInterRocks; +import org.tron.common.storage.DbSourceInter; +import org.tron.common.storage.WriteOptionsWrapper; import org.tron.common.utils.FileUtil; import org.tron.core.config.args.Args; import org.tron.core.db.common.iterator.RockStoreIterator; @Slf4j @NoArgsConstructor -public class RocksDbDataSourceImpl implements DbSourceInterRocks, +public class RocksDbDataSourceImpl implements DbSourceInter, Iterable> { private String dataBaseName; @@ -226,13 +227,13 @@ public void putData(byte[] key, byte[] value) { } @Override - public void putData(byte[] key, byte[] value, WriteOptions writeOpt) { + public void putData(byte[] key, byte[] value, WriteOptionsWrapper optionsWrapper) { if (quitIfNotAlive()) { return; } resetDbLock.readLock().lock(); try { - database.put(writeOpt, key, value); + database.put(optionsWrapper.rocks, key, value); } catch (RocksDBException e) { logger.error("RocksDBException:{}", e); } finally { @@ -272,13 +273,13 @@ public void deleteData(byte[] key) { } @Override - public void deleteData(byte[] key, WriteOptions writeOpt) { + public void deleteData(byte[] key, WriteOptionsWrapper optionsWrapper) { if (quitIfNotAlive()) { return; } resetDbLock.readLock().lock(); try { - database.delete(writeOpt, key); + database.delete(optionsWrapper.rocks, key); } catch (RocksDBException e) { logger.error("RocksDBException:{}", e); } finally { @@ -349,13 +350,13 @@ public void updateByBatch(Map rows) { } @Override - public void updateByBatch(Map rows, WriteOptions writeOptions) { + public void updateByBatch(Map rows, WriteOptionsWrapper optionsWrapper) { if (quitIfNotAlive()) { return; } resetDbLock.readLock().lock(); try { - updateByBatchInner(rows, writeOptions); + updateByBatchInner(rows, optionsWrapper.rocks); } catch (Exception e) { try { updateByBatchInner(rows); diff --git a/src/main/java/org/tron/core/config/DefaultConfig.java b/src/main/java/org/tron/core/config/DefaultConfig.java index 0bef8ae0d75..90c84559acf 100755 --- a/src/main/java/org/tron/core/config/DefaultConfig.java +++ b/src/main/java/org/tron/core/config/DefaultConfig.java @@ -2,6 +2,7 @@ import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.BooleanUtils; +import org.rocksdb.RocksDB; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.ApplicationContext; import org.springframework.context.annotation.Bean; @@ -11,8 +12,8 @@ import org.tron.core.config.args.Args; import org.tron.core.db.RevokingDatabase; import org.tron.core.db.RevokingStore; -import org.tron.core.db.TransactionCache; import org.tron.core.db.RevokingStoreRocks; +import org.tron.core.db.TransactionCache; import org.tron.core.db.api.IndexHelper; import org.tron.core.db.backup.BackupRocksDBAspect; import org.tron.core.db.backup.NeedBeanCondition; @@ -25,6 +26,10 @@ @Import(CommonConfig.class) public class DefaultConfig { + static { + RocksDB.loadLibrary(); + } + @Autowired ApplicationContext appCtx; @@ -55,6 +60,8 @@ public RevokingDatabase revokingDatabase() { revokingDatabase = new SnapshotManager(); } else if (dbVersion == 3) { revokingDatabase = RevokingStoreRocks.getInstance(); + } else if (dbVersion == 4) { + revokingDatabase = new SnapshotManager(); } else { throw new RuntimeException("db version is error."); } diff --git a/src/main/java/org/tron/core/config/args/Args.java b/src/main/java/org/tron/core/config/args/Args.java index 75e91de9517..d126b415814 100644 --- a/src/main/java/org/tron/core/config/args/Args.java +++ b/src/main/java/org/tron/core/config/args/Args.java @@ -1,5 +1,7 @@ package org.tron.core.config.args; +import static java.lang.Math.max; + import com.beust.jcommander.JCommander; import com.beust.jcommander.Parameter; import com.typesafe.config.Config; @@ -627,9 +629,10 @@ public static void setParam(final String[] args, final String confFileName) { .filter(StringUtils::isNotEmpty) .orElse(Storage.getIndexSwitchFromConfig(config))); - INSTANCE.storage.setTransactionHistoreSwitch(Optional.ofNullable(INSTANCE.storageTransactionHistoreSwitch) - .filter(StringUtils::isNotEmpty) - .orElse(Storage.getTransactionHistoreSwitchFromConfig(config))); + INSTANCE.storage + .setTransactionHistoreSwitch(Optional.ofNullable(INSTANCE.storageTransactionHistoreSwitch) + .filter(StringUtils::isNotEmpty) + .orElse(Storage.getTransactionHistoreSwitchFromConfig(config))); INSTANCE.storage.setPropertyMapFromConfig(config); @@ -1137,11 +1140,13 @@ private static double calcMaxTimeRatio() { private static void initRocksDbSettings(Config config) { String prefix = "storage.dbSettings."; - if (Args.getInstance().getStorage().getDbVersion() == 3) { + if (Args.getInstance().getStorage().getDbVersion() == 3 + || Args.getInstance().getStorage().getDbVersion() == 4) { int levelNumber = config.hasPath(prefix + "levelNumber") - ? config.getInt(prefix + "levelNumber") : 6; + ? config.getInt(prefix + "levelNumber") : 7; int compactThreads = config.hasPath(prefix + "compactThreads") - ? config.getInt(prefix + "compactThreads") : 8; + ? config.getInt(prefix + "compactThreads") + : max(Runtime.getRuntime().availableProcessors(), 1); int blocksize = config.hasPath(prefix + "blocksize") ? config.getInt(prefix + "blocksize") : 16; long maxBytesForLevelBase = config.hasPath(prefix + "maxBytesForLevelBase") @@ -1149,7 +1154,7 @@ private static void initRocksDbSettings(Config config) { double maxBytesForLevelMultiplier = config.hasPath(prefix + "maxBytesForLevelMultiplier") ? config.getDouble(prefix + "maxBytesForLevelMultiplier") : 10; String compressionStr = config.hasPath(prefix + "compressionTypeListStr") - ? config.getString(prefix + "compressionTypeListStr") : "no:no:lz4:lz4:zstd:zstd"; + ? config.getString(prefix + "compressionTypeListStr") : "no:no:lz4:lz4:lz4:zstd:zstd"; int level0FileNumCompactionTrigger = config.hasPath(prefix + "level0FileNumCompactionTrigger") ? config .getInt(prefix + "level0FileNumCompactionTrigger") : 2; @@ -1168,9 +1173,10 @@ private static void initRocksDbSettings(Config config) { private static void initRocksDbBackupProperty(Config config) { boolean enable = false; - if (Args.getInstance().getStorage().getDbVersion() == 3) { - enable = config.hasPath("storage.backup.enable") ? config.getBoolean("storage.backup.enable") - : false; + if (Args.getInstance().getStorage().getDbVersion() == 3 + || Args.getInstance().getStorage().getDbVersion() == 4) { + enable = + config.hasPath("storage.backup.enable") && config.getBoolean("storage.backup.enable"); } String propPath = config.hasPath("storage.backup.propPath") ? config.getString("storage.backup.propPath") : "prop.properties"; diff --git a/src/main/java/org/tron/core/db/AbstractRevokingStore.java b/src/main/java/org/tron/core/db/AbstractRevokingStore.java index e56344c7ea8..aa710737628 100644 --- a/src/main/java/org/tron/core/db/AbstractRevokingStore.java +++ b/src/main/java/org/tron/core/db/AbstractRevokingStore.java @@ -22,6 +22,7 @@ import lombok.extern.slf4j.Slf4j; import org.iq80.leveldb.WriteOptions; import org.tron.common.storage.SourceInter; +import org.tron.common.storage.WriteOptionsWrapper; import org.tron.common.storage.leveldb.LevelDbDataSourceImpl; import org.tron.common.utils.FileUtil; import org.tron.common.utils.Utils; @@ -41,7 +42,7 @@ public abstract class AbstractRevokingStore implements RevokingDatabase { private boolean disabled = true; private int activeDialog = 0; private AtomicInteger maxSize = new AtomicInteger(DEFAULT_STACK_MAX_SIZE); - private WriteOptions writeOptions = new WriteOptions() + private WriteOptionsWrapper writeOptionsWrapper = WriteOptionsWrapper.getInstance() .sync(Args.getInstance().getStorage().isDbSync()); private List dbs = new ArrayList<>(); @@ -97,10 +98,10 @@ public synchronized void check() { byte[] realValue = value.length == 1 ? null : Arrays.copyOfRange(value, 1, value.length); if (realValue != null) { - dbMap.get(db).putData(realKey, realValue, new WriteOptions() + dbMap.get(db).putData(realKey, realValue, WriteOptionsWrapper.getInstance() .sync(Args.getInstance().getStorage().isDbSync())); } else { - dbMap.get(db).deleteData(realKey, new WriteOptions() + dbMap.get(db).deleteData(realKey, WriteOptionsWrapper.getInstance() .sync(Args.getInstance().getStorage().isDbSync())); } } @@ -255,15 +256,15 @@ public synchronized void commit() { @Override public synchronized void pop() { - prune(writeOptions); + prune(writeOptionsWrapper); } @Override public synchronized void fastPop() { - prune(new WriteOptions()); + prune(WriteOptionsWrapper.getInstance()); } - private synchronized void prune(WriteOptions options) { + private synchronized void prune(WriteOptionsWrapper optionsWrapper) { if (activeDialog != 0) { throw new RevokingStoreIllegalStateException("activeDialog has to be equal 0"); } @@ -276,9 +277,9 @@ private synchronized void prune(WriteOptions options) { try { RevokingState state = stack.peekLast(); - state.oldValues.forEach((k, v) -> k.database.putData(k.key, v, options)); - state.newIds.forEach(e -> e.database.deleteData(e.key, options)); - state.removed.forEach((k, v) -> k.database.putData(k.key, v, options)); + state.oldValues.forEach((k, v) -> k.database.putData(k.key, v, optionsWrapper)); + state.newIds.forEach(e -> e.database.deleteData(e.key, optionsWrapper)); + state.removed.forEach((k, v) -> k.database.putData(k.key, v, optionsWrapper)); stack.pollLast(); } finally { disabled = false; diff --git a/src/main/java/org/tron/core/db/RevokingStoreRocks.java b/src/main/java/org/tron/core/db/RevokingStoreRocks.java index 21688993992..4c2fb47ec6e 100644 --- a/src/main/java/org/tron/core/db/RevokingStoreRocks.java +++ b/src/main/java/org/tron/core/db/RevokingStoreRocks.java @@ -20,9 +20,9 @@ import lombok.Getter; import lombok.ToString; import lombok.extern.slf4j.Slf4j; -import org.rocksdb.RocksDB; import org.rocksdb.WriteOptions; -import org.tron.common.storage.SourceInterRocks; +import org.tron.common.storage.SourceInter; +import org.tron.common.storage.WriteOptionsWrapper; import org.tron.common.storage.leveldb.RocksDbDataSourceImpl; import org.tron.common.utils.FileUtil; import org.tron.common.utils.Utils; @@ -35,16 +35,12 @@ @Slf4j public class RevokingStoreRocks implements RevokingDatabase { - static { - RocksDB.loadLibrary(); - } - private static final int DEFAULT_STACK_MAX_SIZE = 256; private Deque stack = new LinkedList<>(); private boolean disabled = true; private int activeDialog = 0; private AtomicInteger maxSize = new AtomicInteger(DEFAULT_STACK_MAX_SIZE); - private WriteOptions writeOptions = new WriteOptions().setSync(true); + private WriteOptionsWrapper optionsWrapper = WriteOptionsWrapper.getInstance().sync(true); @Getter private List dbs = new ArrayList<>(); @@ -119,9 +115,9 @@ public synchronized void check() { byte[] realValue = value.length == 1 ? null : Arrays.copyOfRange(value, 1, value.length); if (realValue != null) { - dbMap.get(db).putData(realKey, realValue, new WriteOptions().setSync(true)); + dbMap.get(db).putData(realKey, realValue, WriteOptionsWrapper.getInstance().sync(true)); } else { - dbMap.get(db).deleteData(realKey, new WriteOptions().setSync(true)); + dbMap.get(db).deleteData(realKey, WriteOptionsWrapper.getInstance().sync(true)); } } } @@ -276,15 +272,15 @@ public void commit() throws RevokingStoreIllegalStateException { @Override public void pop() throws RevokingStoreIllegalStateException { - prune(writeOptions); + prune(optionsWrapper); } @Override public void fastPop() throws RevokingStoreIllegalStateException { - prune(new WriteOptions()); + prune(WriteOptionsWrapper.getInstance()); } - private synchronized void prune(WriteOptions options) { + private synchronized void prune(WriteOptionsWrapper optionsWrapper) { if (activeDialog != 0) { throw new RevokingStoreIllegalStateException("activeDialog has to be equal 0"); } @@ -297,9 +293,9 @@ private synchronized void prune(WriteOptions options) { try { RevokingState state = stack.peekLast(); - state.oldValues.forEach((k, v) -> k.database.putData(k.key, v, options)); - state.newIds.forEach(e -> e.database.deleteData(e.key, options)); - state.removed.forEach((k, v) -> k.database.putData(k.key, v, options)); + state.oldValues.forEach((k, v) -> k.database.putData(k.key, v, optionsWrapper)); + state.newIds.forEach(e -> e.database.deleteData(e.key, optionsWrapper)); + state.removed.forEach((k, v) -> k.database.putData(k.key, v, optionsWrapper)); stack.pollLast(); } finally { disabled = false; @@ -475,7 +471,7 @@ static class RevokingState { @ToString public static class RevokingTuple { - private SourceInterRocks database; + private SourceInter database; private byte[] key; } diff --git a/src/main/java/org/tron/core/db/TronStoreWithRevoking.java b/src/main/java/org/tron/core/db/TronStoreWithRevoking.java index d7b0a0dc218..3dbed934d15 100755 --- a/src/main/java/org/tron/core/db/TronStoreWithRevoking.java +++ b/src/main/java/org/tron/core/db/TronStoreWithRevoking.java @@ -18,6 +18,8 @@ import org.tron.core.db.api.IndexHelper; import org.tron.core.db2.common.DB; import org.tron.core.db2.common.IRevokingDB; +import org.tron.core.db2.common.LevelDB; +import org.tron.core.db2.common.RocksDB; import org.tron.core.db2.core.ITronChainBase; import org.tron.core.db2.core.RevokingDBWithCachingNewValue; import org.tron.core.db2.core.RevokingDBWithCachingOldValue; @@ -46,9 +48,11 @@ protected TronStoreWithRevoking(String dbName) { if (dbVersion == 1) { this.revokingDB = new RevokingDBWithCachingOldValue(dbName); } else if (dbVersion == 2) { - this.revokingDB = new RevokingDBWithCachingNewValue(dbName); + this.revokingDB = new RevokingDBWithCachingNewValue(dbName, LevelDB.class); } else if (dbVersion == 3) { this.revokingDB = new RevokingRocksDBWithCachingOldValue(dbName); + } else if (dbVersion == 4) { + this.revokingDB = new RevokingDBWithCachingNewValue(dbName, RocksDB.class); } else { throw new RuntimeException("db version is error."); } diff --git a/src/main/java/org/tron/core/db/backup/BackupDbUtil.java b/src/main/java/org/tron/core/db/backup/BackupDbUtil.java index 514b7ce158a..bed35e5d8d4 100644 --- a/src/main/java/org/tron/core/db/backup/BackupDbUtil.java +++ b/src/main/java/org/tron/core/db/backup/BackupDbUtil.java @@ -2,7 +2,6 @@ import java.util.List; import lombok.Getter; -import lombok.Setter; import lombok.extern.slf4j.Slf4j; import org.rocksdb.RocksDBException; import org.springframework.beans.factory.annotation.Autowired; @@ -13,6 +12,9 @@ import org.tron.core.config.args.Args; import org.tron.core.db.RevokingDatabase; import org.tron.core.db.RevokingStoreRocks; +import org.tron.core.db2.core.RevokingDBWithCachingNewValue; +import org.tron.core.db2.core.SnapshotManager; +import org.tron.core.db2.core.SnapshotRoot; @Slf4j @Component @@ -136,16 +138,32 @@ public void doBackup(BlockCapsule block) { } private void backup(int i) throws RocksDBException { - List stores = ((RevokingStoreRocks) db).getDbs(); - for (RocksDbDataSourceImpl store : stores) { - store.backup(i); + if (Args.getInstance().getStorage().getDbVersion() == 3) { + List stores = ((RevokingStoreRocks) db).getDbs(); + for (RocksDbDataSourceImpl store : stores) { + store.backup(i); + } + } else if (Args.getInstance().getStorage().getDbVersion() == 4) { + List stores = ((SnapshotManager) db).getDbs(); + for (RevokingDBWithCachingNewValue store : stores) { + ((org.tron.core.db2.common.RocksDB) ((SnapshotRoot) (store.getHead().getRoot())).getDb()) + .getDb().backup(i); + } } } private void deleteBackup(int i) { - List stores = ((RevokingStoreRocks) db).getDbs(); - for (RocksDbDataSourceImpl store : stores) { - store.deleteDbBakPath(i); + if (Args.getInstance().getStorage().getDbVersion() == 3) { + List stores = ((RevokingStoreRocks) db).getDbs(); + for (RocksDbDataSourceImpl store : stores) { + store.deleteDbBakPath(i); + } + } else if (Args.getInstance().getStorage().getDbVersion() == 4) { + List stores = ((SnapshotManager) db).getDbs(); + for (RevokingDBWithCachingNewValue store : stores) { + ((org.tron.core.db2.common.RocksDB) ((SnapshotRoot) (store.getHead().getRoot())).getDb()) + .getDb().deleteDbBakPath(i); + } } } } diff --git a/src/main/java/org/tron/core/db/backup/NeedBeanCondition.java b/src/main/java/org/tron/core/db/backup/NeedBeanCondition.java index a9e222bf8b0..5fad3dede66 100644 --- a/src/main/java/org/tron/core/db/backup/NeedBeanCondition.java +++ b/src/main/java/org/tron/core/db/backup/NeedBeanCondition.java @@ -9,7 +9,7 @@ public class NeedBeanCondition implements Condition { @Override public boolean matches(ConditionContext context, AnnotatedTypeMetadata metadata) { - return (Args.getInstance().getStorage().getDbVersion() == 3) && Args.getInstance() + return (Args.getInstance().getStorage().getDbVersion() == 3 || Args.getInstance().getStorage().getDbVersion() == 4) && Args.getInstance() .getDbBackupConfig().isEnable(); } } \ No newline at end of file diff --git a/src/main/java/org/tron/core/db2/common/LevelDB.java b/src/main/java/org/tron/core/db2/common/LevelDB.java index 64cab4bcb4b..6546bc32655 100644 --- a/src/main/java/org/tron/core/db2/common/LevelDB.java +++ b/src/main/java/org/tron/core/db2/common/LevelDB.java @@ -5,6 +5,7 @@ import java.util.Map; import lombok.Getter; import org.iq80.leveldb.WriteOptions; +import org.tron.common.storage.WriteOptionsWrapper; import org.tron.common.storage.leveldb.LevelDbDataSourceImpl; import org.tron.core.config.args.Args; import org.tron.core.db.common.WrappedByteArray; @@ -13,7 +14,7 @@ public class LevelDB implements DB, Flusher { @Getter private LevelDbDataSourceImpl db; - private WriteOptions writeOptions = new WriteOptions() + private WriteOptionsWrapper writeOptions = WriteOptionsWrapper.getInstance() .sync(Args.getInstance().getStorage().isDbSync()); public LevelDB(String parentName, String name) { diff --git a/src/main/java/org/tron/core/db2/common/RocksDB.java b/src/main/java/org/tron/core/db2/common/RocksDB.java new file mode 100644 index 00000000000..d0ea7c3bef7 --- /dev/null +++ b/src/main/java/org/tron/core/db2/common/RocksDB.java @@ -0,0 +1,73 @@ +package org.tron.core.db2.common; + + +import com.google.common.collect.Maps; +import java.util.HashMap; +import java.util.Map; +import lombok.Getter; +import org.tron.common.storage.WriteOptionsWrapper; +import org.tron.common.storage.leveldb.RocksDbDataSourceImpl; +import org.tron.core.config.args.Args; +import org.tron.core.db.common.WrappedByteArray; +import org.tron.core.db.common.iterator.DBIterator; + +public class RocksDB implements DB, Flusher { + + @Getter + private RocksDbDataSourceImpl db; + private WriteOptionsWrapper optionsWrapper = WriteOptionsWrapper.getInstance() + .sync(Args.getInstance().getStorage().isDbSync()); + + public RocksDB(String parentName, String name) { + db = new RocksDbDataSourceImpl(parentName, name); + db.initDB(); + } + + @Override + public byte[] get(byte[] key) { + return db.getData(key); + } + + @Override + public void put(byte[] key, byte[] value) { + db.putData(key, value); + } + + @Override + public long size() { + return db.getTotal(); + } + + @Override + public boolean isEmpty() { + return size() == 0; + } + + @Override + public void remove(byte[] key) { + db.deleteData(key); + } + + @Override + public DBIterator iterator() { + return db.iterator(); + } + + @Override + public void flush(Map batch) { + Map rows = batch.entrySet().stream() + .map(e -> Maps.immutableEntry(e.getKey().getBytes(), e.getValue().getBytes())) + .collect(HashMap::new, (m, k) -> m.put(k.getKey(), k.getValue()), HashMap::putAll); + db.updateByBatch(rows, optionsWrapper); + } + + @Override + public void close() { + db.closeDB(); + } + + @Override + public void reset() { + db.resetDb(); + } +} \ No newline at end of file diff --git a/src/main/java/org/tron/core/db2/core/RevokingDBWithCachingNewValue.java b/src/main/java/org/tron/core/db2/core/RevokingDBWithCachingNewValue.java index 8420507a603..d078dcf05df 100644 --- a/src/main/java/org/tron/core/db2/core/RevokingDBWithCachingNewValue.java +++ b/src/main/java/org/tron/core/db2/core/RevokingDBWithCachingNewValue.java @@ -16,6 +16,7 @@ import org.tron.core.db2.common.DB; import org.tron.core.db2.common.IRevokingDB; import org.tron.core.db2.common.LevelDB; +import org.tron.core.db2.common.RocksDB; import org.tron.core.db2.common.Value; import org.tron.core.exception.ItemNotFoundException; @@ -28,12 +29,6 @@ public class RevokingDBWithCachingNewValue implements IRevokingDB { private String dbName; private Class clz; - public RevokingDBWithCachingNewValue(String dbName) { - this.dbName = dbName; - head = new SnapshotRoot(Args.getInstance().getOutputDirectoryByDbName(dbName), dbName); - mode.set(true); - } - public RevokingDBWithCachingNewValue(String dbName, Class clz) { this.dbName = dbName; this.clz = clz; @@ -74,11 +69,7 @@ public synchronized void close() { public synchronized void reset() { head().reset(); head().close(); - if (clz == null) { - head = new SnapshotRoot(Args.getInstance().getOutputDirectoryByDbName(dbName), dbName); - } else { - head = new SnapshotRoot(Args.getInstance().getOutputDirectoryByDbName(dbName), dbName, clz); - } + head = new SnapshotRoot(Args.getInstance().getOutputDirectoryByDbName(dbName), dbName, clz); } @Override diff --git a/src/main/java/org/tron/core/db2/core/SnapshotManager.java b/src/main/java/org/tron/core/db2/core/SnapshotManager.java index 04f16053982..7f1d5cce3a1 100644 --- a/src/main/java/org/tron/core/db2/core/SnapshotManager.java +++ b/src/main/java/org/tron/core/db2/core/SnapshotManager.java @@ -23,6 +23,7 @@ import lombok.Setter; import lombok.extern.slf4j.Slf4j; import org.iq80.leveldb.WriteOptions; +import org.tron.common.storage.WriteOptionsWrapper; import org.tron.common.storage.leveldb.LevelDbDataSourceImpl; import org.tron.core.config.args.Args; import org.tron.core.db.RevokingDatabase; @@ -320,7 +321,7 @@ private void createCheckPoint() { tmpLevelDbDataSource.updateByBatch(batch.entrySet().stream() .map(e -> Maps.immutableEntry(e.getKey().getBytes(), e.getValue().getBytes())) .collect(HashMap::new, (m, k) -> m.put(k.getKey(), k.getValue()), HashMap::putAll), - new WriteOptions().sync(Args.getInstance().getStorage().isDbSync())); + WriteOptionsWrapper.getInstance().sync(Args.getInstance().getStorage().isDbSync())); } private void deleteCheckPoint() { @@ -331,7 +332,7 @@ private void deleteCheckPoint() { } } - tmpLevelDbDataSource.updateByBatch(hmap, new WriteOptions() + tmpLevelDbDataSource.updateByBatch(hmap, WriteOptionsWrapper.getInstance() .sync(Args.getInstance().getStorage().isDbSync())); } diff --git a/src/main/java/org/tron/core/db2/core/SnapshotRoot.java b/src/main/java/org/tron/core/db2/core/SnapshotRoot.java index 995ef9018e3..5309eab69be 100644 --- a/src/main/java/org/tron/core/db2/core/SnapshotRoot.java +++ b/src/main/java/org/tron/core/db2/core/SnapshotRoot.java @@ -2,7 +2,6 @@ import com.google.common.collect.Maps; import com.google.common.collect.Streams; -import java.lang.ref.WeakReference; import java.lang.reflect.Constructor; import java.lang.reflect.InvocationTargetException; import java.util.HashMap; @@ -15,25 +14,21 @@ import org.tron.core.db2.common.DB; import org.tron.core.db2.common.Flusher; import org.tron.core.db2.common.LevelDB; +import org.tron.core.db2.common.RocksDB; import org.tron.core.db2.common.TxCacheDB; -import org.tron.core.exception.BadItemException; public class SnapshotRoot extends AbstractSnapshot { @Getter private Snapshot solidity; - public SnapshotRoot(String parentName, String name) { - db = new LevelDB(parentName, name); - solidity = this; - } - public SnapshotRoot(String parentName, String name, Class clz) { try { - if (clz == LevelDB.class) { + if (clz == LevelDB.class || clz == RocksDB.class) { Constructor constructor = clz.getConstructor(String.class, String.class); @SuppressWarnings("unchecked") - DB db = (DB) constructor.newInstance((Object) parentName, (Object) name); + DB db = (DB) constructor + .newInstance((Object) parentName, (Object) name); this.db = db; } else if (clz == TxCacheDB.class) { @SuppressWarnings("unchecked") @@ -98,7 +93,7 @@ public Snapshot getRoot() { } @Override - public Iterator> iterator() { + public Iterator> iterator() { return db.iterator(); } diff --git a/src/main/resources/config.conf b/src/main/resources/config.conf index 454212c00f4..d569d16e0ae 100644 --- a/src/main/resources/config.conf +++ b/src/main/resources/config.conf @@ -54,7 +54,7 @@ storage { //we'd strongly recommend that do not modify it unless you know every item's meaning clearly. dbSettings = { levelNumber = 7 - compactThreads = 32 + //compactThreads = 32 blocksize = 64 // n * KB maxBytesForLevelBase = 256 // n * MB maxBytesForLevelMultiplier = 10 @@ -71,7 +71,7 @@ storage { properties = "prop.properties" // record which bak directory is valid bak1path = "bak1/database" // you must set two backup directories to prevent application halt unexpected(e.g. kill -9). bak2path = "bak2/database" - frequency = 10000 // indicate backup db once every 10000 blocks processed. + frequency = 2000 // indicate backup db once every 10000 blocks processed. } } From fbdf0803a5919cf043110d90e38d0cdf6326ea41 Mon Sep 17 00:00:00 2001 From: sun haoyu Date: Wed, 23 Jan 2019 16:12:50 +0800 Subject: [PATCH 07/13] db2 rocksdb --- src/main/java/org/tron/core/Constant.java | 1 - .../org/tron/core/config/DefaultConfig.java | 5 - .../java/org/tron/core/config/args/Args.java | 15 +- .../org/tron/core/config/args/Storage.java | 11 + .../org/tron/core/db/RevokingStoreRocks.java | 478 ------------------ .../tron/core/db/TronStoreWithRevoking.java | 12 +- .../org/tron/core/db/backup/BackupDbUtil.java | 29 +- .../core/db/backup/NeedBeanCondition.java | 2 +- .../RevokingRocksDBWithCachingOldValue.java | 153 ------ src/main/resources/config-dbbackup.conf | 427 ---------------- src/main/resources/config.conf | 7 +- .../tron/core/db/backup/BackupDbUtilTest.java | 14 +- .../org/tron/core/net/node/BaseNetTest.java | 2 - .../org/tron/core/net/node/BroadTest.java | 3 - .../net/node/GetBlockChainSummaryTest.java | 2 - .../core/net/node/GetLostBlockIdsTest.java | 2 - .../org/tron/core/net/node/NodeImplTest.java | 2 - .../net/node/StartFetchSyncBlockTest.java | 2 - src/test/resources/config-test-dbbackup.conf | 3 +- 19 files changed, 49 insertions(+), 1121 deletions(-) delete mode 100644 src/main/java/org/tron/core/db/RevokingStoreRocks.java delete mode 100644 src/main/java/org/tron/core/db2/core/RevokingRocksDBWithCachingOldValue.java delete mode 100644 src/main/resources/config-dbbackup.conf diff --git a/src/main/java/org/tron/core/Constant.java b/src/main/java/org/tron/core/Constant.java index c1247c0f52e..bfc4fffff2a 100644 --- a/src/main/java/org/tron/core/Constant.java +++ b/src/main/java/org/tron/core/Constant.java @@ -32,7 +32,6 @@ public class Constant { //config for junit test public static final String TEST_CONF = "config-test.conf"; - public static final String TESTBACKUP_CONF = "config-dbbackup.conf"; public static final String DATABASE_DIR = "storage.directory"; diff --git a/src/main/java/org/tron/core/config/DefaultConfig.java b/src/main/java/org/tron/core/config/DefaultConfig.java index 90c84559acf..3aef308e145 100755 --- a/src/main/java/org/tron/core/config/DefaultConfig.java +++ b/src/main/java/org/tron/core/config/DefaultConfig.java @@ -12,7 +12,6 @@ import org.tron.core.config.args.Args; import org.tron.core.db.RevokingDatabase; import org.tron.core.db.RevokingStore; -import org.tron.core.db.RevokingStoreRocks; import org.tron.core.db.TransactionCache; import org.tron.core.db.api.IndexHelper; import org.tron.core.db.backup.BackupRocksDBAspect; @@ -58,10 +57,6 @@ public RevokingDatabase revokingDatabase() { revokingDatabase = RevokingStore.getInstance(); } else if (dbVersion == 2) { revokingDatabase = new SnapshotManager(); - } else if (dbVersion == 3) { - revokingDatabase = RevokingStoreRocks.getInstance(); - } else if (dbVersion == 4) { - revokingDatabase = new SnapshotManager(); } else { throw new RuntimeException("db version is error."); } diff --git a/src/main/java/org/tron/core/config/args/Args.java b/src/main/java/org/tron/core/config/args/Args.java index d126b415814..b4e43d75c73 100644 --- a/src/main/java/org/tron/core/config/args/Args.java +++ b/src/main/java/org/tron/core/config/args/Args.java @@ -123,6 +123,10 @@ public class Args { @Parameter(names = {"--storage-db-version"}, description = "Storage db version.(1 or 2)") private String storageDbVersion = ""; + @Parameter(names = { + "--storage-db-engine"}, description = "Storage db engine.(leveldb or rocksdb)") + private String storageDbEngine = ""; + @Parameter(names = { "--storage-db-synchronous"}, description = "Storage db is synchronous or not.(true or flase)") private String storageDbSynchronous = ""; @@ -612,6 +616,10 @@ public static void setParam(final String[] args, final String confFileName) { .map(Integer::valueOf) .orElse(Storage.getDbVersionFromConfig(config))); + INSTANCE.storage.setDbEngine(Optional.ofNullable(INSTANCE.storageDbEngine) + .filter(StringUtils::isNotEmpty) + .orElse(Storage.getDbEngineFromConfig(config))); + INSTANCE.storage.setDbSync(Optional.ofNullable(INSTANCE.storageDbSynchronous) .filter(StringUtils::isNotEmpty) .map(Boolean::valueOf) @@ -1140,8 +1148,7 @@ private static double calcMaxTimeRatio() { private static void initRocksDbSettings(Config config) { String prefix = "storage.dbSettings."; - if (Args.getInstance().getStorage().getDbVersion() == 3 - || Args.getInstance().getStorage().getDbVersion() == 4) { + if (Args.getInstance().getStorage().getDbEngine().equals("ROCKSDB")) { int levelNumber = config.hasPath(prefix + "levelNumber") ? config.getInt(prefix + "levelNumber") : 7; int compactThreads = config.hasPath(prefix + "compactThreads") @@ -1173,8 +1180,7 @@ private static void initRocksDbSettings(Config config) { private static void initRocksDbBackupProperty(Config config) { boolean enable = false; - if (Args.getInstance().getStorage().getDbVersion() == 3 - || Args.getInstance().getStorage().getDbVersion() == 4) { + if (Args.getInstance().getStorage().getDbEngine().toUpperCase().equals("ROCKSDB")) { enable = config.hasPath("storage.backup.enable") && config.getBoolean("storage.backup.enable"); } @@ -1223,6 +1229,7 @@ private static void logConfig() { logger.info("Code version : {}", Version.getVersion()); logger.info("************************ DB config *************************"); logger.info("DB version : {}", args.getStorage().getDbVersion()); + logger.info("DB engine : {}", args.getStorage().getDbEngine()); logger.info("***************************************************************"); logger.info("\n"); } diff --git a/src/main/java/org/tron/core/config/args/Storage.java b/src/main/java/org/tron/core/config/args/Storage.java index bbe87615ea5..2ab40500592 100644 --- a/src/main/java/org/tron/core/config/args/Storage.java +++ b/src/main/java/org/tron/core/config/args/Storage.java @@ -42,6 +42,7 @@ public class Storage { */ private static final String DB_DIRECTORY_CONFIG_KEY = "storage.db.directory"; private static final String DB_VERSION_CONFIG_KEY = "storage.db.version"; + private static final String DB_ENGINE_CONFIG_KEY = "storage.db.engine"; private static final String DB_SYNC_CONFIG_KEY = "storage.db.sync"; private static final String INDEX_DIRECTORY_CONFIG_KEY = "storage.index.directory"; private static final String INDEX_SWITCH_CONFIG_KEY = "storage.index.switch"; @@ -64,6 +65,7 @@ public class Storage { * Default values of directory */ private static final int DEFAULT_DB_VERSION = 2; + private static final String DEFAULT_DB_ENGINE = "LEVELDB"; private static final boolean DEFAULT_DB_SYNC = false; private static final String DEFAULT_DB_DIRECTORY = "database"; private static final String DEFAULT_INDEX_DIRECTORY = "index"; @@ -95,6 +97,10 @@ public class Storage { @Setter private int dbVersion; + @Getter + @Setter + private String dbEngine; + @Getter @Setter private boolean dbSync; @@ -135,6 +141,11 @@ public static int getDbVersionFromConfig(final Config config) { config.getInt(DB_VERSION_CONFIG_KEY) : DEFAULT_DB_VERSION; } + public static String getDbEngineFromConfig(final Config config) { + return config.hasPath(DB_ENGINE_CONFIG_KEY) ? + config.getString(DB_ENGINE_CONFIG_KEY) : DEFAULT_DB_ENGINE; + } + public static Boolean getDbVersionSyncFromConfig(final Config config) { return config.hasPath(DB_SYNC_CONFIG_KEY) ? config.getBoolean(DB_SYNC_CONFIG_KEY) : DEFAULT_DB_SYNC; diff --git a/src/main/java/org/tron/core/db/RevokingStoreRocks.java b/src/main/java/org/tron/core/db/RevokingStoreRocks.java deleted file mode 100644 index 4c2fb47ec6e..00000000000 --- a/src/main/java/org/tron/core/db/RevokingStoreRocks.java +++ /dev/null @@ -1,478 +0,0 @@ -package org.tron.core.db; - -import static org.tron.core.db2.core.SnapshotManager.simpleDecode; - -import com.google.common.collect.Maps; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Deque; -import java.util.HashMap; -import java.util.HashSet; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Set; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.stream.Collectors; -import lombok.AllArgsConstructor; -import lombok.EqualsAndHashCode; -import lombok.Getter; -import lombok.ToString; -import lombok.extern.slf4j.Slf4j; -import org.rocksdb.WriteOptions; -import org.tron.common.storage.SourceInter; -import org.tron.common.storage.WriteOptionsWrapper; -import org.tron.common.storage.leveldb.RocksDbDataSourceImpl; -import org.tron.common.utils.FileUtil; -import org.tron.common.utils.Utils; -import org.tron.core.config.args.Args; -import org.tron.core.db2.common.IRevokingDB; -import org.tron.core.db2.core.ISession; -import org.tron.core.db2.core.RevokingRocksDBWithCachingOldValue; -import org.tron.core.exception.RevokingStoreIllegalStateException; - -@Slf4j -public class RevokingStoreRocks implements RevokingDatabase { - - private static final int DEFAULT_STACK_MAX_SIZE = 256; - private Deque stack = new LinkedList<>(); - private boolean disabled = true; - private int activeDialog = 0; - private AtomicInteger maxSize = new AtomicInteger(DEFAULT_STACK_MAX_SIZE); - private WriteOptionsWrapper optionsWrapper = WriteOptionsWrapper.getInstance().sync(true); - - @Getter - private List dbs = new ArrayList<>(); - - private static volatile RevokingStoreRocks instance; - - private RevokingStoreRocks() { - } - - public static void releaseInstance() { - instance = null; - } - - public static RevokingStoreRocks getInstance() { - if (instance == null) { - synchronized (RevokingStoreRocks.class) { - if (instance == null) { - instance = new RevokingStoreRocks(); - } - } - } - return instance; - } - - @Override - public ISession buildSession() { - return buildSession(false); - } - - @Override - public ISession buildSession(boolean forceEnable) { - if (disabled && !forceEnable) { - return new Dialog(this); - } - - boolean disableOnExit = disabled && forceEnable; - if (forceEnable) { - disabled = false; - } - - while (stack.size() > maxSize.get()) { - stack.poll(); - } - - stack.add(new RevokingState()); - ++activeDialog; - return new Dialog(this, disableOnExit); - } - - @Override - public void setMode(boolean mode) { - - } - - - @Override - public synchronized void check() { - RocksDbDataSourceImpl check = - new RocksDbDataSourceImpl(Args.getInstance().getOutputDirectoryByDbName("tmp"), "tmp"); - check.initDB(); - - if (!check.allKeys().isEmpty()) { - Map dbMap = dbs.stream() - .map(db -> Maps.immutableEntry(db.getDBName(), db)) - .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); - - for (Map.Entry e : check) { - byte[] key = e.getKey(); - byte[] value = e.getValue(); - String db = simpleDecode(key); - byte[] realKey = Arrays.copyOfRange(key, db.getBytes().length + 4, key.length); - - byte[] realValue = value.length == 1 ? null : Arrays.copyOfRange(value, 1, value.length); - if (realValue != null) { - dbMap.get(db).putData(realKey, realValue, WriteOptionsWrapper.getInstance().sync(true)); - } else { - dbMap.get(db).deleteData(realKey, WriteOptionsWrapper.getInstance().sync(true)); - } - } - } - - check.closeDB(); - FileUtil.recursiveDelete(check.getDbPath().toString()); - } - - public synchronized void onCreate(RevokingTuple tuple, byte[] value) { - if (disabled) { - return; - } - - addIfEmpty(); - RevokingState state = stack.peekLast(); - state.newIds.add(tuple); - } - - public synchronized void onModify(RevokingTuple tuple, byte[] value) { - if (disabled) { - return; - } - - addIfEmpty(); - RevokingState state = stack.peekLast(); - if (state.newIds.contains(tuple) || state.oldValues.containsKey(tuple)) { - return; - } - - state.oldValues.put(tuple, Utils.clone(value)); - } - - public synchronized void onRemove(RevokingTuple tuple, byte[] value) { - if (disabled) { - return; - } - - addIfEmpty(); - RevokingState state = stack.peekLast(); - if (state.newIds.contains(tuple)) { - state.newIds.remove(tuple); - return; - } - - if (state.oldValues.containsKey(tuple)) { - state.removed.put(tuple, state.oldValues.get(tuple)); - state.oldValues.remove(tuple); - return; - } - - if (state.removed.containsKey(tuple)) { - return; - } - - state.removed.put(tuple, Utils.clone(value)); - } - - @Override - public void add(IRevokingDB revokingDB) { - dbs.add(((RevokingRocksDBWithCachingOldValue) revokingDB).getDbSource()); - - } - - @Override - public void merge() throws RevokingStoreIllegalStateException { - if (activeDialog <= 0) { - throw new RevokingStoreIllegalStateException("activeDialog has to be greater than 0"); - } - - if (activeDialog == 1 && stack.size() == 1) { - stack.pollLast(); - --activeDialog; - return; - } - - if (stack.size() < 2) { - return; - } - - RevokingState state = stack.peekLast(); - @SuppressWarnings("unchecked") - List list = (List) stack; - RevokingState prevState = list.get(stack.size() - 2); - - state.oldValues.entrySet().stream() - .filter(e -> !prevState.newIds.contains(e.getKey())) - .filter(e -> !prevState.oldValues.containsKey(e.getKey())) - .forEach(e -> prevState.oldValues.put(e.getKey(), e.getValue())); - - prevState.newIds.addAll(state.newIds); - - state.removed.entrySet().stream() - .filter(e -> { - boolean has = prevState.newIds.contains(e.getKey()); - if (has) { - prevState.newIds.remove(e.getKey()); - } - - return !has; - }) - .filter(e -> { - boolean has = prevState.oldValues.containsKey(e.getKey()); - if (has) { - prevState.removed.put(e.getKey(), prevState.oldValues.get(e.getKey())); - prevState.oldValues.remove(e.getKey()); - } - - return !has; - }) - .forEach(e -> prevState.removed.put(e.getKey(), e.getValue())); - - stack.pollLast(); - --activeDialog; - } - - @Override - public void revoke() throws RevokingStoreIllegalStateException { - if (disabled) { - return; - } - - if (activeDialog <= 0) { - throw new RevokingStoreIllegalStateException("activeDialog has to be greater than 0"); - } - - disabled = true; - - try { - RevokingState state = stack.peekLast(); - if (Objects.isNull(state)) { - return; - } - - state.oldValues.forEach((k, v) -> k.database.putData(k.key, v)); - state.newIds.forEach(e -> e.database.deleteData(e.key)); - state.removed.forEach((k, v) -> k.database.putData(k.key, v)); - stack.pollLast(); - } finally { - disabled = false; - } - --activeDialog; - } - - @Override - public void commit() throws RevokingStoreIllegalStateException { - if (activeDialog <= 0) { - throw new RevokingStoreIllegalStateException("activeDialog has to be greater than 0"); - } - - --activeDialog; - } - - @Override - public void pop() throws RevokingStoreIllegalStateException { - prune(optionsWrapper); - } - - @Override - public void fastPop() throws RevokingStoreIllegalStateException { - prune(WriteOptionsWrapper.getInstance()); - } - - private synchronized void prune(WriteOptionsWrapper optionsWrapper) { - if (activeDialog != 0) { - throw new RevokingStoreIllegalStateException("activeDialog has to be equal 0"); - } - - if (stack.isEmpty()) { - throw new RevokingStoreIllegalStateException("stack is empty"); - } - - disabled = true; - - try { - RevokingState state = stack.peekLast(); - state.oldValues.forEach((k, v) -> k.database.putData(k.key, v, optionsWrapper)); - state.newIds.forEach(e -> e.database.deleteData(e.key, optionsWrapper)); - state.removed.forEach((k, v) -> k.database.putData(k.key, v, optionsWrapper)); - stack.pollLast(); - } finally { - disabled = false; - } - } - - @Override - public synchronized void enable() { - disabled = false; - } - - @Override - public synchronized void disable() { - disabled = true; - } - - @Override - public void setMaxFlushCount(int maxFlushCount) { - - } - - private void addIfEmpty() { - if (stack.isEmpty()) { - stack.add(new RevokingStoreRocks.RevokingState()); - } - } - - @Override - public synchronized int size() { - return stack.size(); - } - - @Override - public void setMaxSize(int maxSize) { - this.maxSize.set(maxSize); - } - - public int getMaxSize() { - return maxSize.get(); - } - - public synchronized void shutdown() { - System.err.println("******** begin to pop revokingDb ********"); - System.err.println("******** before revokingDb size:" + size()); - try { - disable(); - boolean exit = false; - while (!exit) { - try { - commit(); - } catch (RevokingStoreIllegalStateException e) { - exit = true; - } - } - - while (true) { - try { - pop(); - } catch (RevokingStoreIllegalStateException e) { - break; - } - } - } catch (Exception e) { - System.err.println("******** failed to pop revokingStore. " + e); - } finally { - System.err.println("******** after revokingStore size:" + stack.size()); - System.err.println("******** after revokingStore contains:" + stack); - System.err.println("******** end to pop revokingStore ********"); - } - } - - @Slf4j - @Getter // only for unit test - public static class Dialog implements ISession { - - private RevokingDatabase revokingDatabase; - private boolean applyRevoking = true; - private boolean disableOnExit = false; - - public Dialog(RevokingStoreRocks.Dialog dialog) { - this.revokingDatabase = dialog.revokingDatabase; - this.applyRevoking = dialog.applyRevoking; - dialog.applyRevoking = false; - } - - public Dialog(RevokingDatabase revokingDatabase) { - this(revokingDatabase, false); - } - - public Dialog(RevokingDatabase revokingDatabase, boolean disableOnExit) { - this.revokingDatabase = revokingDatabase; - this.disableOnExit = disableOnExit; - } - - @Override - public void commit() { - applyRevoking = false; - revokingDatabase.commit(); - } - - @Override - public void revoke() { - if (applyRevoking) { - revokingDatabase.revoke(); - } - - applyRevoking = false; - } - - @Override - public void merge() { - if (applyRevoking) { - revokingDatabase.merge(); - } - - applyRevoking = false; - } - - void copy(RevokingStoreRocks.Dialog dialog) { - if (this.equals(dialog)) { - return; - } - - if (applyRevoking) { - revokingDatabase.revoke(); - } - applyRevoking = dialog.applyRevoking; - dialog.applyRevoking = false; - } - - @Override - public void destroy() { - try { - if (applyRevoking) { - revokingDatabase.revoke(); - } - } catch (Exception e) { - logger.error("revoke database error.", e); - } - if (disableOnExit) { - revokingDatabase.disable(); - } - } - - @Override - public void close() { - try { - if (applyRevoking) { - revokingDatabase.revoke(); - } - } catch (Exception e) { - logger.error("revoke database error.", e); - throw new RevokingStoreIllegalStateException(e); - } - if (disableOnExit) { - revokingDatabase.disable(); - } - } - } - - @ToString - @Getter // only for unit test - static class RevokingState { - - Map oldValues = new HashMap<>(); - Set newIds = new HashSet<>(); - Map removed = new HashMap<>(); - } - - @AllArgsConstructor - @EqualsAndHashCode - @Getter - @ToString - public static class RevokingTuple { - - private SourceInter database; - private byte[] key; - } - -} \ No newline at end of file diff --git a/src/main/java/org/tron/core/db/TronStoreWithRevoking.java b/src/main/java/org/tron/core/db/TronStoreWithRevoking.java index 3dbed934d15..090841812b5 100755 --- a/src/main/java/org/tron/core/db/TronStoreWithRevoking.java +++ b/src/main/java/org/tron/core/db/TronStoreWithRevoking.java @@ -23,7 +23,6 @@ import org.tron.core.db2.core.ITronChainBase; import org.tron.core.db2.core.RevokingDBWithCachingNewValue; import org.tron.core.db2.core.RevokingDBWithCachingOldValue; -import org.tron.core.db2.core.RevokingRocksDBWithCachingOldValue; import org.tron.core.exception.BadItemException; import org.tron.core.exception.ItemNotFoundException; @@ -45,14 +44,15 @@ public abstract class TronStoreWithRevoking implements I protected TronStoreWithRevoking(String dbName) { this.dbName = dbName; int dbVersion = Args.getInstance().getStorage().getDbVersion(); + String dbEngine = Args.getInstance().getStorage().getDbEngine(); if (dbVersion == 1) { this.revokingDB = new RevokingDBWithCachingOldValue(dbName); } else if (dbVersion == 2) { - this.revokingDB = new RevokingDBWithCachingNewValue(dbName, LevelDB.class); - } else if (dbVersion == 3) { - this.revokingDB = new RevokingRocksDBWithCachingOldValue(dbName); - } else if (dbVersion == 4) { - this.revokingDB = new RevokingDBWithCachingNewValue(dbName, RocksDB.class); + if (dbEngine.toUpperCase().equals("LEVELDB")) { + this.revokingDB = new RevokingDBWithCachingNewValue(dbName, LevelDB.class); + } else if (dbEngine.toUpperCase().equals("ROCKSDB")) { + this.revokingDB = new RevokingDBWithCachingNewValue(dbName, RocksDB.class); + } } else { throw new RuntimeException("db version is error."); } diff --git a/src/main/java/org/tron/core/db/backup/BackupDbUtil.java b/src/main/java/org/tron/core/db/backup/BackupDbUtil.java index bed35e5d8d4..b74cb587834 100644 --- a/src/main/java/org/tron/core/db/backup/BackupDbUtil.java +++ b/src/main/java/org/tron/core/db/backup/BackupDbUtil.java @@ -6,12 +6,10 @@ import org.rocksdb.RocksDBException; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; -import org.tron.common.storage.leveldb.RocksDbDataSourceImpl; import org.tron.common.utils.PropUtil; import org.tron.core.capsule.BlockCapsule; import org.tron.core.config.args.Args; import org.tron.core.db.RevokingDatabase; -import org.tron.core.db.RevokingStoreRocks; import org.tron.core.db2.core.RevokingDBWithCachingNewValue; import org.tron.core.db2.core.SnapshotManager; import org.tron.core.db2.core.SnapshotRoot; @@ -138,14 +136,10 @@ public void doBackup(BlockCapsule block) { } private void backup(int i) throws RocksDBException { - if (Args.getInstance().getStorage().getDbVersion() == 3) { - List stores = ((RevokingStoreRocks) db).getDbs(); - for (RocksDbDataSourceImpl store : stores) { - store.backup(i); - } - } else if (Args.getInstance().getStorage().getDbVersion() == 4) { - List stores = ((SnapshotManager) db).getDbs(); - for (RevokingDBWithCachingNewValue store : stores) { + List stores = ((SnapshotManager) db).getDbs(); + for (RevokingDBWithCachingNewValue store : stores) { + if (((SnapshotRoot) (store.getHead().getRoot())).getDb().getClass() + == org.tron.core.db2.common.RocksDB.class) { ((org.tron.core.db2.common.RocksDB) ((SnapshotRoot) (store.getHead().getRoot())).getDb()) .getDb().backup(i); } @@ -153,15 +147,12 @@ private void backup(int i) throws RocksDBException { } private void deleteBackup(int i) { - if (Args.getInstance().getStorage().getDbVersion() == 3) { - List stores = ((RevokingStoreRocks) db).getDbs(); - for (RocksDbDataSourceImpl store : stores) { - store.deleteDbBakPath(i); - } - } else if (Args.getInstance().getStorage().getDbVersion() == 4) { - List stores = ((SnapshotManager) db).getDbs(); - for (RevokingDBWithCachingNewValue store : stores) { - ((org.tron.core.db2.common.RocksDB) ((SnapshotRoot) (store.getHead().getRoot())).getDb()) + List stores = ((SnapshotManager) db).getDbs(); + for (RevokingDBWithCachingNewValue store : stores) { + if (((SnapshotRoot) (store.getHead().getRoot())).getDb().getClass() + == org.tron.core.db2.common.RocksDB.class) { + ((org.tron.core.db2.common.RocksDB) (((SnapshotRoot) (store.getHead().getRoot())) + .getDb())) .getDb().deleteDbBakPath(i); } } diff --git a/src/main/java/org/tron/core/db/backup/NeedBeanCondition.java b/src/main/java/org/tron/core/db/backup/NeedBeanCondition.java index 5fad3dede66..dbaf04b4b61 100644 --- a/src/main/java/org/tron/core/db/backup/NeedBeanCondition.java +++ b/src/main/java/org/tron/core/db/backup/NeedBeanCondition.java @@ -9,7 +9,7 @@ public class NeedBeanCondition implements Condition { @Override public boolean matches(ConditionContext context, AnnotatedTypeMetadata metadata) { - return (Args.getInstance().getStorage().getDbVersion() == 3 || Args.getInstance().getStorage().getDbVersion() == 4) && Args.getInstance() + return (Args.getInstance().getStorage().getDbVersion() == 2 && Args.getInstance().getStorage().getDbEngine().toUpperCase().equals("ROCKSDB")) && Args.getInstance() .getDbBackupConfig().isEnable(); } } \ No newline at end of file diff --git a/src/main/java/org/tron/core/db2/core/RevokingRocksDBWithCachingOldValue.java b/src/main/java/org/tron/core/db2/core/RevokingRocksDBWithCachingOldValue.java deleted file mode 100644 index 25c8684b32f..00000000000 --- a/src/main/java/org/tron/core/db2/core/RevokingRocksDBWithCachingOldValue.java +++ /dev/null @@ -1,153 +0,0 @@ -package org.tron.core.db2.core; - -import java.util.Iterator; -import java.util.Map; -import java.util.Map.Entry; -import java.util.Objects; -import java.util.Set; -import java.util.Spliterator; -import java.util.function.Consumer; -import lombok.Getter; -import lombok.NoArgsConstructor; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.ArrayUtils; -import org.tron.common.storage.leveldb.RocksDbDataSourceImpl; -import org.tron.core.config.args.Args; -import org.tron.core.db.AbstractRevokingStore; -import org.tron.core.db.RevokingStore; -import org.tron.core.db.RevokingStoreRocks; -import org.tron.core.db.RevokingStoreRocks.RevokingTuple; -import org.tron.core.db2.common.IRevokingDB; -import org.tron.core.exception.ItemNotFoundException; - -@Slf4j -@NoArgsConstructor -public class RevokingRocksDBWithCachingOldValue implements IRevokingDB { - - private RevokingStoreRocks revokingDatabase; - - @Getter - private RocksDbDataSourceImpl dbSource; - - public RevokingRocksDBWithCachingOldValue(String dbName) { - this(dbName, RevokingStoreRocks.getInstance()); - } - - // only for unit test - public RevokingRocksDBWithCachingOldValue(String dbName, RevokingStoreRocks revokingDatabase) { - dbSource = new RocksDbDataSourceImpl(Args.getInstance().getOutputDirectoryByDbName(dbName), - dbName); - dbSource.initDB(); - this.revokingDatabase = revokingDatabase; - } - - @Override - public void put(byte[] key, byte[] value) { - if (Objects.isNull(key) || Objects.isNull(value)) { - return; - } - byte[] oldValue = dbSource.getData(key); - if (ArrayUtils.isNotEmpty(oldValue)) { - onModify(key, oldValue); - } - - dbSource.putData(key, value); - - if (ArrayUtils.isEmpty(oldValue)) { - onCreate(key); - } - } - - @Override - public void delete(byte[] key) { - onDelete(key); - dbSource.deleteData(key); - } - - @Override - public boolean has(byte[] key) { - return dbSource.getData(key) != null; - } - - @Override - public byte[] get(byte[] key) throws ItemNotFoundException { - byte[] value = dbSource.getData(key); - if (ArrayUtils.isEmpty(value)) { - throw new ItemNotFoundException(); - } - return value; - } - - @Override - public byte[] getUnchecked(byte[] key) { - try { - return get(key); - } catch (ItemNotFoundException e) { - return null; - } - } - - @Override - public void close() { - dbSource.closeDB(); - } - - @Override - public void reset() { - dbSource.resetDb(); - } - - @Override - public void setMode(boolean mode) { - - } - - /** - * This should be called just after an object is created - */ - private void onCreate(byte[] key) { - revokingDatabase.onCreate(new RevokingTuple(dbSource, key), null); - } - - /** - * This should be called just before an object is modified - */ - private void onModify(byte[] key, byte[] value) { - revokingDatabase.onModify(new RevokingTuple(dbSource, key), value); - } - - /** - * This should be called just before an object is removed. - */ - private void onDelete(byte[] key) { - byte[] value; - if (Objects.nonNull(value = dbSource.getData(key))) { - revokingDatabase.onRemove(new RevokingTuple(dbSource, key), value); - } - } - - @Override - public Iterator> iterator() { - return dbSource.iterator(); - } - - @Override - public void forEach(Consumer> action) { - - } - - @Override - public Spliterator> spliterator() { - return null; - } - - @Override - public Set getlatestValues(long limit) { - return dbSource.getlatestValues(limit); - } - - @Override - public Set getValuesNext(byte[] key, long limit) { - return dbSource.getValuesNext(key, limit); - } -} diff --git a/src/main/resources/config-dbbackup.conf b/src/main/resources/config-dbbackup.conf deleted file mode 100644 index cb7cb6ab46d..00000000000 --- a/src/main/resources/config-dbbackup.conf +++ /dev/null @@ -1,427 +0,0 @@ -net { - type = mainnet - # type = testnet -} - -storage { - # Directory for storing persistent data - db.version = 3, - db.directory = "database", - index.directory = "index", - - # You can custom these 14 databases' configs: - - # account, account-index, asset-issue, block, block-index, - # block_KDB, peers, properties, recent-block, trans, - # utxo, votes, witness, witness_schedule. - - # Otherwise, db configs will remain defualt and data will be stored in - # the path of "output-directory" or which is set by "-d" ("--output-directory"). - - # Attention: name is a required field that must be set !!! - properties = [ -// { -// name = "account", -// path = "storage_directory_test", -// createIfMissing = true, -// paranoidChecks = true, -// verifyChecksums = true, - // compressionType = 1, // compressed with snappy - // blockSize = 4096, // 4 KB = 4 * 1024 B - // writeBufferSize = 10485760, // 10 MB = 10 * 1024 * 1024 B - // cacheSize = 10485760, // 10 MB = 10 * 1024 * 1024 B - // maxOpenFiles = 100 - // }, -// { -// name = "account-index", -// path = "storage_directory_test", -// createIfMissing = true, -// paranoidChecks = true, -// verifyChecksums = true, - // compressionType = 1, // compressed with snappy - // blockSize = 4096, // 4 KB = 4 * 1024 B - // writeBufferSize = 10485760, // 10 MB = 10 * 1024 * 1024 B - // cacheSize = 10485760, // 10 MB = 10 * 1024 * 1024 B - // maxOpenFiles = 100 - // }, - ] - - needToUpdateAsset = true - - //dbsettings is needed when using rocksdb as the storage implement (db.version=3). - //we'd strongly recommend that do not modify it unless you know every item's meaning clearly. - dbSettings = { - levelNumber = 7 - compactThreads = 32 - blocksize = 64 // n * KB - maxBytesForLevelBase = 256 // n * MB - maxBytesForLevelMultiplier = 10 - level0FileNumCompactionTrigger = 4 - compressionTypeListStr = "no:no:lz4:lz4:lz4:zstd:zstd" - targetFileSizeBase = 256 // n * MB - targetFileSizeMultiplier = 1 - } - - //backup settings when using rocks db as the storage implement (db.version=3). - //if you want to use the backup plugin, please confirm set the db.version=3 above. - backup = { - enable = true // indicate whether enable the backup plugin - properties = "prop.properties" // record which bak directory is valid - bak1path = "bak1/database" // you must set two backup directories to prevent application halt unexpected(e.g. kill -9). - bak2path = "bak2/database" - frequency = 10000 // indicate backup db once every 10000 blocks processed. - } -} - -node.discovery = { - enable = true - persist = true - bind.ip = "" - external.ip = null -} - -node.backup { - port = 10001 - - # my priority, each member should use different priority - priority = 8 - - # peer's ip list, can't contain mine - members = [ - # "ip", - # "ip" - ] -} - -node { - # trust node for solidity node - # trustNode = "ip:port" - trustNode = "127.0.0.1:50051" - - # expose extension api to public or not - walletExtensionApi = true - - listen.port = 18888 - - connection.timeout = 2 - - tcpNettyWorkThreadNum = 0 - - udpNettyWorkThreadNum = 1 - - # Number of validate sign thread, default availableProcessors / 2 - # validateSignThreadNum = 16 - - connectFactor = 0.3 - activeConnectFactor = 0.1 - - maxActiveNodes = 30 - - maxActiveNodesWithSameIp = 2 - - minParticipationRate = 15 - - # check the peer data transfer ,disconnect factor - disconnectNumberFactor = 0.4 - maxConnectNumberFactor = 0.8 - receiveTcpMinDataLength = 2048 - isOpenFullTcpDisconnect = true - - p2p { - version = 11111 # 11111: mainnet; 20180622: testnet - } - - active = [ - # Active establish connection in any case - # Sample entries: - # "ip:port", - # "ip:port" - ] - - passive = [ - # Passive accept connection in any case - # Sample entries: - # "ip:port", - # "ip:port" - ] - - http { - fullNodePort = 8090 - solidityPort = 8091 - } - - rpc { - port = 50051 - #solidityPort = 50061 - # Number of gRPC thread, default availableProcessors / 2 - # thread = 16 - - # The maximum number of concurrent calls permitted for each incoming connection - # maxConcurrentCallsPerConnection = - - # The HTTP/2 flow control window, default 1MB - # flowControlWindow = - - # Connection being idle for longer than which will be gracefully terminated - maxConnectionIdleInMillis = 60000 - - # Connection lasting longer than which will be gracefully terminated - # maxConnectionAgeInMillis = - - # The maximum message size allowed to be received on the server, default 4MB - # maxMessageSize = - - # The maximum size of header list allowed to be received, default 8192 - # maxHeaderListSize = - } - - # Limits the maximum percentage (default 75%) of producing block interval - # to provide sufficient time to perform other operations e.g. broadcast block - # blockProducedTimeOut = 75 - - # Limits the maximum number (default 700) of transaction from network layer - # netMaxTrxPerSecond = 700 -} - - - -seed.node = { - # List of the seed nodes - # Seed nodes are stable full nodes - # example: - # ip.list = [ - # "ip:port", - # "ip:port" - # ] - ip.list = [ - "54.236.37.243:18888", - "52.53.189.99:18888", - "18.196.99.16:18888", - "34.253.187.192:18888", - "52.56.56.149:18888", - "35.180.51.163:18888", - "54.252.224.209:18888", - "18.228.15.36:18888", - "52.15.93.92:18888", - "34.220.77.106:18888", - "13.127.47.162:18888", - "13.124.62.58:18888", - "13.229.128.108:18888", - "35.182.37.246:18888", - "34.200.228.125:18888", - "18.220.232.201:18888", - "13.57.30.186:18888", - "35.165.103.105:18888", - "18.184.238.21:18888", - "34.250.140.143:18888", - "35.176.192.130:18888", - "52.47.197.188:18888", - "52.62.210.100:18888", - "13.231.4.243:18888", - "18.231.76.29:18888", - "35.154.90.144:18888", - "13.125.210.234:18888", - "13.250.40.82:18888", - "35.183.101.48:18888" - ] -} - -genesis.block = { - # Reserve balance - assets = [ - { - accountName = "Zion" - accountType = "AssetIssue" - address = "TLLM21wteSPs4hKjbxgmH1L6poyMjeTbHm" - balance = "99000000000000000" - }, - { - accountName = "Sun" - accountType = "AssetIssue" - address = "TXmVpin5vq5gdZsciyyjdZgKRUju4st1wM" - balance = "0" - }, - { - accountName = "Blackhole" - accountType = "AssetIssue" - address = "TLsV52sRDL79HXGGm9yzwKibb6BeruhUzy" - balance = "-9223372036854775808" - } - ] - - witnesses = [ - { - address: THKJYuUmMKKARNf7s2VT51g5uPY6KEqnat, - url = "http://GR1.com", - voteCount = 100000026 - }, - { - address: TVDmPWGYxgi5DNeW8hXrzrhY8Y6zgxPNg4, - url = "http://GR2.com", - voteCount = 100000025 - }, - { - address: TWKZN1JJPFydd5rMgMCV5aZTSiwmoksSZv, - url = "http://GR3.com", - voteCount = 100000024 - }, - { - address: TDarXEG2rAD57oa7JTK785Yb2Et32UzY32, - url = "http://GR4.com", - voteCount = 100000023 - }, - { - address: TAmFfS4Tmm8yKeoqZN8x51ASwdQBdnVizt, - url = "http://GR5.com", - voteCount = 100000022 - }, - { - address: TK6V5Pw2UWQWpySnZyCDZaAvu1y48oRgXN, - url = "http://GR6.com", - voteCount = 100000021 - }, - { - address: TGqFJPFiEqdZx52ZR4QcKHz4Zr3QXA24VL, - url = "http://GR7.com", - voteCount = 100000020 - }, - { - address: TC1ZCj9Ne3j5v3TLx5ZCDLD55MU9g3XqQW, - url = "http://GR8.com", - voteCount = 100000019 - }, - { - address: TWm3id3mrQ42guf7c4oVpYExyTYnEGy3JL, - url = "http://GR9.com", - voteCount = 100000018 - }, - { - address: TCvwc3FV3ssq2rD82rMmjhT4PVXYTsFcKV, - url = "http://GR10.com", - voteCount = 100000017 - }, - { - address: TFuC2Qge4GxA2U9abKxk1pw3YZvGM5XRir, - url = "http://GR11.com", - voteCount = 100000016 - }, - { - address: TNGoca1VHC6Y5Jd2B1VFpFEhizVk92Rz85, - url = "http://GR12.com", - voteCount = 100000015 - }, - { - address: TLCjmH6SqGK8twZ9XrBDWpBbfyvEXihhNS, - url = "http://GR13.com", - voteCount = 100000014 - }, - { - address: TEEzguTtCihbRPfjf1CvW8Euxz1kKuvtR9, - url = "http://GR14.com", - voteCount = 100000013 - }, - { - address: TZHvwiw9cehbMxrtTbmAexm9oPo4eFFvLS, - url = "http://GR15.com", - voteCount = 100000012 - }, - { - address: TGK6iAKgBmHeQyp5hn3imB71EDnFPkXiPR, - url = "http://GR16.com", - voteCount = 100000011 - }, - { - address: TLaqfGrxZ3dykAFps7M2B4gETTX1yixPgN, - url = "http://GR17.com", - voteCount = 100000010 - }, - { - address: TX3ZceVew6yLC5hWTXnjrUFtiFfUDGKGty, - url = "http://GR18.com", - voteCount = 100000009 - }, - { - address: TYednHaV9zXpnPchSywVpnseQxY9Pxw4do, - url = "http://GR19.com", - voteCount = 100000008 - }, - { - address: TCf5cqLffPccEY7hcsabiFnMfdipfyryvr, - url = "http://GR20.com", - voteCount = 100000007 - }, - { - address: TAa14iLEKPAetX49mzaxZmH6saRxcX7dT5, - url = "http://GR21.com", - voteCount = 100000006 - }, - { - address: TBYsHxDmFaRmfCF3jZNmgeJE8sDnTNKHbz, - url = "http://GR22.com", - voteCount = 100000005 - }, - { - address: TEVAq8dmSQyTYK7uP1ZnZpa6MBVR83GsV6, - url = "http://GR23.com", - voteCount = 100000004 - }, - { - address: TRKJzrZxN34YyB8aBqqPDt7g4fv6sieemz, - url = "http://GR24.com", - voteCount = 100000003 - }, - { - address: TRMP6SKeFUt5NtMLzJv8kdpYuHRnEGjGfe, - url = "http://GR25.com", - voteCount = 100000002 - }, - { - address: TDbNE1VajxjpgM5p7FyGNDASt3UVoFbiD3, - url = "http://GR26.com", - voteCount = 100000001 - }, - { - address: TLTDZBcPoJ8tZ6TTEeEqEvwYFk2wgotSfD, - url = "http://GR27.com", - voteCount = 100000000 - } - ] - - timestamp = "0" #2017-8-26 12:00:00 - - parentHash = "0xe58f33f9baf9305dc6f82b9f1934ea8f0ade2defb951258d50167028c780351f" -} - -localwitness = [ -] - -#localwitnesskeystore = [ -# "localwitnesskeystore.json" -#] - -block = { - needSyncCheck = true - maintenanceTimeInterval = 21600000 - proposalExpireTime = 259200000 // 3 day: 259200000(ms) -} - -# Transaction reference block, default is "head", configure to "solid" can avoid TaPos error -# trx.reference.block = "head" // head;solid; - -vm = { - supportConstant = false - minTimeRatio = 0.0 - maxTimeRatio = 5.0 - - # In rare cases, transactions that will be within the specified maximum execution time (default 10(ms)) are re-executed and packaged - # longRunningTime = 10 -} - -committee = { - allowCreationOfContracts = 0 //mainnet:0 (reset by committee),test:1 - allowAdaptiveEnergy = 0 //mainnet:0 (reset by committee),test:1 -} - -log.level = { - root = "INFO" // TRACE;DEBUG;INFO;WARN;ERROR -} diff --git a/src/main/resources/config.conf b/src/main/resources/config.conf index d569d16e0ae..4b339d0ecee 100644 --- a/src/main/resources/config.conf +++ b/src/main/resources/config.conf @@ -6,6 +6,7 @@ net { storage { # Directory for storing persistent data db.version = 2, + db.engine = "ROCKSDB", db.sync = false, db.directory = "database", index.directory = "index", @@ -50,7 +51,7 @@ storage { needToUpdateAsset = true - //dbsettings is needed when using rocksdb as the storage implement (db.version=3). + //dbsettings is needed when using rocksdb as the storage implement (db.version=2 and db.engine="ROCKSDB"). //we'd strongly recommend that do not modify it unless you know every item's meaning clearly. dbSettings = { levelNumber = 7 @@ -64,8 +65,8 @@ storage { targetFileSizeMultiplier = 1 } - //backup settings when using rocks db as the storage implement (db.version=3). - //if you want to use the backup plugin, please confirm set the db.version=3 above. + //backup settings when using rocks db as the storage implement (db.version=2 and db.engine="ROCKSDB"). + //if you want to use the backup plugin, please confirm set the db.version=2 and db.engine="ROCKSDB" above. backup = { enable = true // indicate whether enable the backup plugin properties = "prop.properties" // record which bak directory is valid diff --git a/src/test/java/org/tron/core/db/backup/BackupDbUtilTest.java b/src/test/java/org/tron/core/db/backup/BackupDbUtilTest.java index 97408bc9bb0..e3c00ccd4d8 100644 --- a/src/test/java/org/tron/core/db/backup/BackupDbUtilTest.java +++ b/src/test/java/org/tron/core/db/backup/BackupDbUtilTest.java @@ -9,7 +9,6 @@ import org.junit.Before; import org.junit.Test; import org.rocksdb.RocksDB; -import org.springframework.beans.factory.annotation.Autowired; import org.tron.common.application.TronApplicationContext; import org.tron.common.storage.leveldb.RocksDbDataSourceImpl; import org.tron.common.utils.FileUtil; @@ -18,7 +17,8 @@ import org.tron.core.config.args.Args; import org.tron.core.db.Manager; import org.tron.core.db.ManagerForTest; -import org.tron.core.db.RevokingStoreRocks; +import org.tron.core.db2.core.RevokingDBWithCachingNewValue; +import org.tron.core.db2.core.SnapshotManager; @Slf4j public class BackupDbUtilTest { @@ -68,7 +68,7 @@ public void before() { @After public void after() { FileUtil.deleteDir(new File(dbPath)); - RevokingStoreRocks.releaseInstance(); + //SnapshotManager.releaseInstance(); } @Test @@ -76,13 +76,7 @@ public void testDoBackup() { PropUtil.writeProperty(prop_path, BackupDbUtil.getDB_BACKUP_STATE(), String.valueOf("11")); mng_test.pushNTestBlock(50); - List alist = ((RevokingStoreRocks)dbBackupUtil.getDb()).getDbs(); - logger.info("alist.size():" + alist.size()); - for(RocksDbDataSourceImpl rocks :alist) { - logger.info("sss:" + rocks.getDatabase().toString()); - logger.info("aaa:" + rocks.getDBName()); - logger.info("bbb:" + rocks.getDbPath()); - } + List alist = ((SnapshotManager)dbBackupUtil.getDb()).getDbs(); Assert.assertTrue(dbManager.getDynamicPropertiesStore().getLatestBlockHeaderNumber() == 50); Assert.assertTrue("22".equals( diff --git a/src/test/java/org/tron/core/net/node/BaseNetTest.java b/src/test/java/org/tron/core/net/node/BaseNetTest.java index 00ea0041618..4bf680d22ff 100644 --- a/src/test/java/org/tron/core/net/node/BaseNetTest.java +++ b/src/test/java/org/tron/core/net/node/BaseNetTest.java @@ -32,7 +32,6 @@ import org.tron.core.config.DefaultConfig; import org.tron.core.config.args.Args; import org.tron.core.db.Manager; -import org.tron.core.db.RevokingStoreRocks; import org.tron.core.net.peer.PeerConnection; import org.tron.core.services.RpcApiService; import org.tron.core.services.WitnessService; @@ -163,7 +162,6 @@ public void destroy() { for (PeerConnection peer : peerConnections) { peer.close(); } - RevokingStoreRocks.releaseInstance(); context.destroy(); node.shutDown(); appT.shutdownServices(); diff --git a/src/test/java/org/tron/core/net/node/BroadTest.java b/src/test/java/org/tron/core/net/node/BroadTest.java index 12c53de1240..fb7910c9a92 100644 --- a/src/test/java/org/tron/core/net/node/BroadTest.java +++ b/src/test/java/org/tron/core/net/node/BroadTest.java @@ -12,7 +12,6 @@ import org.junit.After; import org.junit.Assert; import org.junit.Before; -import org.junit.Ignore; import org.junit.Test; import org.tron.common.application.Application; import org.tron.common.application.ApplicationFactory; @@ -31,7 +30,6 @@ import org.tron.core.config.args.Args; import org.tron.core.db.ByteArrayWrapper; import org.tron.core.db.Manager; -import org.tron.core.db.RevokingStoreRocks; import org.tron.core.net.message.BlockMessage; import org.tron.core.net.message.MessageTypes; import org.tron.core.net.message.TransactionMessage; @@ -296,7 +294,6 @@ public void destroy() { for (PeerConnection peer : peerConnections) { peer.close(); } - RevokingStoreRocks.releaseInstance(); context.destroy(); handshakeHandlerTest.close(); appT.shutdownServices(); diff --git a/src/test/java/org/tron/core/net/node/GetBlockChainSummaryTest.java b/src/test/java/org/tron/core/net/node/GetBlockChainSummaryTest.java index dea1da0e7fa..74e53bed873 100644 --- a/src/test/java/org/tron/core/net/node/GetBlockChainSummaryTest.java +++ b/src/test/java/org/tron/core/net/node/GetBlockChainSummaryTest.java @@ -38,7 +38,6 @@ import org.tron.core.db.BlockStore; import org.tron.core.db.ByteArrayWrapper; import org.tron.core.db.Manager; -import org.tron.core.db.RevokingStoreRocks; import org.tron.core.net.node.override.HandshakeHandlerTest; import org.tron.core.net.node.override.PeerClientTest; import org.tron.core.net.node.override.TronChannelInitializerTest; @@ -345,7 +344,6 @@ public static void destroy() { handshakeHandlerTest.close(); appT.shutdownServices(); appT.shutdown(); - RevokingStoreRocks.releaseInstance(); context.destroy(); FileUtil.deleteDir(new File(dbPath)); } diff --git a/src/test/java/org/tron/core/net/node/GetLostBlockIdsTest.java b/src/test/java/org/tron/core/net/node/GetLostBlockIdsTest.java index 1b09c04cd34..94dd6bdb186 100644 --- a/src/test/java/org/tron/core/net/node/GetLostBlockIdsTest.java +++ b/src/test/java/org/tron/core/net/node/GetLostBlockIdsTest.java @@ -39,7 +39,6 @@ import org.tron.core.config.args.Args; import org.tron.core.db.ByteArrayWrapper; import org.tron.core.db.Manager; -import org.tron.core.db.RevokingStoreRocks; import org.tron.core.exception.StoreException; import org.tron.core.net.node.override.HandshakeHandlerTest; import org.tron.core.net.node.override.PeerClientTest; @@ -325,7 +324,6 @@ public static void destroy() { peer.close(); } handshakeHandlerTest.close(); - RevokingStoreRocks.releaseInstance(); appT.shutdownServices(); appT.shutdown(); context.destroy(); diff --git a/src/test/java/org/tron/core/net/node/NodeImplTest.java b/src/test/java/org/tron/core/net/node/NodeImplTest.java index 037ce1e2ebf..227888f3d68 100644 --- a/src/test/java/org/tron/core/net/node/NodeImplTest.java +++ b/src/test/java/org/tron/core/net/node/NodeImplTest.java @@ -27,7 +27,6 @@ import org.tron.core.config.Parameter.NetConstants; import org.tron.core.config.args.Args; import org.tron.core.db.Manager; -import org.tron.core.db.RevokingStoreRocks; import org.tron.core.net.message.BlockMessage; import org.tron.core.net.peer.PeerConnection; import org.tron.protos.Protocol.Block; @@ -197,7 +196,6 @@ public static void init() { @AfterClass public static void destroy() { Args.clearParam(); - RevokingStoreRocks.releaseInstance(); context.destroy(); appT.shutdownServices(); appT.shutdown(); diff --git a/src/test/java/org/tron/core/net/node/StartFetchSyncBlockTest.java b/src/test/java/org/tron/core/net/node/StartFetchSyncBlockTest.java index 4ebaa9a9d79..672b0fde837 100644 --- a/src/test/java/org/tron/core/net/node/StartFetchSyncBlockTest.java +++ b/src/test/java/org/tron/core/net/node/StartFetchSyncBlockTest.java @@ -27,7 +27,6 @@ import org.tron.core.config.args.Args; import org.tron.core.db.ByteArrayWrapper; import org.tron.core.db.Manager; -import org.tron.core.db.RevokingStoreRocks; import org.tron.core.net.message.BlockMessage; import org.tron.core.net.node.override.HandshakeHandlerTest; import org.tron.core.net.node.override.PeerClientTest; @@ -250,7 +249,6 @@ public void run() { public static void destroy() { Args.clearParam(); handshakeHandlerTest.close(); - RevokingStoreRocks.releaseInstance(); context.destroy(); appT.shutdownServices(); appT.shutdown(); diff --git a/src/test/resources/config-test-dbbackup.conf b/src/test/resources/config-test-dbbackup.conf index d7cf2e1faaf..71e32a1c8b7 100644 --- a/src/test/resources/config-test-dbbackup.conf +++ b/src/test/resources/config-test-dbbackup.conf @@ -5,7 +5,8 @@ net { storage { # Directory for storing persistent data - db.version = 3, + db.version = 2, + db.engine = "ROCKSDB", db.directory = "database", index.directory = "index", From d5c67fdfce4cbb22e95fd643bffd8d7d18bb3450 Mon Sep 17 00:00:00 2001 From: sun haoyu Date: Wed, 30 Jan 2019 11:31:35 +0800 Subject: [PATCH 08/13] add engine prop for every db --- .../leveldb/LevelDbDataSourceImpl.java | 35 +++++++++++++++++-- .../leveldb/RocksDbDataSourceImpl.java | 33 +++++++++++++++++ .../java/org/tron/common/utils/PropUtil.java | 8 +++-- src/main/resources/config.conf | 2 +- .../leveldb/RocksDbDataSourceImplTest.java | 35 +++++++++++++++++-- 5 files changed, 105 insertions(+), 8 deletions(-) diff --git a/src/main/java/org/tron/common/storage/leveldb/LevelDbDataSourceImpl.java b/src/main/java/org/tron/common/storage/leveldb/LevelDbDataSourceImpl.java index b375a817004..5d68081b841 100644 --- a/src/main/java/org/tron/common/storage/leveldb/LevelDbDataSourceImpl.java +++ b/src/main/java/org/tron/common/storage/leveldb/LevelDbDataSourceImpl.java @@ -45,6 +45,7 @@ import org.tron.common.storage.DbSourceInter; import org.tron.common.storage.WriteOptionsWrapper; import org.tron.common.utils.FileUtil; +import org.tron.common.utils.PropUtil; import org.tron.core.config.args.Args; import org.tron.core.db.common.iterator.StoreIterator; @@ -65,13 +66,43 @@ public class LevelDbDataSourceImpl implements DbSourceInter, public LevelDbDataSourceImpl(String parentName, String name) { this.dataBaseName = name; this.parentName = Paths.get( - parentName, - Args.getInstance().getStorage().getDbDirectory() + parentName, + Args.getInstance().getStorage().getDbDirectory() ).toString(); } + public boolean checkOrInitEngine() { + String dir = Args.getInstance().getOutputDirectory() + Args.getInstance().getStorage().getDbDirectory() + File.separator + dataBaseName; + String enginePath = dir + File.separator + "engine.properties"; + + if (FileUtil.createDirIfNotExists(dir)) { + if (!FileUtil.createFileIfNotExists(enginePath)) { + return false; + } + } else { + return false; + } + + String engine = PropUtil.readProperty(enginePath, "ENGINE"); + if (engine.equals("")) { + if (!PropUtil.writeProperty(enginePath, "ENGINE", "LEVELDB")) { + return false; + } + } + engine = PropUtil.readProperty(enginePath, "ENGINE"); + if (engine.equals("LEVELDB")) { + return true; + } else { + return false; + } + } + @Override public void initDB() { + if (!checkOrInitEngine()) { + logger.error("database engine do not match"); + throw new RuntimeException("Failed to initialize database"); + } resetDbLock.writeLock().lock(); try { logger.debug("~> LevelDbDataSourceImpl.initDB(): " + dataBaseName); diff --git a/src/main/java/org/tron/common/storage/leveldb/RocksDbDataSourceImpl.java b/src/main/java/org/tron/common/storage/leveldb/RocksDbDataSourceImpl.java index 1c5ba55bc81..903a5624cc4 100644 --- a/src/main/java/org/tron/common/storage/leveldb/RocksDbDataSourceImpl.java +++ b/src/main/java/org/tron/common/storage/leveldb/RocksDbDataSourceImpl.java @@ -30,6 +30,7 @@ import org.tron.common.storage.DbSourceInter; import org.tron.common.storage.WriteOptionsWrapper; import org.tron.common.utils.FileUtil; +import org.tron.common.utils.PropUtil; import org.tron.core.config.args.Args; import org.tron.core.db.common.iterator.RockStoreIterator; @@ -131,7 +132,39 @@ public String getDBName() { public void setDBName(String name) { } + public boolean checkOrInitEngine() { + String dir = + Args.getInstance().getOutputDirectory() + Args.getInstance().getStorage().getDbDirectory() + File.separator + dataBaseName; + String enginePath = dir + File.separator + "engine.properties"; + + if (FileUtil.createDirIfNotExists(dir)) { + if (!FileUtil.createFileIfNotExists(enginePath)) { + return false; + } + } else { + return false; + } + + // for the first init engine + String engine = PropUtil.readProperty(enginePath, "ENGINE"); + if (engine.equals("")) { + if (!PropUtil.writeProperty(enginePath, "ENGINE", "ROCKSDB")) { + return false; + } + } + engine = PropUtil.readProperty(enginePath, "ENGINE"); + if (engine.equals("ROCKSDB")) { + return true; + } else { + return false; + } + } + public void initDB() { + if (!checkOrInitEngine()) { + logger.error("database engine do not match"); + throw new RuntimeException("Failed to initialize database"); + } initDB(DBSettings.getSettings()); } diff --git a/src/main/java/org/tron/common/utils/PropUtil.java b/src/main/java/org/tron/common/utils/PropUtil.java index 30bc6ec89be..d57cd8ff4cd 100644 --- a/src/main/java/org/tron/common/utils/PropUtil.java +++ b/src/main/java/org/tron/common/utils/PropUtil.java @@ -30,7 +30,7 @@ public static String readProperty(String file, String key) { String value = new String(prop.getProperty(key, "").getBytes("ISO-8859-1"), "UTF-8"); return value; } catch (Exception e) { - logger.warn("{}", e); + logger.error("{}", e); return ""; } finally { if (prop != null) { @@ -52,12 +52,12 @@ public static String readProperty(String file, String key) { is = null; } } catch (Exception e) { - logger.warn("{}", e); + logger.error("{}", e); } } } - public static void writeProperty(String file, String key, String value) { + public static boolean writeProperty(String file, String key, String value) { FileInputStream fis = null; Properties properties = new Properties(); try { @@ -71,7 +71,9 @@ public static void writeProperty(String file, String key, String value) { out.close(); } catch (Exception e) { logger.warn("{}", e); + return false; } + return true; } } \ No newline at end of file diff --git a/src/main/resources/config.conf b/src/main/resources/config.conf index 4b339d0ecee..3c893d71a45 100644 --- a/src/main/resources/config.conf +++ b/src/main/resources/config.conf @@ -6,7 +6,7 @@ net { storage { # Directory for storing persistent data db.version = 2, - db.engine = "ROCKSDB", + db.engine = "LEVELDB", db.sync = false, db.directory = "database", index.directory = "index", diff --git a/src/test/java/org/tron/common/storage/leveldb/RocksDbDataSourceImplTest.java b/src/test/java/org/tron/common/storage/leveldb/RocksDbDataSourceImplTest.java index ae36773adf3..1d20833278e 100644 --- a/src/test/java/org/tron/common/storage/leveldb/RocksDbDataSourceImplTest.java +++ b/src/test/java/org/tron/common/storage/leveldb/RocksDbDataSourceImplTest.java @@ -18,7 +18,7 @@ import org.junit.Test; import org.tron.common.utils.ByteArray; import org.tron.common.utils.FileUtil; -import org.tron.core.Constant; +import org.tron.common.utils.PropUtil; import org.tron.core.config.args.Args; @Slf4j @@ -42,7 +42,7 @@ public class RocksDbDataSourceImplTest { @Before public void initDb() { - Args.setParam(new String[]{"--output-directory", dbPath}, Constant.TEST_CONF); + Args.setParam(new String[]{"--output-directory", dbPath}, "config-test-dbbackup.conf"); dataSourceTest = new RocksDbDataSourceImpl(dbPath + File.separator, "test_rocksDb"); } @@ -250,4 +250,35 @@ public void getValuesPrev() { dataSource.resetDb(); dataSource.closeDB(); } + + @Test + public void testCheckOrInitEngine() { + String dir = Args.getInstance().getOutputDirectory() + Args.getInstance().getStorage().getDbDirectory(); + String enginePath = dir + File.separator + "test_engine" + File.separator + "engine.properties"; + FileUtil.createDirIfNotExists(dir + File.separator + "test_engine"); + FileUtil.createFileIfNotExists(enginePath); + boolean b = PropUtil.writeProperty(enginePath, "ENGINE", "ROCKSDB"); + Assert.assertEquals(PropUtil.readProperty(enginePath, "ENGINE"), + "ROCKSDB"); + + RocksDbDataSourceImpl dataSource; + dataSource = new RocksDbDataSourceImpl(dir, "test_engine"); + dataSource.initDB(); + Assert.assertNotNull(dataSource.getDatabase()); + dataSource.closeDB(); + + dataSource = null; + System.gc(); + b = PropUtil.writeProperty(enginePath, "ENGINE", "LEVELDB"); + Assert.assertEquals(PropUtil.readProperty(enginePath, "ENGINE"), + "LEVELDB"); + dataSource = new RocksDbDataSourceImpl(dir, "test_engine"); + try { + dataSource.initDB(); + } catch (Exception e) { + Assert.assertTrue(e.getMessage().contains("Failed to")); + } + Assert.assertNull(dataSource.getDatabase()); + PropUtil.writeProperty(enginePath, "ENGINE", "ROCKSDB"); + } } From dfdfe1790da0375f4f7e4ceb38b6f710628d600f Mon Sep 17 00:00:00 2001 From: sun haoyu Date: Wed, 30 Jan 2019 18:30:53 +0800 Subject: [PATCH 09/13] add db convert --- build.gradle | 3 +- .../org/tron/common/storage/DBSettings.java | 11 ++ src/main/java/org/tron/program/DBConvert.java | 161 ++++++++++++++++++ 3 files changed, 174 insertions(+), 1 deletion(-) create mode 100644 src/main/java/org/tron/program/DBConvert.java diff --git a/build.gradle b/build.gradle index 9705cf60fd6..c38c4fb9262 100755 --- a/build.gradle +++ b/build.gradle @@ -339,5 +339,6 @@ def binaryRelease(taskName, jarName, mainClass) { artifacts { archives(binaryRelease('buildSolidityNodeJar', 'SolidityNode', 'org.tron.program.SolidityNode'), binaryRelease('buildFullNodeJar', 'FullNode', 'org.tron.program.FullNode'), - binaryRelease('buildKeystoreFactoryJar', 'KeystoreFactory', 'org.tron.program.KeystoreFactory')) + binaryRelease('buildKeystoreFactoryJar', 'KeystoreFactory', 'org.tron.program.KeystoreFactory'), + binaryRelease('buildDBConvertJar', 'DBConvert', 'org.tron.program.DBConvert')) } \ No newline at end of file diff --git a/src/main/java/org/tron/common/storage/DBSettings.java b/src/main/java/org/tron/common/storage/DBSettings.java index 11fe1a4283a..92fe01fcae0 100644 --- a/src/main/java/org/tron/common/storage/DBSettings.java +++ b/src/main/java/org/tron/common/storage/DBSettings.java @@ -40,7 +40,18 @@ private DBSettings() { } + public static DBSettings getDefaultSettings() { + DBSettings defaultSettings = new DBSettings(); + return defaultSettings.withLevelNumber(7).withBlockSize(64).withCompactThreads(32) + .withCompressionTypeList("no:no:no:lz4:lz4:zstd:zstd").withTargetFileSizeBase(256) + .withMaxBytesForLevelMultiplier(10).withTargetFileSizeMultiplier(1).withMaxBytesForLevelBase(256).withMaxOpenFiles(-1) + .withEnableStatistics(false); + } + public static DBSettings getSettings() { + if (settings == null) { + return getDefaultSettings(); + } return settings; } diff --git a/src/main/java/org/tron/program/DBConvert.java b/src/main/java/org/tron/program/DBConvert.java new file mode 100644 index 00000000000..9dc108eaf65 --- /dev/null +++ b/src/main/java/org/tron/program/DBConvert.java @@ -0,0 +1,161 @@ +package org.tron.program; + +import static org.fusesource.leveldbjni.JniDBFactory.factory; + +import java.io.File; +import java.io.IOException; +import lombok.extern.slf4j.Slf4j; +import org.iq80.leveldb.CompressionType; +import org.iq80.leveldb.DB; +import org.iq80.leveldb.DBIterator; +import org.rocksdb.BlockBasedTableConfig; +import org.rocksdb.BloomFilter; +import org.rocksdb.Options; +import org.rocksdb.RocksDB; +import org.rocksdb.RocksDBException; +import org.tron.common.utils.FileUtil; + +@Slf4j +public class DBConvert { + + static { + RocksDB.loadLibrary(); + } + + private static org.iq80.leveldb.Options newDefaultLevelDbOptions() { + org.iq80.leveldb.Options dbOptions = new org.iq80.leveldb.Options(); + dbOptions.createIfMissing(true); + dbOptions.paranoidChecks(true); + dbOptions.verifyChecksums(true); + dbOptions.compressionType(CompressionType.SNAPPY); + dbOptions.blockSize(4 * 1024); + dbOptions.writeBufferSize(10 * 1024 * 1024); + dbOptions.cacheSize(10 * 1024 * 1024L); + dbOptions.maxOpenFiles(100); + return dbOptions; + } + + public DB newLevelDB(String dir, String dbname) throws IOException { + DB database = null; + File file = new File(dir + dbname); + org.iq80.leveldb.Options dbOptions = newDefaultLevelDbOptions(); + try { + database = factory.open(file, dbOptions); + } catch (IOException e) { + if (e.getMessage().contains("Corruption:")) { + factory.repair(file, dbOptions); + database = factory.open(file, dbOptions); + } else { + throw e; + } + } + return database; + } + + private Options newDefaultRocksDbOptions() { + Options options = new Options(); + options.setCreateIfMissing(true); + options.setIncreaseParallelism(1); + options.setNumLevels(7); + options.setMaxOpenFiles(-1); + options.setTargetFileSizeBase(256 * 1024 * 1024); + options.setBaseBackgroundCompactions(Math.max(1, Runtime.getRuntime().availableProcessors())); + options.setLevel0FileNumCompactionTrigger(4); + final BlockBasedTableConfig tableCfg; + options.setTableFormatConfig(tableCfg = new BlockBasedTableConfig()); + tableCfg.setBlockSize(64 * 1024); + tableCfg.setBlockCacheSize(32 * 1024 * 1024); + tableCfg.setCacheIndexAndFilterBlocks(true); + tableCfg.setPinL0FilterAndIndexBlocksInCache(true); + tableCfg.setFilter(new BloomFilter(10, false)); + return options; + } + + public RocksDB newRocksDB(String dir, String dbname) { + RocksDB database = null; + try (Options options = newDefaultRocksDbOptions()) { + database = RocksDB.open(options, dir + dbname); + } catch (Exception ignore) { + logger.error(ignore.getMessage()); + } + return database; + } + + public boolean convertLeveltoRocks(DB level, RocksDB rocks) { + int count = 0; + + DBIterator iterator = level.iterator(); + try { + for (iterator.seekToFirst(); iterator.hasNext(); iterator.next()) { + byte[] key = iterator.peekNext().getKey(); + byte[] value = iterator.peekNext().getValue(); + count++; + rocks.put(key, value); + } + } catch (RocksDBException e) { + logger.error(e.getMessage()); + return false; + } finally { + try { + iterator.close(); + } catch (IOException e1) { + logger.error(e1.getMessage()); + } + } + logger.info("covert {} items from LevelDb to RocksDb", count); + return true; + } + + public boolean doConvert(String[] args) { + String levelDbDir = ""; + String rocksDbDir = ""; + + if (args.length < 2) { + levelDbDir = "output-directory"; + rocksDbDir = "output-directory-rocks"; + } else { + levelDbDir = args[0]; + rocksDbDir = args[1]; + } + + String srcDir = levelDbDir + File.separator + "database" + File.separator; + String dstDir = rocksDbDir + File.separator + "database" + File.separator; + + File levels = new File(srcDir); + if (!levels.exists()) { + System.out.println(srcDir + " not exists."); + return false; + } + + File[] aa = levels.listFiles(); + if (aa == null || aa.length == 0) { + return false; + } + for (File file : aa) { + DB level = null; + try { + level = newLevelDB(srcDir, file.getName()); + } catch (IOException e) { + e.printStackTrace(); + } + + FileUtil.createDirIfNotExists(dstDir + file.getName()); + + RocksDB rocks = null; + rocks = newRocksDB(dstDir, file.getName()); + + if (convertLeveltoRocks(level, rocks)) { + System.out.println("success"); + } else { + System.out.println("failure"); + } + } + + return true; + } + + public static void main(String[] args) { + DBConvert convert = new DBConvert(); + convert.doConvert(args); + } +} \ No newline at end of file From 21c35cd72eead7d6e6b8737c6aa6dac656bff83a Mon Sep 17 00:00:00 2001 From: sun haoyu Date: Thu, 31 Jan 2019 14:03:23 +0800 Subject: [PATCH 10/13] remove Args aggressive --- .../leveldb/RocksDbDataSourceImpl.java | 33 ++++--------------- .../org/tron/core/db/backup/BackupDbUtil.java | 20 +++++++++-- .../org/tron/core/db2/common/RocksDB.java | 3 +- src/main/resources/config.conf | 2 +- 4 files changed, 27 insertions(+), 31 deletions(-) diff --git a/src/main/java/org/tron/common/storage/leveldb/RocksDbDataSourceImpl.java b/src/main/java/org/tron/common/storage/leveldb/RocksDbDataSourceImpl.java index 903a5624cc4..7c64ef313ed 100644 --- a/src/main/java/org/tron/common/storage/leveldb/RocksDbDataSourceImpl.java +++ b/src/main/java/org/tron/common/storage/leveldb/RocksDbDataSourceImpl.java @@ -31,7 +31,6 @@ import org.tron.common.storage.WriteOptionsWrapper; import org.tron.common.utils.FileUtil; import org.tron.common.utils.PropUtil; -import org.tron.core.config.args.Args; import org.tron.core.db.common.iterator.RockStoreIterator; @Slf4j @@ -49,10 +48,7 @@ public class RocksDbDataSourceImpl implements DbSourceInter, public RocksDbDataSourceImpl(String parentName, String name) { this.dataBaseName = name; - this.parentName = Paths.get( - parentName, - Args.getInstance().getStorage().getDbDirectory() - ).toString(); + this.parentName = parentName; } public Path getDbPath() { @@ -133,8 +129,7 @@ public void setDBName(String name) { } public boolean checkOrInitEngine() { - String dir = - Args.getInstance().getOutputDirectory() + Args.getInstance().getStorage().getDbDirectory() + File.separator + dataBaseName; + String dir = getDbPath().toString(); String enginePath = dir + File.separator + "engine.properties"; if (FileUtil.createDirIfNotExists(dir)) { @@ -486,27 +481,11 @@ public Set getValuesNext(byte[] key, long limit) { } } - // rocksdb use - public void backup(int i) throws RocksDBException { - if (i == 1) { - Checkpoint.create(database) - .createCheckpoint(Args.getInstance().getDbBackupConfig().getBak1path() - + this.getDBName()); - } else { - Checkpoint.create(database) - .createCheckpoint(Args.getInstance().getDbBackupConfig().getBak2path() - + this.getDBName()); - } + public void backup(String dir) throws RocksDBException { + Checkpoint.create(database).createCheckpoint(dir + this.getDBName()); } - // rocksdb use - public boolean deleteDbBakPath(int i) { - if (i == 1) { - return FileUtil.deleteDir(new File(Args.getInstance().getDbBackupConfig().getBak1path() - + this.getDBName())); - } else { - return FileUtil.deleteDir((new File(Args.getInstance().getDbBackupConfig().getBak2path() - + this.getDBName()))); - } + public boolean deleteDbBakPath(String dir) { + return FileUtil.deleteDir(new File(dir + this.getDBName())); } } \ No newline at end of file diff --git a/src/main/java/org/tron/core/db/backup/BackupDbUtil.java b/src/main/java/org/tron/core/db/backup/BackupDbUtil.java index b74cb587834..8a5806a8d16 100644 --- a/src/main/java/org/tron/core/db/backup/BackupDbUtil.java +++ b/src/main/java/org/tron/core/db/backup/BackupDbUtil.java @@ -136,24 +136,40 @@ public void doBackup(BlockCapsule block) { } private void backup(int i) throws RocksDBException { + String path = ""; + if (i == DB_BACKUP_INDEX1) { + path = args.getDbBackupConfig().getBak1path(); + } else if (i == DB_BACKUP_INDEX2) { + path = args.getDbBackupConfig().getBak2path(); + } else { + throw new RuntimeException("Error backup with undefined index"); + } List stores = ((SnapshotManager) db).getDbs(); for (RevokingDBWithCachingNewValue store : stores) { if (((SnapshotRoot) (store.getHead().getRoot())).getDb().getClass() == org.tron.core.db2.common.RocksDB.class) { ((org.tron.core.db2.common.RocksDB) ((SnapshotRoot) (store.getHead().getRoot())).getDb()) - .getDb().backup(i); + .getDb().backup(path); } } } private void deleteBackup(int i) { + String path = ""; + if (i == DB_BACKUP_INDEX1) { + path = args.getDbBackupConfig().getBak1path(); + } else if (i == DB_BACKUP_INDEX2) { + path = args.getDbBackupConfig().getBak2path(); + } else { + throw new RuntimeException("Error deleteBackup with undefined index"); + } List stores = ((SnapshotManager) db).getDbs(); for (RevokingDBWithCachingNewValue store : stores) { if (((SnapshotRoot) (store.getHead().getRoot())).getDb().getClass() == org.tron.core.db2.common.RocksDB.class) { ((org.tron.core.db2.common.RocksDB) (((SnapshotRoot) (store.getHead().getRoot())) .getDb())) - .getDb().deleteDbBakPath(i); + .getDb().deleteDbBakPath(path); } } } diff --git a/src/main/java/org/tron/core/db2/common/RocksDB.java b/src/main/java/org/tron/core/db2/common/RocksDB.java index d0ea7c3bef7..432b49efd79 100644 --- a/src/main/java/org/tron/core/db2/common/RocksDB.java +++ b/src/main/java/org/tron/core/db2/common/RocksDB.java @@ -2,6 +2,7 @@ import com.google.common.collect.Maps; +import java.nio.file.Paths; import java.util.HashMap; import java.util.Map; import lombok.Getter; @@ -19,7 +20,7 @@ public class RocksDB implements DB, Flusher { .sync(Args.getInstance().getStorage().isDbSync()); public RocksDB(String parentName, String name) { - db = new RocksDbDataSourceImpl(parentName, name); + db = new RocksDbDataSourceImpl(Paths.get(parentName, Args.getInstance().getStorage().getDbDirectory()).toString(), name); db.initDB(); } diff --git a/src/main/resources/config.conf b/src/main/resources/config.conf index 3c893d71a45..4b339d0ecee 100644 --- a/src/main/resources/config.conf +++ b/src/main/resources/config.conf @@ -6,7 +6,7 @@ net { storage { # Directory for storing persistent data db.version = 2, - db.engine = "LEVELDB", + db.engine = "ROCKSDB", db.sync = false, db.directory = "database", index.directory = "index", From 4ac37850a1f14e0fa9fe371dd7b181a607b52f6d Mon Sep 17 00:00:00 2001 From: sun haoyu Date: Thu, 31 Jan 2019 15:40:04 +0800 Subject: [PATCH 11/13] optimise convert db with adding check --- src/main/java/org/tron/program/DBConvert.java | 176 ++++++++++++------ src/main/resources/config.conf | 2 +- 2 files changed, 124 insertions(+), 54 deletions(-) diff --git a/src/main/java/org/tron/program/DBConvert.java b/src/main/java/org/tron/program/DBConvert.java index 9dc108eaf65..680cf844b92 100644 --- a/src/main/java/org/tron/program/DBConvert.java +++ b/src/main/java/org/tron/program/DBConvert.java @@ -4,6 +4,9 @@ import java.io.File; import java.io.IOException; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; import lombok.extern.slf4j.Slf4j; import org.iq80.leveldb.CompressionType; import org.iq80.leveldb.DB; @@ -13,6 +16,7 @@ import org.rocksdb.Options; import org.rocksdb.RocksDB; import org.rocksdb.RocksDBException; +import org.rocksdb.RocksIterator; import org.tron.common.utils.FileUtil; @Slf4j @@ -22,6 +26,27 @@ public class DBConvert { RocksDB.loadLibrary(); } + private String srcDir; + private String dstDir; + private String dbName; + private Path srcDbPath; + private Path dstDbPath; + + private int srcDbKeyCount = 0; + private int dstDbKeyCount = 0; + private int srcDbKeySum = 0; + private int dstDbKeySum = 0; + private int srcDbValueSum = 0; + private int dstDbValueSum = 0; + + public DBConvert(String src, String dst, String name) { + this.srcDir = src; + this.dstDir = dst; + this.dbName = name; + this.srcDbPath = Paths.get(this.srcDir, name); + this.dstDbPath = Paths.get(this.dstDir, name); + } + private static org.iq80.leveldb.Options newDefaultLevelDbOptions() { org.iq80.leveldb.Options dbOptions = new org.iq80.leveldb.Options(); dbOptions.createIfMissing(true); @@ -35,9 +60,9 @@ private static org.iq80.leveldb.Options newDefaultLevelDbOptions() { return dbOptions; } - public DB newLevelDB(String dir, String dbname) throws IOException { + public DB newLevelDB(Path db) throws IOException { DB database = null; - File file = new File(dir + dbname); + File file = db.toFile(); org.iq80.leveldb.Options dbOptions = newDefaultLevelDbOptions(); try { database = factory.open(file, dbOptions); @@ -61,6 +86,17 @@ private Options newDefaultRocksDbOptions() { options.setTargetFileSizeBase(256 * 1024 * 1024); options.setBaseBackgroundCompactions(Math.max(1, Runtime.getRuntime().availableProcessors())); options.setLevel0FileNumCompactionTrigger(4); + options.setCompressionPerLevel(new ArrayList() { + { + add(org.rocksdb.CompressionType.NO_COMPRESSION); + add(org.rocksdb.CompressionType.NO_COMPRESSION); + add(org.rocksdb.CompressionType.NO_COMPRESSION); + add(org.rocksdb.CompressionType.LZ4_COMPRESSION); + add(org.rocksdb.CompressionType.LZ4_COMPRESSION); + add(org.rocksdb.CompressionType.ZSTD_COMPRESSION); + add(org.rocksdb.CompressionType.ZSTD_COMPRESSION); + } + }); final BlockBasedTableConfig tableCfg; options.setTableFormatConfig(tableCfg = new BlockBasedTableConfig()); tableCfg.setBlockSize(64 * 1024); @@ -71,25 +107,26 @@ private Options newDefaultRocksDbOptions() { return options; } - public RocksDB newRocksDB(String dir, String dbname) { + public RocksDB newRocksDB(Path db) { RocksDB database = null; try (Options options = newDefaultRocksDbOptions()) { - database = RocksDB.open(options, dir + dbname); + database = RocksDB.open(options, db.toString()); } catch (Exception ignore) { logger.error(ignore.getMessage()); } return database; } - public boolean convertLeveltoRocks(DB level, RocksDB rocks) { - int count = 0; - - DBIterator iterator = level.iterator(); + public boolean convertLevelToRocks(DB level, RocksDB rocks) { + // convert + DBIterator levelIterator = level.iterator(); try { - for (iterator.seekToFirst(); iterator.hasNext(); iterator.next()) { - byte[] key = iterator.peekNext().getKey(); - byte[] value = iterator.peekNext().getValue(); - count++; + for (levelIterator.seekToFirst(); levelIterator.hasNext(); levelIterator.next()) { + byte[] key = levelIterator.peekNext().getKey(); + byte[] value = levelIterator.peekNext().getValue(); + srcDbKeyCount++; + srcDbKeySum = byteArrayToIntWithOne(srcDbKeySum, key); + srcDbValueSum = byteArrayToIntWithOne(srcDbValueSum, value); rocks.put(key, value); } } catch (RocksDBException e) { @@ -97,65 +134,98 @@ public boolean convertLeveltoRocks(DB level, RocksDB rocks) { return false; } finally { try { - iterator.close(); + levelIterator.close(); } catch (IOException e1) { logger.error(e1.getMessage()); } } - logger.info("covert {} items from LevelDb to RocksDb", count); - return true; - } - - public boolean doConvert(String[] args) { - String levelDbDir = ""; - String rocksDbDir = ""; - if (args.length < 2) { - levelDbDir = "output-directory"; - rocksDbDir = "output-directory-rocks"; - } else { - levelDbDir = args[0]; - rocksDbDir = args[1]; + // check + try (final RocksIterator rocksIterator = rocks.newIterator()) { + for (rocksIterator.seekToLast(); rocksIterator.isValid(); rocksIterator.prev()) { + byte[] key = rocksIterator.key(); + byte[] value = rocksIterator.value(); + dstDbKeyCount++; + dstDbKeySum = byteArrayToIntWithOne(dstDbKeySum, key); + dstDbValueSum = byteArrayToIntWithOne(dstDbValueSum, value); + } } - String srcDir = levelDbDir + File.separator + "database" + File.separator; - String dstDir = rocksDbDir + File.separator + "database" + File.separator; + return dstDbKeyCount == srcDbKeyCount && dstDbKeySum == srcDbKeySum + && dstDbValueSum == srcDbValueSum; + } + + public boolean doConvert() { - File levels = new File(srcDir); - if (!levels.exists()) { - System.out.println(srcDir + " not exists."); + File levelDbFile = srcDbPath.toFile(); + if (!levelDbFile.exists()) { + System.out.println(srcDbPath.toString() + " not exists."); return false; } - File[] aa = levels.listFiles(); - if (aa == null || aa.length == 0) { - return false; + DB level = null; + try { + level = newLevelDB(srcDbPath); + } catch (IOException e) { + e.printStackTrace(); } - for (File file : aa) { - DB level = null; - try { - level = newLevelDB(srcDir, file.getName()); - } catch (IOException e) { - e.printStackTrace(); - } - FileUtil.createDirIfNotExists(dstDir + file.getName()); + FileUtil.createDirIfNotExists(dstDir); + RocksDB rocks = null; + rocks = newRocksDB(dstDbPath); - RocksDB rocks = null; - rocks = newRocksDB(dstDir, file.getName()); + return convertLevelToRocks(level, rocks); + } - if (convertLeveltoRocks(level, rocks)) { - System.out.println("success"); - } else { - System.out.println("failure"); - } + public int byteArrayToIntWithOne(int sum, byte[] b) { + for (byte aByte : b) { + sum += (int) aByte; } - - return true; + return sum; } public static void main(String[] args) { - DBConvert convert = new DBConvert(); - convert.doConvert(args); + String dbSrc; + String dbDst; + if (args.length < 2) { + dbSrc = "output-directory/database"; + dbDst = "output-directory-dst/database"; + } else { + dbSrc = args[0]; + dbDst = args[1]; + } + File dbDirectory = new File(dbSrc); + if (!dbDirectory.exists()) { + System.out.println(dbSrc + "is not exists."); + return; + } + File[] files = dbDirectory.listFiles(); + if (files == null || files.length == 0) { + System.out.println(dbSrc + " not contains any database."); + return; + } + long time = System.currentTimeMillis(); + for (File file : files) { + if (!file.isDirectory()) { + System.out.println(file.getName() + " is not a database directory, ignore it."); + continue; + } + try { + DBConvert convert = new DBConvert(dbSrc, dbDst, file.getName()); + if (convert.doConvert()) { + System.out.println(String + .format("Convert database %s successful with %s key-value. keySum: %d, valueSum: %d", + convert.dbName, + convert.srcDbKeyCount, convert.dstDbKeySum, convert.dstDbValueSum)); + } else { + System.out.println(String.format("Convert database %s failure", convert.dbName)); + } + } catch (Exception e) { + System.out.println(e.getMessage()); + return; + } + } + System.out.println(String + .format("database convert use %d seconds total.", (System.currentTimeMillis() - time) / 1000)); } } \ No newline at end of file diff --git a/src/main/resources/config.conf b/src/main/resources/config.conf index 4b339d0ecee..3c893d71a45 100644 --- a/src/main/resources/config.conf +++ b/src/main/resources/config.conf @@ -6,7 +6,7 @@ net { storage { # Directory for storing persistent data db.version = 2, - db.engine = "ROCKSDB", + db.engine = "LEVELDB", db.sync = false, db.directory = "database", index.directory = "index", From 01023d7430493a321c6f4b0cbbc1a3efe1cef00e Mon Sep 17 00:00:00 2001 From: sun haoyu Date: Thu, 31 Jan 2019 18:00:20 +0800 Subject: [PATCH 12/13] modify some params --- src/main/java/org/tron/program/DBConvert.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/tron/program/DBConvert.java b/src/main/java/org/tron/program/DBConvert.java index 680cf844b92..17fb0b1938c 100644 --- a/src/main/java/org/tron/program/DBConvert.java +++ b/src/main/java/org/tron/program/DBConvert.java @@ -83,8 +83,10 @@ private Options newDefaultRocksDbOptions() { options.setIncreaseParallelism(1); options.setNumLevels(7); options.setMaxOpenFiles(-1); - options.setTargetFileSizeBase(256 * 1024 * 1024); - options.setBaseBackgroundCompactions(Math.max(1, Runtime.getRuntime().availableProcessors())); + options.setTargetFileSizeBase(64 * 1024 * 1024); + options.setTargetFileSizeMultiplier(1); + options.setMaxBytesForLevelBase(512 * 1024 * 1024); + options.setMaxBackgroundCompactions(Math.max(1, Runtime.getRuntime().availableProcessors())); options.setLevel0FileNumCompactionTrigger(4); options.setCompressionPerLevel(new ArrayList() { { From df3a146fe571d596640258d62cb660d6828141ff Mon Sep 17 00:00:00 2001 From: sun haoyu Date: Thu, 14 Feb 2019 16:31:41 +0800 Subject: [PATCH 13/13] setLevelCompactionDynamicLevelBytes with true --- src/main/java/org/tron/program/DBConvert.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/java/org/tron/program/DBConvert.java b/src/main/java/org/tron/program/DBConvert.java index 17fb0b1938c..906bd2901e2 100644 --- a/src/main/java/org/tron/program/DBConvert.java +++ b/src/main/java/org/tron/program/DBConvert.java @@ -88,6 +88,7 @@ private Options newDefaultRocksDbOptions() { options.setMaxBytesForLevelBase(512 * 1024 * 1024); options.setMaxBackgroundCompactions(Math.max(1, Runtime.getRuntime().availableProcessors())); options.setLevel0FileNumCompactionTrigger(4); + options.setLevelCompactionDynamicLevelBytes(true); options.setCompressionPerLevel(new ArrayList() { { add(org.rocksdb.CompressionType.NO_COMPRESSION);