Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
90 commits
Select commit Hold shift + click to select a range
ee8f9ae
optimize blockLock; MessageQueue does not send message if channel is …
317787106 Jul 14, 2022
4b72d5e
Merge pull request #1 from 317787106/hotfix/blocklock
317787106 Jul 14, 2022
fe8efd1
clear message from MessageQueue before channel close
317787106 Jul 15, 2022
6ccb554
rollback
317787106 Jul 18, 2022
cfec5b1
rollback
317787106 Jul 18, 2022
7ecd325
Merge branch 'develop' of https://github.com/tronprotocol/java-tron i…
xxo1shine Jul 21, 2022
3f6b1db
fix error that use ZMQ.Socket to send data may fail in multithreads w…
317787106 Jul 22, 2022
b927f73
rollback
317787106 Jul 22, 2022
d83b65c
resolve sonarlint error of tronState
317787106 Jul 26, 2022
2eb91bf
perf(net): speed up TCP connection establishment
xxo1shine Jul 26, 2022
665cbb6
Merge branch 'develop' of https://github.com/tronprotocol/java-tron i…
xxo1shine Jul 26, 2022
aee957f
tool(db):add db path move function.
halibobo1205 Jul 22, 2022
72375d7
feat: catch error if param address is not base58
Jul 26, 2022
a509383
feat(block): increase the probability that the block processing threa…
jwrct Jul 26, 2022
9445506
feat: catch ClassCastException when getJSONObject
Jul 26, 2022
ba09cbd
feat(block): remove test log
jwrct Jul 26, 2022
3889e04
feat: change implementation of web3_clientVersion, remove VERSION_NAME
Jul 26, 2022
e8478b1
api(getblock): rename params
halibobo1205 Jul 26, 2022
7f385e4
Merge pull request #4552 from halibobo1205/feat/api_query_header
halibobo1205 Jul 27, 2022
fd836ba
feat: add api to get bandwidth price history
Jul 27, 2022
f219fee
feat(bandwidth_prices): load bandwidth prices when init
Jul 27, 2022
ad4bf2f
feat: set saveBandwidthPriceHistoryDone as 0 before test
Jul 27, 2022
d8a09e1
feat: remove check andwidthPriceHistoryDone
Jul 28, 2022
efc103a
Merge pull request #4556 from tronprotocol/feature/api_enhance
waynercheung Jul 28, 2022
e67f010
Merge pull request #4551 from chengtx01/feature/optimize_blocks_proce…
jwrct Jul 28, 2022
2e69d76
Merge pull request #4549 from wubin01/work
xxo1shine Jul 28, 2022
f327fe5
Merge pull request #4543 from 317787106/hotfix/zeromqError
317787106 Jul 28, 2022
c5acd14
Merge pull request #4536 from 317787106/hotfix/blocklock
317787106 Jul 28, 2022
f899621
Merge pull request #4545 from halibobo1205/feat/db_transfer_tool
halibobo1205 Jul 28, 2022
25ab125
perf(block): optimize block processing thread acquires the lock code
jwrct Jul 29, 2022
d56d5b5
feat: catch DecoderException when address is not base58 in GetRewardS…
Jul 29, 2022
294b632
feat: update error msg
Jul 29, 2022
5c2bc4f
Merge pull request #4560 from tronprotocol/feature/api_enhance
waynercheung Jul 29, 2022
0e74efc
tool(db):db path move function.
halibobo1205 Jul 29, 2022
a8359e8
Merge pull request #4557 from chengtx01/feature/optimize_blocks_proce…
jwrct Aug 1, 2022
1015549
Merge pull request #4559 from halibobo1205/feat/db_transfer_tool
halibobo1205 Aug 1, 2022
cac3e60
fix(net): solve the null pointer problem caused by the change of peer ID
xxo1shine Aug 1, 2022
46635cb
Merge branch 'release_v4.5.2' of https://github.com/tronprotocol/java…
xxo1shine Aug 1, 2022
8d23840
Merge remote-tracking branch 'upstream/release_v4.5.2' into release_v…
317787106 Aug 1, 2022
44a5ce2
merge pr 4536 4537
317787106 Aug 1, 2022
2866ea7
Merge pull request #4562 from 317787106/hotfix/merge_pr_4536_4537
317787106 Aug 2, 2022
e5f84fc
tool(db):db path move, print error and exit if path or config error.
halibobo1205 Aug 2, 2022
1c70db1
Merge pull request #4563 from halibobo1205/feat/db_transfer_tool
zhang0125 Aug 2, 2022
2b7db1b
refactor(tx-cache): use Bloom filters to refactor txCacheDB to reduce…
zhang0125 Jul 14, 2022
8f3410d
refactor(tx-cache): add log information and modify default parameters
zhang0125 Jul 21, 2022
e0be599
refactor(tx-cache): recover partial transaction db for light nodes
zhang0125 Jul 21, 2022
8198fb9
refactor(tx-cache): add volatile to increase multi-threaded safety
zhang0125 Aug 1, 2022
7d17172
refactor(tx-cache): limit param estimatedBlockTransactions
zhang0125 Aug 2, 2022
63bb018
Merge pull request #4538 from tronprotocol/feature/tx_cache
zhang0125 Aug 3, 2022
299d685
perf(net): compatible network configuration
xxo1shine Aug 3, 2022
c0162ec
Merge branch 'release_v4.5.2' of https://github.com/tronprotocol/java…
xxo1shine Aug 3, 2022
50471dd
Merge pull request #4561 from wubin01/work
xxo1shine Aug 3, 2022
45f0119
feat(log): modify log output level
forfreeday Aug 4, 2022
454d410
Merge pull request #4568 from forfreeday/feature/optimized_error_log
forfreeday Aug 4, 2022
a077c5d
perf(net): optimize disconnect logic
xxo1shine Aug 4, 2022
69fc806
Merge pull request #4569 from wubin01/work
xxo1shine Aug 5, 2022
078943e
feat(jsonrpc): if revert happens when executing contract in eth_estim…
Aug 5, 2022
8afd14d
feat(jsonrpc): catch exception when call and return err msg
Aug 5, 2022
dbc2524
Merge pull request #4570 from tronprotocol/feature/api_enhance
waynercheung Aug 5, 2022
77fb40f
perf(net): configurable block cache validity period
xxo1shine Aug 8, 2022
b6a6b83
test(api):add test for query block header.
halibobo1205 Aug 8, 2022
144693f
Merge pull request #4576 from halibobo1205/feature/api_header_query_test
forfreeday Aug 9, 2022
8c91dd4
test(db-mv):add test for db mv func.
halibobo1205 Aug 8, 2022
e4175e5
fix(solidityNode): fix solidity node startup issue
jwrct Aug 9, 2022
42eec0d
perf(solidityNode): optimize solidity node startup process
jwrct Aug 9, 2022
8b559ef
Merge pull request #4577 from chengtx01/feature/optimize_blocks_proce…
jwrct Aug 10, 2022
f3ec4a0
Merge pull request #4571 from wubin01/work
xxo1shine Aug 10, 2022
3630506
fix(lite):fix merge function when split and merge base on liteNode.
halibobo1205 Aug 10, 2022
a2c22ce
Merge pull request #4578 from halibobo1205/fix/lite_tool_merge
zhang0125 Aug 10, 2022
4e8972d
Merge pull request #4575 from halibobo1205/feature/db_transfer_tool_test
zhang0125 Aug 10, 2022
40b250c
fix(tx-cache): fix lite node init error.
zhang0125 Aug 10, 2022
ffa9ad2
Merge pull request #4579 from tronprotocol/fix/lite_node_init
zhang0125 Aug 10, 2022
afbd927
log(lite-tool):merge function.
halibobo1205 Aug 10, 2022
c1e8416
Merge pull request #4580 from halibobo1205/fix/lite_tool_merge
halibobo1205 Aug 10, 2022
f53fee8
feat(tx-cache): change the default value of the txCache.estimatedTran…
zhang0125 Aug 10, 2022
f5d9e31
Merge pull request #4581 from tronprotocol/feature/txcache_config
zhang0125 Aug 11, 2022
f7f88b0
fix(block): fix different result code when fork.
halibobo1205 Aug 11, 2022
5c1868e
Merge pull request #4583 from tronprotocol/fix/different_result_code_…
halibobo1205 Aug 11, 2022
8f1b4c2
close the peer connection when the service is down
xxo1shine Aug 12, 2022
43ecda4
Merge branch 'release_v4.5.2' of https://github.com/tronprotocol/java…
xxo1shine Aug 12, 2022
a1025af
Merge pull request #4586 from wubin01/work
xxo1shine Aug 12, 2022
378dd3e
fix(vm, hard_fork): init energy limit hard fork while init db manager
yanghang8612 Aug 12, 2022
1fba2ef
Merge pull request #4588 from tronprotocol/feature/init_hard_fork
yanghang8612 Aug 12, 2022
6d8cea7
perf(net): optimize message sending thread shutdown logic
xxo1shine Aug 12, 2022
a8b222b
Merge pull request #4589 from wubin01/work
xxo1shine Aug 12, 2022
2cb7b7b
feat(version): update Version
forfreeday Aug 15, 2022
2d88f6e
Merge pull request #4590 from forfreeday/feature/update_version
forfreeday Aug 15, 2022
a9809b5
Merge pull request #4594 from tronprotocol/release_v4.5.2
lvs007 Aug 18, 2022
4d8cb29
update a new version. version name:GreatVoyage-v4.5.1-125-ga9809b5bb,…
lvs007 Aug 18, 2022
349116e
Merge pull request #4595 from tronprotocol/version/GreatVoyage-v4.5.1…
lvs007 Aug 18, 2022
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion Tron protobuf protocol document.md
Original file line number Diff line number Diff line change
Expand Up @@ -2019,7 +2019,7 @@ message `SmartContract` has mutiple attributes and nested message `ABI`
TOO_MANY_PEERS = 0x04;
DUPLICATE_PEER = 0x05;
INCOMPATIBLE_PROTOCOL = 0x06;
NULL_IDENTITY = 0x07;
RANDOM_ELIMINATION = 0x07;
PEER_QUITING = 0x08;
UNEXPECTED_IDENTITY = 0x09;
LOCAL_IDENTITY = 0x0A;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,9 @@ private TransactionCapsule getTransactionFromBlockStore(byte[] key, long blockNu

private TransactionCapsule getTransactionFromKhaosDatabase(byte[] key, long high) {
List<KhaosBlock> khaosBlocks = khaosDatabase.getMiniStore().getBlockByNum(high);
if (khaosBlocks == null) {
return null;
}
for (KhaosBlock bl : khaosBlocks) {
for (TransactionCapsule e : bl.getBlk().getTransactions()) {
if (e.getTransactionId().equals(Sha256Hash.wrap(key))) {
Expand Down
200 changes: 109 additions & 91 deletions chainbase/src/main/java/org/tron/core/db2/common/TxCacheDB.java
Original file line number Diff line number Diff line change
@@ -1,25 +1,19 @@
package org.tron.core.db2.common;

import com.google.common.collect.ArrayListMultimap;
import com.google.common.collect.Iterators;
import com.google.common.collect.Maps;
import com.google.common.collect.Multimap;
import com.google.common.hash.BloomFilter;
import com.google.common.hash.Funnels;
import com.google.common.primitives.Longs;

import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
import java.util.WeakHashMap;

import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.ArrayUtils;
import org.bouncycastle.util.encoders.Hex;
import org.iq80.leveldb.WriteOptions;
import org.tron.common.parameter.CommonParameter;
import org.tron.common.prometheus.MetricKeys;
import org.tron.common.prometheus.Metrics;
import org.tron.common.storage.leveldb.LevelDbDataSourceImpl;
import org.tron.common.storage.rocksdb.RocksDbDataSourceImpl;
import org.tron.common.utils.ByteArray;
Expand All @@ -34,44 +28,69 @@
public class TxCacheDB implements DB<byte[], byte[]>, Flusher {

// > 65_536(= 2^16) blocks, that is the number of the reference block
private final int BLOCK_COUNT = 70_000;
private final long MAX_SIZE = 65536;
private Map<Key, Long> db = new WeakHashMap<>();
private Multimap<Long, Key> blockNumMap = ArrayListMultimap.create();
private String name;
private RecentTransactionStore recentTransactionStore;
private static final long MAX_BLOCK_SIZE = 65536;
// estimated number transactions in one block
private final int TRANSACTION_COUNT;

private static final long INVALID_BLOCK = -1;

// Since the filter cannot query for specific record information,
// FAKE_TRANSACTION represent the record presence.
private final byte[] FAKE_TRANSACTION = ByteArray.fromLong(0);

// a pair of bloom filters record the recent transactions
private BloomFilter<byte[]>[] bloomFilters = new BloomFilter[2];
// filterStartBlock record the start block of the active filter
private volatile long filterStartBlock = INVALID_BLOCK;
// currentFilterIndex records the index of the active filter
private volatile int currentFilterIndex = 0;

// record the last metric block to avoid duplication
private long lastMetricBlock = 0;

private final String name;

// add a persistent storage, the store name is: trans-cache
// when fullnode startup, transactionCache initializes transactions from this store
private DB<byte[], byte[]> persistentStore;

// replace persistentStore and optimizes startup performance
private RecentTransactionStore recentTransactionStore;

public TxCacheDB(String name, RecentTransactionStore recentTransactionStore) {
this.name = name;
this.TRANSACTION_COUNT =
CommonParameter.getInstance().getStorage().getEstimatedBlockTransactions();
this.recentTransactionStore = recentTransactionStore;
int dbVersion = CommonParameter.getInstance().getStorage().getDbVersion();
String dbEngine = CommonParameter.getInstance().getStorage().getDbEngine();
if (dbVersion == 2) {
if ("LEVELDB".equals(dbEngine.toUpperCase())) {
this.persistentStore = new LevelDB(
new LevelDbDataSourceImpl(StorageUtils.getOutputDirectoryByDbName(name),
name, StorageUtils.getOptionsByDbName(name),
new WriteOptions().sync(CommonParameter.getInstance()
.getStorage().isDbSync())));
new LevelDbDataSourceImpl(StorageUtils.getOutputDirectoryByDbName(name),
name, StorageUtils.getOptionsByDbName(name),
new WriteOptions().sync(CommonParameter.getInstance()
.getStorage().isDbSync())));
} else if ("ROCKSDB".equals(dbEngine.toUpperCase())) {
String parentPath = Paths
.get(StorageUtils.getOutputDirectoryByDbName(name), CommonParameter
.getInstance().getStorage().getDbDirectory()).toString();
.get(StorageUtils.getOutputDirectoryByDbName(name), CommonParameter
.getInstance().getStorage().getDbDirectory()).toString();

this.persistentStore = new RocksDB(
new RocksDbDataSourceImpl(parentPath,
name, CommonParameter.getInstance()
.getRocksDBCustomSettings()));
new RocksDbDataSourceImpl(parentPath,
name, CommonParameter.getInstance()
.getRocksDBCustomSettings()));
} else {
throw new RuntimeException("db type is not supported.");
}
} else {
throw new RuntimeException("db version is not supported.");
}
this.bloomFilters[0] = BloomFilter.create(Funnels.byteArrayFunnel(),
MAX_BLOCK_SIZE * TRANSACTION_COUNT);
this.bloomFilters[1] = BloomFilter.create(Funnels.byteArrayFunnel(),
MAX_BLOCK_SIZE * TRANSACTION_COUNT);

init();
}

Expand All @@ -81,57 +100,52 @@ public TxCacheDB(String name, RecentTransactionStore recentTransactionStore) {
private void initCache() {
long start = System.currentTimeMillis();
DBIterator iterator = (DBIterator) persistentStore.iterator();
long persistentSize = 0;
while (iterator.hasNext()) {
Entry<byte[], byte[]> entry = iterator.next();
byte[] key = entry.getKey();
byte[] value = entry.getValue();
if (key == null || value == null) {
if (ArrayUtils.isEmpty(entry.getKey()) || ArrayUtils.isEmpty(entry.getValue())) {
return;
}
Key k = Key.copyOf(key);
Long v = Longs.fromByteArray(value);
blockNumMap.put(v, k);
db.put(k, v);
bloomFilters[1].put(entry.getKey());
persistentSize++;
}
logger.info("Transaction cache init-1 db-size:{}, blockNumMap-size:{}, cost:{}ms",
db.size(), blockNumMap.size(), System.currentTimeMillis() - start);
logger.info("load transaction cache from persistentStore "
+ "db-size:{}, filter-size:{}, filter-fpp:{}, cost:{}ms",
persistentSize,
bloomFilters[1].approximateElementCount(), bloomFilters[1].expectedFpp(),
System.currentTimeMillis() - start);
}

private void init() {
long size = recentTransactionStore.size();
if (size != MAX_SIZE) {
if (size != MAX_BLOCK_SIZE) {
// 0. load from persistentStore
initCache();
return;
}

// 1. load from recentTransactionStore
long start = System.currentTimeMillis();
List<byte[]> l1 = new ArrayList<>();
List<RecentTransactionItem> l2 = new ArrayList<>();
Iterator<Entry<byte[], BytesCapsule>> iterator = recentTransactionStore.iterator();
while (iterator.hasNext()) {
l1.add(iterator.next().getValue().getData());
}

l1.forEach(v -> l2.add(JsonUtil.json2Obj(new String(v), RecentTransactionItem.class)));

l2.forEach(v -> v.getTransactionIds().forEach(tid -> putTransaction(tid, v.getNum())));
for (Entry<byte[], BytesCapsule> bytesCapsuleEntry : recentTransactionStore) {
byte[] data = bytesCapsuleEntry.getValue().getData();
RecentTransactionItem trx =
JsonUtil.json2Obj(new String(data), RecentTransactionItem.class);

logger.info("Transaction cache init-2 db-size:{}, blockNumMap-size:{}, cost:{}ms",
db.size(), blockNumMap.size(), System.currentTimeMillis() - start);
}
trx.getTransactionIds().forEach(tid -> bloomFilters[1].put(Hex.decode(tid)));
}

private void putTransaction(String key, long value) {
Key k = Key.copyOf(Hex.decode(key));
Long v = Longs.fromByteArray(ByteArray.fromLong(value));
blockNumMap.put(v, k);
db.put(k, v);
logger.info("load transaction cache from recentTransactionStore"
+ " filter-size:{}, filter-fpp:{}, cost:{}ms",
bloomFilters[1].approximateElementCount(), bloomFilters[1].expectedFpp(),
System.currentTimeMillis() - start);
}


@Override
public byte[] get(byte[] key) {
Long v = db.get(Key.of(key));
return v == null ? null : Longs.toByteArray(v);
if (!bloomFilters[0].mightContain(key) && !bloomFilters[1].mightContain(key)) {
return null;
}
// this means exist
return FAKE_TRANSACTION;
}

@Override
Expand All @@ -140,46 +154,54 @@ public void put(byte[] key, byte[] value) {
return;
}

Key k = Key.copyOf(key);
Long v = Longs.fromByteArray(value);
blockNumMap.put(v, k);
db.put(k, v);
// put the data into persistent storage
persistentStore.put(key, value);
removeEldest();
}
long blockNum = Longs.fromByteArray(value);
if (filterStartBlock == INVALID_BLOCK) {
// init active filter start block
filterStartBlock = blockNum;
currentFilterIndex = 0;
logger.info("init tx cache bloomFilters at {}", blockNum);
} else if (blockNum - filterStartBlock > MAX_BLOCK_SIZE) {
// active filter is full
logger.info("active bloomFilters is full (size={} fpp={}), create a new one (start={})",
bloomFilters[currentFilterIndex].approximateElementCount(),
bloomFilters[currentFilterIndex].expectedFpp(),
blockNum);

if (currentFilterIndex == 0) {
currentFilterIndex = 1;
} else {
currentFilterIndex = 0;
}

private void removeEldest() {
Set<Long> keys = blockNumMap.keySet();
if (keys.size() > BLOCK_COUNT) {
keys.stream()
.min(Long::compareTo)
.ifPresent(k -> {
Collection<Key> trxHashs = blockNumMap.get(k);
// remove transaction from persistentStore,
// if foreach is inefficient, change remove-foreach to remove-batch
trxHashs.forEach(key -> persistentStore.remove(key.getBytes()));
blockNumMap.removeAll(k);
logger.debug("******removeEldest block number:{}, block count:{}", k, keys.size());
});
filterStartBlock = blockNum;
bloomFilters[currentFilterIndex] =
BloomFilter.create(Funnels.byteArrayFunnel(),
MAX_BLOCK_SIZE * TRANSACTION_COUNT);
}
bloomFilters[currentFilterIndex].put(key);

if (lastMetricBlock != blockNum) {
lastMetricBlock = blockNum;
Metrics.gaugeSet(MetricKeys.Gauge.TX_CACHE,
bloomFilters[currentFilterIndex].approximateElementCount(), "count");
Metrics.gaugeSet(MetricKeys.Gauge.TX_CACHE,
bloomFilters[currentFilterIndex].expectedFpp(), "fpp");
}
}

@Override
public long size() {
return db.size();
throw new UnsupportedOperationException("TxCacheDB size");
}

@Override
public boolean isEmpty() {
return db.isEmpty();
throw new UnsupportedOperationException("TxCacheDB isEmpty");
}

@Override
public void remove(byte[] key) {
if (key != null) {
db.remove(Key.of(key));
}
throw new UnsupportedOperationException("TxCacheDB remove");
}

@Override
Expand All @@ -189,8 +211,7 @@ public String getDbName() {

@Override
public Iterator<Entry<byte[], byte[]>> iterator() {
return Iterators.transform(db.entrySet().iterator(),
e -> Maps.immutableEntry(e.getKey().getBytes(), Longs.toByteArray(e.getValue())));
throw new UnsupportedOperationException("TxCacheDB iterator");
}

@Override
Expand All @@ -201,15 +222,13 @@ public void flush(Map<WrappedByteArray, WrappedByteArray> batch) {
@Override
public void close() {
reset();
db = null;
blockNumMap = null;
bloomFilters[0] = null;
bloomFilters[1] = null;
persistentStore.close();
}

@Override
public void reset() {
db.clear();
blockNumMap.clear();
}

@Override
Expand All @@ -219,7 +238,6 @@ public TxCacheDB newInstance() {

@Override
public void stat() {
this.persistentStore.stat();
}
}

Loading