From ee8f9aec4149c153fff86447e63f1632ef241ad8 Mon Sep 17 00:00:00 2001 From: jiangyuanshu <317787106@qq.com> Date: Thu, 14 Jul 2022 11:03:56 +0800 Subject: [PATCH 01/51] optimize blockLock; MessageQueue does not send message if channel is disconnected --- .../common/overlay/server/ChannelManager.java | 16 ++++++++-------- .../tron/common/overlay/server/MessageQueue.java | 10 ++++++++++ .../org/tron/core/net/service/SyncService.java | 8 ++++---- 3 files changed, 22 insertions(+), 12 deletions(-) diff --git a/framework/src/main/java/org/tron/common/overlay/server/ChannelManager.java b/framework/src/main/java/org/tron/common/overlay/server/ChannelManager.java index dbcb73baa8b..8c452a2f880 100644 --- a/framework/src/main/java/org/tron/common/overlay/server/ChannelManager.java +++ b/framework/src/main/java/org/tron/common/overlay/server/ChannelManager.java @@ -34,7 +34,7 @@ @Component public class ChannelManager { - private final Map activePeers = new ConcurrentHashMap<>(); + private final Map activeChannels = new ConcurrentHashMap<>(); @Autowired private PeerServer peerServer; @Autowired @@ -121,7 +121,7 @@ public void processDisconnect(Channel channel, ReasonCode reason) { public void notifyDisconnect(Channel channel) { syncPool.onDisconnect(channel); - activePeers.values().remove(channel); + activeChannels.values().remove(channel); if (channel != null) { if (channel.getNodeStatistics() != null) { channel.getNodeStatistics().notifyDisconnect(); @@ -146,7 +146,7 @@ public synchronized boolean processPeer(Channel peer) { return false; } - if (!peer.isActive() && activePeers.size() >= maxActivePeers) { + if (!peer.isActive() && activeChannels.size() >= maxActivePeers) { peer.disconnect(TOO_MANY_PEERS); return false; } @@ -157,7 +157,7 @@ public synchronized boolean processPeer(Channel peer) { } } - Channel channel = activePeers.get(peer.getNodeIdWrapper()); + Channel channel = activeChannels.get(peer.getNodeIdWrapper()); if (channel != null) { if (channel.getStartTime() > peer.getStartTime()) { logger.info("Disconnect connection established later, {}", channel.getNode()); @@ -167,14 +167,14 @@ public synchronized boolean processPeer(Channel peer) { return false; } } - activePeers.put(peer.getNodeIdWrapper(), peer); - logger.info("Add active peer {}, total active peers: {}", peer, activePeers.size()); + activeChannels.put(peer.getNodeIdWrapper(), peer); + logger.info("Add active peer {}, total active peers: {}", peer, activeChannels.size()); return true; } public int getConnectionNum(InetAddress inetAddress) { int cnt = 0; - for (Channel channel : activePeers.values()) { + for (Channel channel : activeChannels.values()) { if (channel.getInetAddress().equals(inetAddress)) { cnt++; } @@ -183,7 +183,7 @@ public int getConnectionNum(InetAddress inetAddress) { } public Collection getActivePeers() { - return activePeers.values(); + return activeChannels.values(); } public Cache getRecentlyDisconnected() { diff --git a/framework/src/main/java/org/tron/common/overlay/server/MessageQueue.java b/framework/src/main/java/org/tron/common/overlay/server/MessageQueue.java index 1919042232f..3e3fc3b2c44 100644 --- a/framework/src/main/java/org/tron/common/overlay/server/MessageQueue.java +++ b/framework/src/main/java/org/tron/common/overlay/server/MessageQueue.java @@ -69,6 +69,11 @@ public void activate(ChannelHandlerContext ctx) { continue; } Message msg = msgQueue.take(); + if (channel.isDisconnect()) { + logger.warn("Failed to send to {} as channel has closed, {}", + ctx.channel().remoteAddress(), msg); + return; + } ctx.writeAndFlush(msg.getSendData()).addListener((ChannelFutureListener) future -> { if (!future.isSuccess() && !channel.isDisconnect()) { logger.warn("Failed to send to {}, {}", ctx.channel().remoteAddress(), msg); @@ -92,6 +97,11 @@ public void setChannel(Channel channel) { } public void fastSend(Message msg) { + if (channel.isDisconnect()) { + logger.warn("Fast send to {} failed as channel has closed, {} ", + ctx.channel().remoteAddress(), msg); + return; + } logger.info("Fast send to {}, {} ", ctx.channel().remoteAddress(), msg); ctx.writeAndFlush(msg.getSendData()).addListener((ChannelFutureListener) future -> { if (!future.isSuccess() && !channel.isDisconnect()) { diff --git a/framework/src/main/java/org/tron/core/net/service/SyncService.java b/framework/src/main/java/org/tron/core/net/service/SyncService.java index 64a721bf014..67c3629d22b 100644 --- a/framework/src/main/java/org/tron/core/net/service/SyncService.java +++ b/framework/src/main/java/org/tron/core/net/service/SyncService.java @@ -234,8 +234,8 @@ private synchronized void handleSyncBlock() { isProcessed[0] = false; - synchronized (tronNetDelegate.getBlockLock()) { - blockWaitToProcess.forEach((msg, peerConnection) -> { + blockWaitToProcess.forEach((msg, peerConnection) -> { + synchronized (tronNetDelegate.getBlockLock()) { if (peerConnection.isDisconnect()) { blockWaitToProcess.remove(msg); invalid(msg.getBlockId()); @@ -254,8 +254,8 @@ private synchronized void handleSyncBlock() { isProcessed[0] = true; processSyncBlock(msg.getBlockCapsule()); } - }); - } + } + }); } } From fe8efd137da3c330e2caa36fa9b222e4490fabb8 Mon Sep 17 00:00:00 2001 From: jiangyuanshu <317787106@qq.com> Date: Fri, 15 Jul 2022 13:01:06 +0800 Subject: [PATCH 02/51] clear message from MessageQueue before channel close --- .../main/java/org/tron/common/overlay/server/MessageQueue.java | 1 + 1 file changed, 1 insertion(+) diff --git a/framework/src/main/java/org/tron/common/overlay/server/MessageQueue.java b/framework/src/main/java/org/tron/common/overlay/server/MessageQueue.java index 3e3fc3b2c44..a8db0225be4 100644 --- a/framework/src/main/java/org/tron/common/overlay/server/MessageQueue.java +++ b/framework/src/main/java/org/tron/common/overlay/server/MessageQueue.java @@ -72,6 +72,7 @@ public void activate(ChannelHandlerContext ctx) { if (channel.isDisconnect()) { logger.warn("Failed to send to {} as channel has closed, {}", ctx.channel().remoteAddress(), msg); + msgQueue.clear(); return; } ctx.writeAndFlush(msg.getSendData()).addListener((ChannelFutureListener) future -> { From 6ccb55406513f8182b01a5c53e930b51f3fc0172 Mon Sep 17 00:00:00 2001 From: jiangyuanshu <317787106@qq.com> Date: Mon, 18 Jul 2022 15:21:18 +0800 Subject: [PATCH 03/51] rollback --- .../org/tron/common/overlay/server/MessageQueue.java | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/framework/src/main/java/org/tron/common/overlay/server/MessageQueue.java b/framework/src/main/java/org/tron/common/overlay/server/MessageQueue.java index 3e3fc3b2c44..1919042232f 100644 --- a/framework/src/main/java/org/tron/common/overlay/server/MessageQueue.java +++ b/framework/src/main/java/org/tron/common/overlay/server/MessageQueue.java @@ -69,11 +69,6 @@ public void activate(ChannelHandlerContext ctx) { continue; } Message msg = msgQueue.take(); - if (channel.isDisconnect()) { - logger.warn("Failed to send to {} as channel has closed, {}", - ctx.channel().remoteAddress(), msg); - return; - } ctx.writeAndFlush(msg.getSendData()).addListener((ChannelFutureListener) future -> { if (!future.isSuccess() && !channel.isDisconnect()) { logger.warn("Failed to send to {}, {}", ctx.channel().remoteAddress(), msg); @@ -97,11 +92,6 @@ public void setChannel(Channel channel) { } public void fastSend(Message msg) { - if (channel.isDisconnect()) { - logger.warn("Fast send to {} failed as channel has closed, {} ", - ctx.channel().remoteAddress(), msg); - return; - } logger.info("Fast send to {}, {} ", ctx.channel().remoteAddress(), msg); ctx.writeAndFlush(msg.getSendData()).addListener((ChannelFutureListener) future -> { if (!future.isSuccess() && !channel.isDisconnect()) { From cfec5b1fd571eacb7b343a483e0acd88ed4238b8 Mon Sep 17 00:00:00 2001 From: jiangyuanshu <317787106@qq.com> Date: Mon, 18 Jul 2022 15:32:32 +0800 Subject: [PATCH 04/51] rollback --- .../common/overlay/server/ChannelManager.java | 16 ++++++++-------- .../org/tron/core/net/service/SyncService.java | 8 ++++---- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/framework/src/main/java/org/tron/common/overlay/server/ChannelManager.java b/framework/src/main/java/org/tron/common/overlay/server/ChannelManager.java index 8c452a2f880..dbcb73baa8b 100644 --- a/framework/src/main/java/org/tron/common/overlay/server/ChannelManager.java +++ b/framework/src/main/java/org/tron/common/overlay/server/ChannelManager.java @@ -34,7 +34,7 @@ @Component public class ChannelManager { - private final Map activeChannels = new ConcurrentHashMap<>(); + private final Map activePeers = new ConcurrentHashMap<>(); @Autowired private PeerServer peerServer; @Autowired @@ -121,7 +121,7 @@ public void processDisconnect(Channel channel, ReasonCode reason) { public void notifyDisconnect(Channel channel) { syncPool.onDisconnect(channel); - activeChannels.values().remove(channel); + activePeers.values().remove(channel); if (channel != null) { if (channel.getNodeStatistics() != null) { channel.getNodeStatistics().notifyDisconnect(); @@ -146,7 +146,7 @@ public synchronized boolean processPeer(Channel peer) { return false; } - if (!peer.isActive() && activeChannels.size() >= maxActivePeers) { + if (!peer.isActive() && activePeers.size() >= maxActivePeers) { peer.disconnect(TOO_MANY_PEERS); return false; } @@ -157,7 +157,7 @@ public synchronized boolean processPeer(Channel peer) { } } - Channel channel = activeChannels.get(peer.getNodeIdWrapper()); + Channel channel = activePeers.get(peer.getNodeIdWrapper()); if (channel != null) { if (channel.getStartTime() > peer.getStartTime()) { logger.info("Disconnect connection established later, {}", channel.getNode()); @@ -167,14 +167,14 @@ public synchronized boolean processPeer(Channel peer) { return false; } } - activeChannels.put(peer.getNodeIdWrapper(), peer); - logger.info("Add active peer {}, total active peers: {}", peer, activeChannels.size()); + activePeers.put(peer.getNodeIdWrapper(), peer); + logger.info("Add active peer {}, total active peers: {}", peer, activePeers.size()); return true; } public int getConnectionNum(InetAddress inetAddress) { int cnt = 0; - for (Channel channel : activeChannels.values()) { + for (Channel channel : activePeers.values()) { if (channel.getInetAddress().equals(inetAddress)) { cnt++; } @@ -183,7 +183,7 @@ public int getConnectionNum(InetAddress inetAddress) { } public Collection getActivePeers() { - return activeChannels.values(); + return activePeers.values(); } public Cache getRecentlyDisconnected() { diff --git a/framework/src/main/java/org/tron/core/net/service/SyncService.java b/framework/src/main/java/org/tron/core/net/service/SyncService.java index 67c3629d22b..64a721bf014 100644 --- a/framework/src/main/java/org/tron/core/net/service/SyncService.java +++ b/framework/src/main/java/org/tron/core/net/service/SyncService.java @@ -234,8 +234,8 @@ private synchronized void handleSyncBlock() { isProcessed[0] = false; - blockWaitToProcess.forEach((msg, peerConnection) -> { - synchronized (tronNetDelegate.getBlockLock()) { + synchronized (tronNetDelegate.getBlockLock()) { + blockWaitToProcess.forEach((msg, peerConnection) -> { if (peerConnection.isDisconnect()) { blockWaitToProcess.remove(msg); invalid(msg.getBlockId()); @@ -254,8 +254,8 @@ private synchronized void handleSyncBlock() { isProcessed[0] = true; processSyncBlock(msg.getBlockCapsule()); } - } - }); + }); + } } } From 3f6b1db07d176b3c7ce0cb71ba8b6ef074ad3913 Mon Sep 17 00:00:00 2001 From: jiangyuanshu <317787106@qq.com> Date: Fri, 22 Jul 2022 10:42:02 +0800 Subject: [PATCH 05/51] fix error that use ZMQ.Socket to send data may fail in multithreads with rare rates --- .../logsfilter/nativequeue/NativeMessageQueue.java | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/framework/src/main/java/org/tron/common/logsfilter/nativequeue/NativeMessageQueue.java b/framework/src/main/java/org/tron/common/logsfilter/nativequeue/NativeMessageQueue.java index 61be8e84fb7..cf2fe6dce0a 100644 --- a/framework/src/main/java/org/tron/common/logsfilter/nativequeue/NativeMessageQueue.java +++ b/framework/src/main/java/org/tron/common/logsfilter/nativequeue/NativeMessageQueue.java @@ -1,10 +1,12 @@ package org.tron.common.logsfilter.nativequeue; import java.util.Objects; +import lombok.extern.slf4j.Slf4j; import org.zeromq.SocketType; import org.zeromq.ZContext; import org.zeromq.ZMQ; +@Slf4j public class NativeMessageQueue { private static final int DEFAULT_BIND_PORT = 5555; @@ -61,7 +63,12 @@ public void publishTrigger(String data, String topic) { return; } - publisher.sendMore(topic); - publisher.send(data); + try { + publisher.sendMore(topic); + publisher.send(data); + } catch (RuntimeException e) { + logger.error("write data to zeromq failed, data:{}, topic:{}, error:{}", data, topic, + e.getMessage()); + } } } From b927f739557dd2a0f690657b1249569507981509 Mon Sep 17 00:00:00 2001 From: jiangyuanshu <317787106@qq.com> Date: Fri, 22 Jul 2022 10:49:43 +0800 Subject: [PATCH 06/51] rollback --- .../org/tron/common/overlay/server/MessageQueue.java | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/framework/src/main/java/org/tron/common/overlay/server/MessageQueue.java b/framework/src/main/java/org/tron/common/overlay/server/MessageQueue.java index a8db0225be4..1919042232f 100644 --- a/framework/src/main/java/org/tron/common/overlay/server/MessageQueue.java +++ b/framework/src/main/java/org/tron/common/overlay/server/MessageQueue.java @@ -69,12 +69,6 @@ public void activate(ChannelHandlerContext ctx) { continue; } Message msg = msgQueue.take(); - if (channel.isDisconnect()) { - logger.warn("Failed to send to {} as channel has closed, {}", - ctx.channel().remoteAddress(), msg); - msgQueue.clear(); - return; - } ctx.writeAndFlush(msg.getSendData()).addListener((ChannelFutureListener) future -> { if (!future.isSuccess() && !channel.isDisconnect()) { logger.warn("Failed to send to {}, {}", ctx.channel().remoteAddress(), msg); @@ -98,11 +92,6 @@ public void setChannel(Channel channel) { } public void fastSend(Message msg) { - if (channel.isDisconnect()) { - logger.warn("Fast send to {} failed as channel has closed, {} ", - ctx.channel().remoteAddress(), msg); - return; - } logger.info("Fast send to {}, {} ", ctx.channel().remoteAddress(), msg); ctx.writeAndFlush(msg.getSendData()).addListener((ChannelFutureListener) future -> { if (!future.isSuccess() && !channel.isDisconnect()) { From d83b65ce4fc919473fda4ad2b6d924c2488b7c65 Mon Sep 17 00:00:00 2001 From: jiangyuanshu <317787106@qq.com> Date: Tue, 26 Jul 2022 11:10:00 +0800 Subject: [PATCH 07/51] resolve sonarlint error of tronState --- .../src/main/java/org/tron/common/overlay/server/Channel.java | 1 + 1 file changed, 1 insertion(+) diff --git a/framework/src/main/java/org/tron/common/overlay/server/Channel.java b/framework/src/main/java/org/tron/common/overlay/server/Channel.java index 0846ff61251..68ff3174f22 100644 --- a/framework/src/main/java/org/tron/common/overlay/server/Channel.java +++ b/framework/src/main/java/org/tron/common/overlay/server/Channel.java @@ -61,6 +61,7 @@ public class Channel { private InetSocketAddress inetSocketAddress; private Node node; private long startTime; + @Getter private TronState tronState = TronState.INIT; private boolean isActive; @Getter From 2eb91bfa09153237ad7bc75069af880f50a03a7a Mon Sep 17 00:00:00 2001 From: wubin01 Date: Tue, 26 Jul 2022 11:28:44 +0800 Subject: [PATCH 08/51] perf(net): speed up TCP connection establishment --- Tron protobuf protocol document.md | 2 +- .../common/parameter/CommonParameter.java | 22 ++- .../src/main/java/org/tron/core/Constant.java | 15 +-- .../common/overlay/server/ChannelManager.java | 12 +- .../server/PeerConnectionCheckService.java | 127 ++++++------------ .../tron/common/overlay/server/SyncPool.java | 24 ++-- .../java/org/tron/core/config/args/Args.java | 45 +++---- .../tron/core/services/NodeInfoService.java | 4 +- .../src/main/resources/config-backup.conf | 2 - framework/src/main/resources/config-beta.conf | 3 - .../src/main/resources/config-localtest.conf | 10 +- .../src/main/resources/config-test-net.conf | 10 +- framework/src/main/resources/config.conf | 17 +-- .../org/tron/core/config/args/ArgsTest.java | 2 +- .../src/test/resources/config-localtest.conf | 10 +- .../test/resources/config-test-dbbackup.conf | 10 +- .../src/test/resources/config-test-index.conf | 2 - .../test/resources/config-test-mainnet.conf | 4 +- .../resources/config-test-storagetest.conf | 4 +- framework/src/test/resources/config-test.conf | 2 - ...inese version of TRON Protocol document.md | 4 +- ...glish version of TRON Protocol document.md | 4 +- protocol/src/main/protos/core/Tron.proto | 2 +- 23 files changed, 118 insertions(+), 219 deletions(-) diff --git a/Tron protobuf protocol document.md b/Tron protobuf protocol document.md index 6a97805b22e..be9558a1d93 100644 --- a/Tron protobuf protocol document.md +++ b/Tron protobuf protocol document.md @@ -2019,7 +2019,7 @@ message `SmartContract` has mutiple attributes and nested message `ABI` TOO_MANY_PEERS = 0x04; DUPLICATE_PEER = 0x05; INCOMPATIBLE_PROTOCOL = 0x06; - NULL_IDENTITY = 0x07; + RANDOM_ELIMINATION = 0x07; PEER_QUITING = 0x08; UNEXPECTED_IDENTITY = 0x09; LOCAL_IDENTITY = 0x0A; diff --git a/common/src/main/java/org/tron/common/parameter/CommonParameter.java b/common/src/main/java/org/tron/common/parameter/CommonParameter.java index df4cf35f71c..6daf213e672 100644 --- a/common/src/main/java/org/tron/common/parameter/CommonParameter.java +++ b/common/src/main/java/org/tron/common/parameter/CommonParameter.java @@ -129,10 +129,16 @@ public class CommonParameter { public int nodeChannelReadTimeout; @Getter @Setter - public int nodeMaxActiveNodes; + public int maxConnections; @Getter @Setter - public int nodeMaxActiveNodesWithSameIp; + public int minConnections; + @Getter + @Setter + public int minActiveConnections; + @Getter + @Setter + public int maxConnectionsWithSameIp; @Getter @Setter public int minParticipationRate; @@ -286,18 +292,6 @@ public class CommonParameter { public List backupMembers; @Getter @Setter - public double connectFactor; - @Getter - @Setter - public double activeConnectFactor; - @Getter - @Setter - public double disconnectNumberFactor; - @Getter - @Setter - public double maxConnectNumberFactor; - @Getter - @Setter public long receiveTcpMinDataLength; @Getter @Setter diff --git a/common/src/main/java/org/tron/core/Constant.java b/common/src/main/java/org/tron/core/Constant.java index d0a557ccf44..1eb10792600 100644 --- a/common/src/main/java/org/tron/core/Constant.java +++ b/common/src/main/java/org/tron/core/Constant.java @@ -83,8 +83,11 @@ public class Constant { public static final String NODE_CONNECTION_TIMEOUT = "node.connection.timeout"; public static final String NODE_FETCH_BLOCK_TIMEOUT = "node.fetchBlock.timeout"; public static final String NODE_CHANNEL_READ_TIMEOUT = "node.channel.read.timeout"; - public static final String NODE_MAX_ACTIVE_NODES = "node.maxActiveNodes"; - public static final String NODE_MAX_ACTIVE_NODES_WITH_SAMEIP = "node.maxActiveNodesWithSameIp"; + public static final String NODE_MAX_CONNECTIONS = "node.maxConnections"; + public static final String NODE_MIN_CONNECTIONS = "node.minConnections"; + public static final String NODE_MIN_ACTIVE_CONNECTIONS = "node.minActiveConnections"; + + public static final String NODE_MAX_ACTIVE_NODES_WITH_SAMEIP = "node.maxConnectionsWithSameIp"; public static final String NODE_MIN_PARTICIPATION_RATE = "node.minParticipationRate"; public static final String NODE_LISTEN_PORT = "node.listen.port"; public static final String NODE_DISCOVERY_PUBLIC_HOME_NODE = "node.discovery.public.home.node"; @@ -160,14 +163,6 @@ public class Constant { public static final String NODE_WALLET_EXTENSION_API = "node.walletExtensionApi"; - public static final String NODE_CONNECT_FACTOR = "node.connectFactor"; - - public static final String NODE_ACTIVE_CONNECT_FACTOR = "node.activeConnectFactor"; - - public static final String NODE_DISCONNECT_NUMBER_FACTOR = "node.disconnectNumberFactor"; - - public static final String NODE_MAX_CONNECT_NUMBER_FACTOR = "node.maxConnectNumberFactor"; - public static final String NODE_RECEIVE_TCP_MIN_DATA_LENGTH = "node.receiveTcpMinDataLength"; public static final String NODE_IS_OPEN_FULL_TCP_DISCONNECT = "node.isOpenFullTcpDisconnect"; diff --git a/framework/src/main/java/org/tron/common/overlay/server/ChannelManager.java b/framework/src/main/java/org/tron/common/overlay/server/ChannelManager.java index dbcb73baa8b..114df13d847 100644 --- a/framework/src/main/java/org/tron/common/overlay/server/ChannelManager.java +++ b/framework/src/main/java/org/tron/common/overlay/server/ChannelManager.java @@ -42,6 +42,8 @@ public class ChannelManager { @Autowired private SyncPool syncPool; @Autowired + private PeerConnectionCheckService peerConnectionCheckService; + @Autowired private FastForward fastForward; private CommonParameter parameter = CommonParameter.getInstance(); private Cache badPeers = CacheBuilder.newBuilder().maximumSize(10000) @@ -63,9 +65,9 @@ public class ChannelManager { @Getter private Map fastForwardNodes = new ConcurrentHashMap(); - private int maxActivePeers = parameter.getNodeMaxActiveNodes(); + private int maxConnections = parameter.getMaxConnections(); - private int getMaxActivePeersWithSameIp = parameter.getNodeMaxActiveNodesWithSameIp(); + private int maxConnectionsWithSameIp = parameter.getMaxConnectionsWithSameIp(); public void init() { if (this.parameter.getNodeListenPort() > 0) { @@ -94,6 +96,7 @@ public void init() { logger.info("Node config, trust {}, active {}, forward {}.", trustNodes.size(), activeNodes.size(), fastForwardNodes.size()); + peerConnectionCheckService.init(); syncPool.init(); fastForward.init(); } @@ -146,12 +149,12 @@ public synchronized boolean processPeer(Channel peer) { return false; } - if (!peer.isActive() && activePeers.size() >= maxActivePeers) { + if (!peer.isActive() && activePeers.size() >= maxConnections) { peer.disconnect(TOO_MANY_PEERS); return false; } - if (getConnectionNum(peer.getInetAddress()) >= getMaxActivePeersWithSameIp) { + if (getConnectionNum(peer.getInetAddress()) >= maxConnectionsWithSameIp) { peer.disconnect(TOO_MANY_PEERS_WITH_SAME_IP); return false; } @@ -195,6 +198,7 @@ public Cache getBadPeers() { } public void close() { + peerConnectionCheckService.close(); syncPool.close(); peerServer.close(); peerClient.close(); diff --git a/framework/src/main/java/org/tron/common/overlay/server/PeerConnectionCheckService.java b/framework/src/main/java/org/tron/common/overlay/server/PeerConnectionCheckService.java index a02825e31e0..edd18bd9ed0 100644 --- a/framework/src/main/java/org/tron/common/overlay/server/PeerConnectionCheckService.java +++ b/framework/src/main/java/org/tron/common/overlay/server/PeerConnectionCheckService.java @@ -1,114 +1,69 @@ package org.tron.common.overlay.server; import java.util.ArrayList; -import java.util.Collections; -import java.util.Comparator; +import java.util.Collection; import java.util.List; +import java.util.Random; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; -import javax.annotation.PostConstruct; -import javax.annotation.PreDestroy; +import java.util.stream.Collectors; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; -import org.tron.common.overlay.discover.node.statistics.NodeStatistics; -import org.tron.common.utils.CollectionUtils; import org.tron.core.config.args.Args; -import org.tron.core.db.Manager; import org.tron.core.net.peer.PeerConnection; -import org.tron.protos.Protocol.ReasonCode; +import org.tron.protos.Protocol; @Slf4j(topic = "net") @Service public class PeerConnectionCheckService { - - public static final long CHECK_TIME = 5 * 60 * 1000L; - private double disconnectNumberFactor = Args.getInstance().getDisconnectNumberFactor(); - private double maxConnectNumberFactor = Args.getInstance().getMaxConnectNumberFactor(); - - @Autowired - private SyncPool pool; - - @Autowired - private ChannelManager channelManager; + private int p2pVersion = Args.getInstance().getNodeP2pVersion(); + private int maxConnections = Args.getInstance().getMaxConnections(); + private int minActiveConnections = Args.getInstance().getMinActiveConnections(); + private boolean isFastForward = Args.getInstance().isFastForward(); + private boolean isOpenFullTcpDisconnect = Args.getInstance().isOpenFullTcpDisconnect(); + private ScheduledExecutorService poolLoopExecutor = Executors.newSingleThreadScheduledExecutor(); @Autowired - private Manager manager; + private SyncPool syncPool; - private ScheduledExecutorService scheduledExecutorService = Executors.newScheduledThreadPool(2, - r -> new Thread(r, "check-peer-connect")); - - @PostConstruct - public void check() { - logger.info("start the PeerConnectionCheckService"); - scheduledExecutorService - .scheduleWithFixedDelay(new CheckDataTransferTask(), 5, 5, TimeUnit.MINUTES); - if (Args.getInstance().isOpenFullTcpDisconnect()) { - scheduledExecutorService - .scheduleWithFixedDelay(new CheckConnectNumberTask(), 4, 1, TimeUnit.MINUTES); + public void init() { + if (isFastForward || !isOpenFullTcpDisconnect) { + return; } + logger.info("Start peer connection check service."); + poolLoopExecutor.scheduleWithFixedDelay(() -> { + try { + check(); + } catch (Throwable t) { + logger.error("Exception in peer connection check.", t); + } + }, 10, 30, TimeUnit.SECONDS); } - @PreDestroy - public void destroy() { - scheduledExecutorService.shutdown(); + public void close() { + logger.info("Close peer connection check service."); + poolLoopExecutor.shutdown(); } - private class CheckDataTransferTask implements Runnable { - - @Override - public void run() { - List peerConnectionList = pool.getActivePeers(); - List willDisconnectPeerList = new ArrayList<>(); - for (PeerConnection peerConnection : peerConnectionList) { - NodeStatistics nodeStatistics = peerConnection.getNodeStatistics(); - if (!nodeStatistics.nodeIsHaveDataTransfer() - && System.currentTimeMillis() - peerConnection.getStartTime() >= CHECK_TIME - && !peerConnection.isTrustPeer() - && !nodeStatistics.isPredefined()) { - //if xxx minutes not have data transfer,disconnect the peer, - //exclude trust peer and active peer - willDisconnectPeerList.add(peerConnection); - } - nodeStatistics.resetTcpFlow(); - } - if (!willDisconnectPeerList.isEmpty() && peerConnectionList.size() - > Args.getInstance().getNodeMaxActiveNodes() * maxConnectNumberFactor) { - Collections.shuffle(willDisconnectPeerList); - for (int i = 0; i < willDisconnectPeerList.size() * disconnectNumberFactor; i++) { - logger.warn("{} does not have data transfer, disconnect the peer", - willDisconnectPeerList.get(i).getInetAddress()); - willDisconnectPeerList.get(i).disconnect(ReasonCode.TOO_MANY_PEERS); - } - } + public void check() { + if (syncPool.getActivePeers().size() < maxConnections) { + return; } - } - - private class CheckConnectNumberTask implements Runnable { - - @Override - public void run() { - if (pool.getActivePeers().size() >= Args.getInstance().getNodeMaxActiveNodes()) { - logger.warn("connection pool is full"); - List peerList = new ArrayList<>(); - for (PeerConnection peer : pool.getActivePeers()) { - if (!peer.isTrustPeer() && !peer.getNodeStatistics().isPredefined()) { - peerList.add(peer); - } - } - if (peerList.size() >= 2) { - peerList.sort( - Comparator.comparingInt((PeerConnection o) -> o.getNodeStatistics().getReputation())); - peerList = CollectionUtils.truncateRandom(peerList, 2, 1); - } - for (PeerConnection peerConnection : peerList) { - logger.warn("connection pool is full, disconnect the peer: {}", - peerConnection.getInetAddress()); - peerConnection.disconnect(ReasonCode.RESET); - } - } + boolean flag = syncPool.getActivePeersCount().get() > minActiveConnections * 2; + Collection peers = syncPool.getActivePeers().stream() + .filter(peer -> peer.isIdle()) + .filter(peer -> !peer.isTrustPeer()) + .filter(peer -> !peer.getNode().isConnectible(p2pVersion)) + .filter(peer -> peer.isActive() == flag) + .collect(Collectors.toList()); + if (peers.size() == 0) { + return; } + List list = new ArrayList(); + peers.forEach(p -> list.add(p)); + PeerConnection peer = list.get(new Random().nextInt(peers.size())); + peer.disconnect(Protocol.ReasonCode.RANDOM_ELIMINATION); } - } diff --git a/framework/src/main/java/org/tron/common/overlay/server/SyncPool.java b/framework/src/main/java/org/tron/common/overlay/server/SyncPool.java index 410266d07fa..4c35f6b628d 100644 --- a/framework/src/main/java/org/tron/common/overlay/server/SyncPool.java +++ b/framework/src/main/java/org/tron/common/overlay/server/SyncPool.java @@ -16,6 +16,7 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Predicate; + import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.ApplicationContext; @@ -37,15 +38,12 @@ @Slf4j(topic = "net") @Component public class SyncPool { - private final List activePeers = Collections .synchronizedList(new ArrayList<>()); - private final AtomicInteger passivePeersCount = new AtomicInteger(0); - private final AtomicInteger activePeersCount = new AtomicInteger(0); - private double factor = Args.getInstance().getConnectFactor(); - private double activeFactor = Args.getInstance().getActiveConnectFactor(); private Cache nodeHandlerCache = CacheBuilder.newBuilder() .maximumSize(1000).expireAfterWrite(180, TimeUnit.SECONDS).recordStats().build(); + private final AtomicInteger passivePeersCount = new AtomicInteger(0); + private final AtomicInteger activePeersCount = new AtomicInteger(0); @Autowired private NodeManager nodeManager; @@ -60,9 +58,7 @@ public class SyncPool { private CommonParameter commonParameter = CommonParameter.getInstance(); - private int maxActiveNodes = commonParameter.getNodeMaxActiveNodes(); - - private int maxActivePeersWithSameIp = commonParameter.getNodeMaxActiveNodesWithSameIp(); + private int maxConnectionsWithSameIp = commonParameter.getMaxConnectionsWithSameIp(); private ScheduledExecutorService poolLoopExecutor = Executors.newSingleThreadScheduledExecutor(); @@ -72,6 +68,10 @@ public class SyncPool { private int disconnectTimeout = 60_000; + private int maxConnections = Args.getInstance().getMaxConnections(); + private int minConnections = Args.getInstance().getMinConnections(); + private int minActiveConnections = Args.getInstance().getMinActiveConnections(); + public void init() { channelManager = ctx.getBean(ChannelManager.class); @@ -123,8 +123,8 @@ private void fillUp() { } }); - int size = Math.max((int) (maxActiveNodes * factor) - activePeers.size(), - (int) (maxActiveNodes * activeFactor - activePeersCount.get())); + int size = Math.max(minConnections - activePeers.size(), + minActiveConnections - activePeersCount.get()); int lackSize = size - connectNodes.size(); if (lackSize > 0) { nodesInUse.add(nodeManager.getPublicHomeNode().getHexId()); @@ -212,7 +212,7 @@ public synchronized void onDisconnect(Channel peer) { } public boolean isCanConnect() { - return passivePeersCount.get() < maxActiveNodes * (1 - activeFactor); + return passivePeersCount.get() < maxConnections - minActiveConnections; } public void close() { @@ -250,7 +250,7 @@ public boolean test(NodeHandler handler) { && handler.getNode().getPort() == nodeManager.getPublicHomeNode().getPort()) || (channelManager.getRecentlyDisconnected().getIfPresent(inetAddress) != null) || (channelManager.getBadPeers().getIfPresent(inetAddress) != null) - || (channelManager.getConnectionNum(inetAddress) >= maxActivePeersWithSameIp) + || (channelManager.getConnectionNum(inetAddress) >= maxConnectionsWithSameIp) || (nodesInUse.contains(handler.getNode().getHexId())) || (nodeHandlerCache.getIfPresent(handler) != null) || (message != null && headNum < message.getLowestBlockNum())); diff --git a/framework/src/main/java/org/tron/core/config/args/Args.java b/framework/src/main/java/org/tron/core/config/args/Args.java index c84e013f707..0e2482e7f37 100644 --- a/framework/src/main/java/org/tron/core/config/args/Args.java +++ b/framework/src/main/java/org/tron/core/config/args/Args.java @@ -116,14 +116,16 @@ public static void clearParam() { PARAMETER.needSyncCheck = false; PARAMETER.nodeDiscoveryEnable = false; PARAMETER.nodeDiscoveryPersist = false; - PARAMETER.nodeConnectionTimeout = 0; + PARAMETER.nodeConnectionTimeout = 2000; PARAMETER.activeNodes = Collections.emptyList(); PARAMETER.passiveNodes = Collections.emptyList(); PARAMETER.fastForwardNodes = Collections.emptyList(); PARAMETER.maxFastForwardNum = 3; PARAMETER.nodeChannelReadTimeout = 0; - PARAMETER.nodeMaxActiveNodes = 30; - PARAMETER.nodeMaxActiveNodesWithSameIp = 2; + PARAMETER.maxConnections = 30; + PARAMETER.minConnections = 8; + PARAMETER.minActiveConnections = 3; + PARAMETER.maxConnectionsWithSameIp = 2; PARAMETER.minParticipationRate = 0; PARAMETER.nodeListenPort = 0; PARAMETER.nodeDiscoveryBindIp = ""; @@ -158,10 +160,6 @@ public static void clearParam() { PARAMETER.solidityNode = false; PARAMETER.trustNodeAddr = ""; PARAMETER.walletExtensionApi = false; - PARAMETER.connectFactor = 0.3; - PARAMETER.activeConnectFactor = 0.1; - PARAMETER.disconnectNumberFactor = 0.4; - PARAMETER.maxConnectNumberFactor = 0.8; PARAMETER.receiveTcpMinDataLength = 2048; PARAMETER.isOpenFullTcpDisconnect = false; PARAMETER.supportConstant = false; @@ -448,7 +446,7 @@ public static void setParam(final String[] args, final String confFileName) { PARAMETER.nodeConnectionTimeout = config.hasPath(Constant.NODE_CONNECTION_TIMEOUT) ? config.getInt(Constant.NODE_CONNECTION_TIMEOUT) * 1000 - : 0; + : 2000; if (!config.hasPath(Constant.NODE_FETCH_BLOCK_TIMEOUT)) { PARAMETER.fetchBlockTimeout = 200; @@ -465,11 +463,19 @@ public static void setParam(final String[] args, final String confFileName) { ? config.getInt(Constant.NODE_CHANNEL_READ_TIMEOUT) : 0; - PARAMETER.nodeMaxActiveNodes = - config.hasPath(Constant.NODE_MAX_ACTIVE_NODES) - ? config.getInt(Constant.NODE_MAX_ACTIVE_NODES) : 30; + PARAMETER.maxConnections = + config.hasPath(Constant.NODE_MAX_CONNECTIONS) + ? config.getInt(Constant.NODE_MAX_CONNECTIONS) : 30; + + PARAMETER.minConnections = + config.hasPath(Constant.NODE_MIN_CONNECTIONS) + ? config.getInt(Constant.NODE_MIN_CONNECTIONS) : 8; + + PARAMETER.minActiveConnections = + config.hasPath(Constant.NODE_MIN_ACTIVE_CONNECTIONS) + ? config.getInt(Constant.NODE_MIN_ACTIVE_CONNECTIONS) : 3; - PARAMETER.nodeMaxActiveNodesWithSameIp = + PARAMETER.maxConnectionsWithSameIp = config.hasPath(Constant.NODE_MAX_ACTIVE_NODES_WITH_SAMEIP) ? config .getInt(Constant.NODE_MAX_ACTIVE_NODES_WITH_SAMEIP) : 2; @@ -659,17 +665,6 @@ public static void setParam(final String[] args, final String confFileName) { config.hasPath(Constant.NODE_WALLET_EXTENSION_API) && config.getBoolean(Constant.NODE_WALLET_EXTENSION_API); - PARAMETER.connectFactor = - config.hasPath(Constant.NODE_CONNECT_FACTOR) - ? config.getDouble(Constant.NODE_CONNECT_FACTOR) : 0.3; - - PARAMETER.activeConnectFactor = config.hasPath(Constant.NODE_ACTIVE_CONNECT_FACTOR) - ? config.getDouble(Constant.NODE_ACTIVE_CONNECT_FACTOR) : 0.1; - - PARAMETER.disconnectNumberFactor = config.hasPath(Constant.NODE_DISCONNECT_NUMBER_FACTOR) - ? config.getDouble(Constant.NODE_DISCONNECT_NUMBER_FACTOR) : 0.4; - PARAMETER.maxConnectNumberFactor = config.hasPath(Constant.NODE_MAX_CONNECT_NUMBER_FACTOR) - ? config.getDouble(Constant.NODE_MAX_CONNECT_NUMBER_FACTOR) : 0.8; PARAMETER.receiveTcpMinDataLength = config.hasPath(Constant.NODE_RECEIVE_TCP_MIN_DATA_LENGTH) ? config.getLong(Constant.NODE_RECEIVE_TCP_MIN_DATA_LENGTH) : 2048; PARAMETER.isOpenFullTcpDisconnect = config.hasPath(Constant.NODE_IS_OPEN_FULL_TCP_DISCONNECT) @@ -1236,8 +1231,8 @@ public static void logConfig() { logger.info("FastForward node size: {}", parameter.getFastForwardNodes().size()); logger.info("FastForward node number: {}", parameter.getMaxFastForwardNum()); logger.info("Seed node size: {}", parameter.getSeedNode().getIpList().size()); - logger.info("Max connection: {}", parameter.getNodeMaxActiveNodes()); - logger.info("Max connection with same IP: {}", parameter.getNodeMaxActiveNodesWithSameIp()); + logger.info("Max connection: {}", parameter.getMaxConnections()); + logger.info("Max connection with same IP: {}", parameter.getMaxConnectionsWithSameIp()); logger.info("Solidity threads: {}", parameter.getSolidityThreads()); logger.info("Trx reference block: {}", parameter.getTrxReferenceBlock()); logger.info("************************ Backup config ************************"); diff --git a/framework/src/main/java/org/tron/core/services/NodeInfoService.java b/framework/src/main/java/org/tron/core/services/NodeInfoService.java index acf7a29d052..79aff5b8a71 100644 --- a/framework/src/main/java/org/tron/core/services/NodeInfoService.java +++ b/framework/src/main/java/org/tron/core/services/NodeInfoService.java @@ -178,8 +178,8 @@ private void setConfigNodeInfo(NodeInfo nodeInfo) { configNodeInfo.setActiveNodeSize(parameter.getActiveNodes().size()); configNodeInfo.setPassiveNodeSize(parameter.getPassiveNodes().size()); configNodeInfo.setSendNodeSize(parameter.getSeedNodes().size()); - configNodeInfo.setMaxConnectCount(parameter.getNodeMaxActiveNodes()); - configNodeInfo.setSameIpMaxConnectCount(parameter.getNodeMaxActiveNodesWithSameIp()); + configNodeInfo.setMaxConnectCount(parameter.getMaxConnections()); + configNodeInfo.setSameIpMaxConnectCount(parameter.getMaxConnectionsWithSameIp()); configNodeInfo.setBackupListenPort(parameter.getBackupPort()); configNodeInfo.setBackupMemberSize(parameter.getBackupMembers().size()); configNodeInfo.setBackupPriority(parameter.getBackupPriority()); diff --git a/framework/src/main/resources/config-backup.conf b/framework/src/main/resources/config-backup.conf index b4d1b55551b..2a1579fdc18 100644 --- a/framework/src/main/resources/config-backup.conf +++ b/framework/src/main/resources/config-backup.conf @@ -88,8 +88,6 @@ node { # Number of validate sign thread, default availableProcessors / 2 # validateSignThreadNum = 16 - maxActiveNodes = 30 - minParticipationRate = 33 p2p { diff --git a/framework/src/main/resources/config-beta.conf b/framework/src/main/resources/config-beta.conf index 278345e6072..1d4a914ce20 100644 --- a/framework/src/main/resources/config-beta.conf +++ b/framework/src/main/resources/config-beta.conf @@ -70,9 +70,6 @@ node { connection.timeout = 2 - - maxActiveNodes = 30 - active = [ "47.93.9.236:18888", "47.93.33.201:18888", diff --git a/framework/src/main/resources/config-localtest.conf b/framework/src/main/resources/config-localtest.conf index 442c4be04b3..239b55d7b4f 100644 --- a/framework/src/main/resources/config-localtest.conf +++ b/framework/src/main/resources/config-localtest.conf @@ -94,19 +94,11 @@ node { # Number of validate sign thread, default availableProcessors / 2 # validateSignThreadNum = 16 - connectFactor = 0.3 - activeConnectFactor = 0.1 - - maxActiveNodes = 30 - - maxActiveNodesWithSameIp = 10 + maxConnectionsWithSameIp = 10 minParticipationRate = 0 # check the peer data transfer ,disconnect factor - disconnectNumberFactor = 0.4 - maxConnectNumberFactor = 0.8 - receiveTcpMinDataLength = 2048 isOpenFullTcpDisconnect = true p2p { diff --git a/framework/src/main/resources/config-test-net.conf b/framework/src/main/resources/config-test-net.conf index 3eed8149686..58c40587fcc 100644 --- a/framework/src/main/resources/config-test-net.conf +++ b/framework/src/main/resources/config-test-net.conf @@ -92,19 +92,11 @@ node { # Number of validate sign thread, default availableProcessors / 2 # validateSignThreadNum = 16 - connectFactor = 0.3 - activeConnectFactor = 0.1 - - maxActiveNodes = 30 - - maxActiveNodesWithSameIp = 2 + maxConnectionsWithSameIp = 2 minParticipationRate = 30 # check the peer data transfer ,disconnect factor - disconnectNumberFactor = 0.4 - maxConnectNumberFactor = 0.8 - receiveTcpMinDataLength = 2048 isOpenFullTcpDisconnect = true p2p { diff --git a/framework/src/main/resources/config.conf b/framework/src/main/resources/config.conf index e14680d6759..74f148e1405 100644 --- a/framework/src/main/resources/config.conf +++ b/framework/src/main/resources/config.conf @@ -94,8 +94,6 @@ storage { node.discovery = { enable = true persist = true - bind.ip = "" - external.ip = null } # custom stop condition @@ -153,24 +151,19 @@ node { udpNettyWorkThreadNum = 1 + maxConnections = 30 + minConnections = 8 + minActiveConnections = 3 + # Number of validate sign thread, default availableProcessors / 2 # validateSignThreadNum = 16 - connectFactor = 0.3 - activeConnectFactor = 0.1 - - maxActiveNodes = 30 - - maxActiveNodesWithSameIp = 2 + maxConnectionsWithSameIp = 2 maxHttpConnectNumber = 50 minParticipationRate = 15 - # check the peer data transfer ,disconnect factor - disconnectNumberFactor = 0.4 - maxConnectNumberFactor = 0.8 - receiveTcpMinDataLength = 2048 isOpenFullTcpDisconnect = true p2p { diff --git a/framework/src/test/java/org/tron/core/config/args/ArgsTest.java b/framework/src/test/java/org/tron/core/config/args/ArgsTest.java index ed79bd5c855..5768fc0b1d8 100644 --- a/framework/src/test/java/org/tron/core/config/args/ArgsTest.java +++ b/framework/src/test/java/org/tron/core/config/args/ArgsTest.java @@ -75,7 +75,7 @@ public void get() { Assert.assertEquals(18888, parameter.getNodeListenPort()); Assert.assertEquals(2000, parameter.getNodeConnectionTimeout()); Assert.assertEquals(0, parameter.getActiveNodes().size()); - Assert.assertEquals(30, parameter.getNodeMaxActiveNodes()); + Assert.assertEquals(30, parameter.getMaxConnections()); Assert.assertEquals(43, parameter.getNodeP2pVersion()); //Assert.assertEquals(30, args.getSyncNodeCount()); diff --git a/framework/src/test/resources/config-localtest.conf b/framework/src/test/resources/config-localtest.conf index 22a41d22564..280e06b4af2 100644 --- a/framework/src/test/resources/config-localtest.conf +++ b/framework/src/test/resources/config-localtest.conf @@ -88,12 +88,7 @@ node { # Number of validate sign thread, default availableProcessors / 2 # validateSignThreadNum = 16 - connectFactor = 0.3 - activeConnectFactor = 0.1 - - maxActiveNodes = 30 - - maxActiveNodesWithSameIp = 10 + maxConnectionsWithSameIp = 10 minParticipationRate = 0 @@ -102,9 +97,6 @@ node { zenTokenId = 1000001 # check the peer data transfer ,disconnect factor - disconnectNumberFactor = 0.4 - maxConnectNumberFactor = 0.8 - receiveTcpMinDataLength = 2048 isOpenFullTcpDisconnect = true p2p { diff --git a/framework/src/test/resources/config-test-dbbackup.conf b/framework/src/test/resources/config-test-dbbackup.conf index b39cb54d354..02c1aae36c2 100644 --- a/framework/src/test/resources/config-test-dbbackup.conf +++ b/framework/src/test/resources/config-test-dbbackup.conf @@ -97,19 +97,11 @@ node { # Number of validate sign thread, default availableProcessors / 2 # validateSignThreadNum = 16 - connectFactor = 0.3 - activeConnectFactor = 0.1 - - maxActiveNodes = 30 - - maxActiveNodesWithSameIp = 2 + maxConnectionsWithSameIp = 2 minParticipationRate = 15 # check the peer data transfer ,disconnect factor - disconnectNumberFactor = 0.4 - maxConnectNumberFactor = 0.8 - receiveTcpMinDataLength = 2048 isOpenFullTcpDisconnect = true p2p { diff --git a/framework/src/test/resources/config-test-index.conf b/framework/src/test/resources/config-test-index.conf index d2feabff04e..b9fbf5b2c43 100644 --- a/framework/src/test/resources/config-test-index.conf +++ b/framework/src/test/resources/config-test-index.conf @@ -73,8 +73,6 @@ node { # } ] - maxActiveNodes = 30 - p2p { version = 43 # 43: testnet; 101: debug } diff --git a/framework/src/test/resources/config-test-mainnet.conf b/framework/src/test/resources/config-test-mainnet.conf index 82d64a466ab..5bcc27b398d 100644 --- a/framework/src/test/resources/config-test-mainnet.conf +++ b/framework/src/test/resources/config-test-mainnet.conf @@ -75,7 +75,9 @@ node { # } ] - maxActiveNodes = 30 + maxConnections = 30 + minConnections = 8 + minActiveConnections = 3 p2p { version = 43 # 43: testnet; 101: debug diff --git a/framework/src/test/resources/config-test-storagetest.conf b/framework/src/test/resources/config-test-storagetest.conf index 1a551b23be3..ef8bfe906fa 100644 --- a/framework/src/test/resources/config-test-storagetest.conf +++ b/framework/src/test/resources/config-test-storagetest.conf @@ -99,7 +99,9 @@ node { # } ] - maxActiveNodes = 30 + maxConnections = 30 + minConnections = 8 + minActiveConnections = 3 p2p { version = 43 # 43: testnet; 101: debug diff --git a/framework/src/test/resources/config-test.conf b/framework/src/test/resources/config-test.conf index 4f851e84bcf..886438cac57 100644 --- a/framework/src/test/resources/config-test.conf +++ b/framework/src/test/resources/config-test.conf @@ -90,8 +90,6 @@ node { # } ] - maxActiveNodes = 30 - p2p { version = 43 # 43: testnet; 101: debug } diff --git a/protocol/src/main/protos/Chinese version of TRON Protocol document.md b/protocol/src/main/protos/Chinese version of TRON Protocol document.md index 283bd5fb8af..f393447e42e 100644 --- a/protocol/src/main/protos/Chinese version of TRON Protocol document.md +++ b/protocol/src/main/protos/Chinese version of TRON Protocol document.md @@ -442,7 +442,7 @@ `TOO_MANY_PEERS` `DUPLICATE_PEER` `INCOMPATIBLE_PROTOCOL` - `NULL_IDENTITY` + `RANDOM_ELIMINATION` `PEER_QUITING` `UNEXPECTED_IDENTITY` `LOCAL_IDENTITY` @@ -459,7 +459,7 @@ TOO_MANY_PEERS = 4; DUPLICATE_PEER = 5; INCOMPATIBLE_PROTOCOL = 6; - NULL_IDENTITY = 7; + RANDOM_ELIMINATION = 7; PEER_QUITING = 8; UNEXPECTED_IDENTITY = 9; LOCAL_IDENTITY = 10; diff --git a/protocol/src/main/protos/English version of TRON Protocol document.md b/protocol/src/main/protos/English version of TRON Protocol document.md index 87a61acd119..7d23f5c1f49 100644 --- a/protocol/src/main/protos/English version of TRON Protocol document.md +++ b/protocol/src/main/protos/English version of TRON Protocol document.md @@ -456,7 +456,7 @@ Input, transaction and head block all require signature. `TOO_MANY_PEERS` `DUPLICATE_PEER` `INCOMPATIBLE_PROTOCOL` - `NULL_IDENTITY` + `RANDOM_ELIMINATION` `PEER_QUITING` `UNEXPECTED_IDENTITY` `LOCAL_IDENTITY` @@ -473,7 +473,7 @@ Input, transaction and head block all require signature. TOO_MANY_PEERS = 4; DUPLICATE_PEER = 5; INCOMPATIBLE_PROTOCOL = 6; - NULL_IDENTITY = 7; + RANDOM_ELIMINATION = 7; PEER_QUITING = 8; UNEXPECTED_IDENTITY = 9; LOCAL_IDENTITY = 10; diff --git a/protocol/src/main/protos/core/Tron.proto b/protocol/src/main/protos/core/Tron.proto index 37079e0504d..9d4af0a3151 100644 --- a/protocol/src/main/protos/core/Tron.proto +++ b/protocol/src/main/protos/core/Tron.proto @@ -551,7 +551,7 @@ enum ReasonCode { TOO_MANY_PEERS = 0x04; DUPLICATE_PEER = 0x05; INCOMPATIBLE_PROTOCOL = 0x06; - NULL_IDENTITY = 0x07; + RANDOM_ELIMINATION = 0x07; PEER_QUITING = 0x08; UNEXPECTED_IDENTITY = 0x09; LOCAL_IDENTITY = 0x0A; From aee957f757b13fb4c302ece412ccfabf0b9ea773 Mon Sep 17 00:00:00 2001 From: halibobo1205 Date: Fri, 22 Jul 2022 14:12:30 +0800 Subject: [PATCH 09/51] tool(db):add db path move function. mv db to new path according to storage.properties settings. --- .../java/org/tron/tool/litefullnode/Util.java | 3 +- plugins/build.gradle | 11 +- .../org/tron/plugins/ArchiveManifest.java | 34 ++-- .../src/main/java/org/tron/plugins/Db.java | 25 +++ .../main/java/org/tron/plugins/DbMove.java | 163 ++++++++++++++++++ .../main/java/org/tron/plugins/Toolkit.java | 19 ++ 6 files changed, 235 insertions(+), 20 deletions(-) create mode 100644 plugins/src/main/java/org/tron/plugins/Db.java create mode 100644 plugins/src/main/java/org/tron/plugins/DbMove.java create mode 100644 plugins/src/main/java/org/tron/plugins/Toolkit.java diff --git a/framework/src/main/java/org/tron/tool/litefullnode/Util.java b/framework/src/main/java/org/tron/tool/litefullnode/Util.java index 1bb00599887..227ce5c5de4 100644 --- a/framework/src/main/java/org/tron/tool/litefullnode/Util.java +++ b/framework/src/main/java/org/tron/tool/litefullnode/Util.java @@ -2,6 +2,7 @@ import java.io.IOException; import java.nio.file.FileSystemException; +import java.nio.file.FileVisitOption; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; @@ -31,7 +32,7 @@ public static void copyDatabases(Path src, Path dest, List subDirs) subDirs.forEach(dir -> { if (FileUtil.isExists(Paths.get(src.toString(), dir).toString())) { try { - Files.walk(Paths.get(src.toString(), dir)) + Files.walk(Paths.get(src.toString(), dir), FileVisitOption.FOLLOW_LINKS) .forEach(source -> copy(source, dest.resolve(src.relativize(source)))); } catch (IOException e) { logger.error("copy database failed, src: {}, dest: {}, error: {}", diff --git a/plugins/build.gradle b/plugins/build.gradle index 494694dee59..f6815e606b3 100644 --- a/plugins/build.gradle +++ b/plugins/build.gradle @@ -30,8 +30,10 @@ dependencies { testCompile group: 'org.mockito', name: 'mockito-core', version: '2.13.0' testCompile group: 'org.hamcrest', name: 'hamcrest-junit', version: '1.0.0.1' testCompile group: 'org.testng', name: 'testng', version: '6.14.3' - // https://mvnrepository.com/artifact/com.beust/jcommander - compile group: 'com.beust', name: 'jcommander', version: '1.78' + compile group: 'info.picocli', name: 'picocli', version: '4.6.3' + compile group: 'com.typesafe', name: 'config', version: '1.3.2' + compile group: 'me.tongfei', name: 'progressbar', version: '0.9.3' + compile 'com.github.halibobo1205.leveldb-java:leveldb:v0.12.5' compile 'com.github.halibobo1205.leveldb-java:leveldb-api:v0.12.5' @@ -127,11 +129,14 @@ applicationDistribution.from("../gradle/java-tron.vmoptions") { into "bin" } createScript(project, 'org.tron.plugins.ArchiveManifest', 'ArchiveManifest') +createScript(project, 'org.tron.plugins.Toolkit', 'Toolkit') def releaseBinary = hasProperty('binaryRelease') ? getProperty('binaryRelease') : 'true' if (releaseBinary == 'true') { artifacts { - archives(binaryRelease('buildArchiveManifestJar', 'ArchiveManifest', 'org.tron.plugins.ArchiveManifest')) + archives( + binaryRelease('buildArchiveManifestJar', 'ArchiveManifest', 'org.tron.plugins.ArchiveManifest'), + binaryRelease('buildToolkitJar', 'Toolkit', 'org.tron.plugins.Toolkit')) } } diff --git a/plugins/src/main/java/org/tron/plugins/ArchiveManifest.java b/plugins/src/main/java/org/tron/plugins/ArchiveManifest.java index caf2cc0dac6..5771323138f 100644 --- a/plugins/src/main/java/org/tron/plugins/ArchiveManifest.java +++ b/plugins/src/main/java/org/tron/plugins/ArchiveManifest.java @@ -3,8 +3,6 @@ import static java.nio.charset.StandardCharsets.UTF_8; import static org.iq80.leveldb.impl.Iq80DBFactory.factory; -import com.beust.jcommander.JCommander; -import com.beust.jcommander.Parameter; import java.io.BufferedInputStream; import java.io.BufferedReader; import java.io.BufferedWriter; @@ -37,6 +35,8 @@ import org.iq80.leveldb.DB; import org.iq80.leveldb.Options; import org.iq80.leveldb.impl.Filename; +import picocli.CommandLine; +import picocli.CommandLine.Option; @Slf4j(topic = "archive") /* @@ -95,12 +95,10 @@ public static void main(String[] args) { public static int run(String[] args) { Args parameters = new Args(); - JCommander jc = JCommander.newBuilder() - .addObject(parameters) - .build(); - jc.parse(args); + CommandLine commandLine = new CommandLine(parameters); + commandLine.parseArgs(args); if (parameters.help) { - jc.usage(); + commandLine.usage(System.out); return 0; } @@ -247,19 +245,23 @@ public static boolean writeProperty(String file, String key, String value) { } public static class Args { - @Parameter - private List parameters = new ArrayList<>(); - @Parameter(names = {"-d", "--database-directory"}, description = "java-tron database directory") - private String databaseDirectory = "output-directory/database"; + @Option(names = {"-d", "--database-directory"}, + defaultValue = "output-directory/database", + description = "java-tron database directory. Default: ${DEFAULT-VALUE}") + private String databaseDirectory; - @Parameter(names = {"-b", "--batch-size" }, description = "deal manifest batch size") - private int maxBatchSize = 80_000; + @Option(names = {"-b", "--batch-size" }, + defaultValue = "80000", + description = "deal manifest batch size. Default: ${DEFAULT-VALUE}") + private int maxBatchSize; - @Parameter(names = {"-m", "--manifest-size" }, description = "manifest min size(M) to archive") - private int maxManifestSize = 0; + @Option(names = {"-m", "--manifest-size" }, + defaultValue = "0", + description = "manifest min size(M) to archive. Default: ${DEFAULT-VALUE}") + private int maxManifestSize; - @Parameter(names = {"-h", "--help"}, help = true) + @Option(names = {"-h", "--help"}, help = true) private boolean help; } } \ No newline at end of file diff --git a/plugins/src/main/java/org/tron/plugins/Db.java b/plugins/src/main/java/org/tron/plugins/Db.java new file mode 100644 index 00000000000..a886d717c6d --- /dev/null +++ b/plugins/src/main/java/org/tron/plugins/Db.java @@ -0,0 +1,25 @@ +package org.tron.plugins; + +import com.typesafe.config.Config; +import com.typesafe.config.ConfigFactory; +import java.nio.file.Paths; +import picocli.CommandLine; + +@CommandLine.Command(name = "db", + mixinStandardHelpOptions = true, + version = "db command 1.0", + description = "An rich command set that provides high-level operations for dbs.", + subcommands = {CommandLine.HelpCommand.class, DbMove.class}, + commandListHeading = "%nCommands:%n%nThe most commonly used git commands are:%n" +) +public class Db { + + static class ConfigConverter implements CommandLine.ITypeConverter { + ConfigConverter() { + } + + public Config convert(String value) { + return ConfigFactory.parseFile(Paths.get(value).toFile()); + } + } +} diff --git a/plugins/src/main/java/org/tron/plugins/DbMove.java b/plugins/src/main/java/org/tron/plugins/DbMove.java new file mode 100644 index 00000000000..67bc3a57706 --- /dev/null +++ b/plugins/src/main/java/org/tron/plugins/DbMove.java @@ -0,0 +1,163 @@ +package org.tron.plugins; + +import com.typesafe.config.Config; +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.StandardCopyOption; +import java.util.Arrays; +import java.util.List; +import java.util.Objects; +import java.util.concurrent.Callable; +import java.util.stream.Collectors; +import me.tongfei.progressbar.ProgressBar; +import picocli.CommandLine; +import picocli.CommandLine.Command; + +@Command(name = "mv", aliases = "move", + description = "mv db to pre-set new path . For example HDD,reduce storage expenses.") +public class DbMove implements Callable { + + private static final String PROPERTIES_CONFIG_KEY = "storage.properties"; + private static final String DB_DIRECTORY_CONFIG_KEY = "storage.db.directory"; + private static final String DEFAULT_DB_DIRECTORY = "database"; + private static final String NAME_CONFIG_KEY = "name"; + private static final String PATH_CONFIG_KEY = "path"; + private static final String NOT_FIND = "The database to be moved cannot be found."; + + @CommandLine.Spec + CommandLine.Model.CommandSpec spec; + + @CommandLine.Option(names = {"-d", "--database-directory"}, + defaultValue = "output-directory", + description = "database directory path. Default: ${DEFAULT-VALUE}") + String database; + + @CommandLine.Option(names = {"-c", "--config"}, + defaultValue = "config.conf", + converter = Db.ConfigConverter.class, + description = " config file. Default: ${DEFAULT-VALUE}") + Config config; + + @CommandLine.Option(names = {"-h", "--help"}, help = true, description = "display a help message") + boolean help; + + @Override + public Integer call() throws Exception { + if (help) { + spec.commandLine().usage(System.out); + return 0; + } + if (config.hasPath(PROPERTIES_CONFIG_KEY)) { + List dbs = config.getConfigList(PROPERTIES_CONFIG_KEY); + if (dbs.isEmpty()) { + spec.commandLine().getOut().println(NOT_FIND); + return 0; + } + String dbPath = config.hasPath(DB_DIRECTORY_CONFIG_KEY) + ? config.getString(DB_DIRECTORY_CONFIG_KEY) : DEFAULT_DB_DIRECTORY; + + dbs = dbs.stream() + .filter(c -> c.hasPath(NAME_CONFIG_KEY) && c.hasPath(PATH_CONFIG_KEY)) + .collect(Collectors.toList()); + + if (dbs.isEmpty()) { + spec.commandLine().getOut().println(NOT_FIND); + return 0; + } + List toBeMove = dbs.stream() + .map(c -> { + try { + return new Property(c.getString(NAME_CONFIG_KEY), + Paths.get(database, dbPath, c.getString(NAME_CONFIG_KEY)), + Paths.get(c.getString(PATH_CONFIG_KEY), dbPath, c.getString(NAME_CONFIG_KEY))); + } catch (IOException e) { + spec.commandLine().getErr().println(e); + } + return null; + }).filter(Objects::nonNull) + .filter(p -> !p.destination.equals(p.original)).collect(Collectors.toList()); + + if (toBeMove.isEmpty()) { + spec.commandLine().getOut().println(NOT_FIND); + return 0; + } + toBeMove = toBeMove.stream() + .filter(property -> { + if (property.destination.toFile().exists()) { + spec.commandLine().getOut().println(String.format("%s already exist,skip.", + property.destination)); + return false; + } else { + return true; + } + }).collect(Collectors.toList()); + + if (toBeMove.isEmpty()) { + spec.commandLine().getOut().println(NOT_FIND); + return 0; + } + ProgressBar.wrap(toBeMove.stream(), "mv task").forEach(this::run); + spec.commandLine().getOut().println("move db done."); + + } else { + spec.commandLine().getOut().println(NOT_FIND); + } + return 0; + } + + private void run(Property p) { + if (p.destination.toFile().mkdirs()) { + ProgressBar.wrap(Arrays.stream(Objects.requireNonNull(p.original.toFile().listFiles())) + .filter(File::isFile).map(File::getName).parallel(), p.name).forEach(file -> { + Path original = Paths.get(p.original.toString(), file); + Path destination = Paths.get(p.destination.toString(), file); + try { + Files.copy(original, destination, + StandardCopyOption.REPLACE_EXISTING); + } catch (IOException e) { + spec.commandLine().getErr().println(e); + } + }); + try { + if (deleteDir(p.original.toFile())) { + Files.createSymbolicLink(p.original, p.destination); + } + } catch (IOException | UnsupportedOperationException x) { + spec.commandLine().getErr().println(x); + } + } else { + spec.commandLine().getErr().println(String.format("%s create failed.", p.destination)); + } + } + + /** + * delete directory. + */ + public static boolean deleteDir(File dir) { + if (dir.isDirectory()) { + String[] children = dir.list(); + if (children != null) { + for (String child : children) { + deleteDir(new File(dir, child)); + } + } + } + return dir.delete(); + } + + static class Property { + + private final String name; + private final Path original; + final Path destination; + + public Property(String name, Path original, Path destination) throws IOException { + this.name = name; + this.original = original.toFile().getCanonicalFile().toPath(); + this.destination = destination.toFile().getCanonicalFile().toPath(); + } + } +} diff --git a/plugins/src/main/java/org/tron/plugins/Toolkit.java b/plugins/src/main/java/org/tron/plugins/Toolkit.java new file mode 100644 index 00000000000..4a087e40b34 --- /dev/null +++ b/plugins/src/main/java/org/tron/plugins/Toolkit.java @@ -0,0 +1,19 @@ +package org.tron.plugins; + +import java.util.concurrent.Callable; +import picocli.CommandLine; + +@CommandLine.Command(name = "tron", subcommands = { CommandLine.HelpCommand.class, Db.class}) +public class Toolkit implements Callable { + + + public static void main(String[] args) { + int exitCode = new CommandLine(new Toolkit()).execute(args); + System.exit(exitCode); + } + + @Override + public Integer call() throws Exception { + return 0; + } +} From 72375d70fcd6a1cc6fc118f026ece571cff4e420 Mon Sep 17 00:00:00 2001 From: Wenhua Zhang Date: Tue, 26 Jul 2022 12:04:58 +0800 Subject: [PATCH 10/51] feat: catch error if param address is not base58 --- .../src/main/java/org/tron/core/services/http/JsonFormat.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/framework/src/main/java/org/tron/core/services/http/JsonFormat.java b/framework/src/main/java/org/tron/core/services/http/JsonFormat.java index d74a6c8f291..96dedb1e20c 100644 --- a/framework/src/main/java/org/tron/core/services/http/JsonFormat.java +++ b/framework/src/main/java/org/tron/core/services/http/JsonFormat.java @@ -1595,6 +1595,8 @@ public ByteString consumeByteString(final String fieldName, boolean selfType) return result; } catch (InvalidEscapeSequence e) { throw parseException(e.getMessage()); + } catch (IllegalArgumentException e) { + throw parseException("INVALID base58 String, " + e.getMessage()); } } From a50938368bacf09cd4071673d36671f0e42f429d Mon Sep 17 00:00:00 2001 From: chengtx01 Date: Tue, 26 Jul 2022 16:10:55 +0800 Subject: [PATCH 11/51] feat(block): increase the probability that the block processing thread acquires the lock --- .../org/tron/consensus/base/BlockHandle.java | 2 + .../org/tron/consensus/dpos/DposTask.java | 55 +-- .../tron/core/consensus/BlockHandleImpl.java | 8 + .../main/java/org/tron/core/db/Manager.java | 358 ++++++++++-------- 4 files changed, 236 insertions(+), 187 deletions(-) diff --git a/consensus/src/main/java/org/tron/consensus/base/BlockHandle.java b/consensus/src/main/java/org/tron/consensus/base/BlockHandle.java index a36722b9083..f1887359f44 100644 --- a/consensus/src/main/java/org/tron/consensus/base/BlockHandle.java +++ b/consensus/src/main/java/org/tron/consensus/base/BlockHandle.java @@ -11,4 +11,6 @@ public interface BlockHandle { BlockCapsule produce(Miner miner, long blockTime, long timeout); + void setBlockWaitLock(boolean flag); + } \ No newline at end of file diff --git a/consensus/src/main/java/org/tron/consensus/dpos/DposTask.java b/consensus/src/main/java/org/tron/consensus/dpos/DposTask.java index 4486aad2d89..dcfa85ca5f3 100644 --- a/consensus/src/main/java/org/tron/consensus/dpos/DposTask.java +++ b/consensus/src/main/java/org/tron/consensus/dpos/DposTask.java @@ -87,36 +87,41 @@ private State produceBlock() { return state; } - synchronized (dposService.getBlockHandle().getLock()) { + dposService.getBlockHandle().setBlockWaitLock(true); + try { + synchronized (dposService.getBlockHandle().getLock()) { - long slot = dposSlot.getSlot(System.currentTimeMillis() + 50); - if (slot == 0) { - return State.NOT_TIME_YET; - } + long slot = dposSlot.getSlot(System.currentTimeMillis() + 50); + if (slot == 0) { + return State.NOT_TIME_YET; + } - ByteString pWitness = dposSlot.getScheduledWitness(slot); + ByteString pWitness = dposSlot.getScheduledWitness(slot); - Miner miner = dposService.getMiners().get(pWitness); - if (miner == null) { - return State.NOT_MY_TURN; - } + Miner miner = dposService.getMiners().get(pWitness); + if (miner == null) { + return State.NOT_MY_TURN; + } - long pTime = dposSlot.getTime(slot); - long timeout = - pTime + BLOCK_PRODUCED_INTERVAL / 2 * dposService.getBlockProduceTimeoutPercent() / 100; - BlockCapsule blockCapsule = dposService.getBlockHandle().produce(miner, pTime, timeout); - if (blockCapsule == null) { - return State.PRODUCE_BLOCK_FAILED; - } + long pTime = dposSlot.getTime(slot); + long timeout = + pTime + BLOCK_PRODUCED_INTERVAL / 2 * dposService.getBlockProduceTimeoutPercent() / 100; + BlockCapsule blockCapsule = dposService.getBlockHandle().produce(miner, pTime, timeout); + if (blockCapsule == null) { + return State.PRODUCE_BLOCK_FAILED; + } - BlockHeader.raw raw = blockCapsule.getInstance().getBlockHeader().getRawData(); - logger.info("Produce block successfully, num: {}, time: {}, witness: {}, ID:{}, parentID:{}", - raw.getNumber(), - new DateTime(raw.getTimestamp()), - ByteArray.toHexString(raw.getWitnessAddress().toByteArray()), - new Sha256Hash(raw.getNumber(), Sha256Hash.of(CommonParameter - .getInstance().isECKeyCryptoEngine(), raw.toByteArray())), - ByteArray.toHexString(raw.getParentHash().toByteArray())); + BlockHeader.raw raw = blockCapsule.getInstance().getBlockHeader().getRawData(); + logger.info("Produce block successfully, num: {}, time: {}, witness: {}, ID:{}, parentID:{}", + raw.getNumber(), + new DateTime(raw.getTimestamp()), + ByteArray.toHexString(raw.getWitnessAddress().toByteArray()), + new Sha256Hash(raw.getNumber(), Sha256Hash.of(CommonParameter + .getInstance().isECKeyCryptoEngine(), raw.toByteArray())), + ByteArray.toHexString(raw.getParentHash().toByteArray())); + } + } finally { + dposService.getBlockHandle().setBlockWaitLock(false); } return State.OK; diff --git a/framework/src/main/java/org/tron/core/consensus/BlockHandleImpl.java b/framework/src/main/java/org/tron/core/consensus/BlockHandleImpl.java index 9c039800ff9..03452546a9b 100644 --- a/framework/src/main/java/org/tron/core/consensus/BlockHandleImpl.java +++ b/framework/src/main/java/org/tron/core/consensus/BlockHandleImpl.java @@ -58,4 +58,12 @@ public BlockCapsule produce(Miner miner, long blockTime, long timeout) { } return blockCapsule; } + + public void setBlockWaitLock(boolean flag) { + if (flag) { + manager.getBlockWaitLock().incrementAndGet(); + } else { + manager.getBlockWaitLock().decrementAndGet(); + } + } } diff --git a/framework/src/main/java/org/tron/core/db/Manager.java b/framework/src/main/java/org/tron/core/db/Manager.java index 2d0f85a94d0..59fcd6f1ee6 100644 --- a/framework/src/main/java/org/tron/core/db/Manager.java +++ b/framework/src/main/java/org/tron/core/db/Manager.java @@ -239,6 +239,10 @@ public class Manager { @Getter private final ThreadLocal blockedTimer = new ThreadLocal<>(); + @Getter + private AtomicInteger blockWaitLock = new AtomicInteger(0); + private Object transactionLock = new Object(); + /** * Cycle thread to rePush Transactions */ @@ -755,25 +759,40 @@ public boolean pushTransaction(final TransactionCapsule trx) throw new ValidateSignatureException("trans sig validate failed"); } - synchronized (this) { - if (isShieldedTransaction(trx.getInstance()) - && shieldedTransInPendingCounts.get() >= shieldedTransInPendingMaxCounts) { - return false; - } - if (!session.valid()) { - session.setValue(revokingStore.buildSession()); + synchronized (transactionLock) { + while (true) { + try { + if (blockWaitLock.get() > 0) { + TimeUnit.MILLISECONDS.sleep(50); + logger.info("waiting for the block processing to complete"); + } else { + break; + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + logger.debug("the wait has been interrupted"); + } } + synchronized (this) { + if (isShieldedTransaction(trx.getInstance()) + && shieldedTransInPendingCounts.get() >= shieldedTransInPendingMaxCounts) { + return false; + } + if (!session.valid()) { + session.setValue(revokingStore.buildSession()); + } - try (ISession tmpSession = revokingStore.buildSession()) { - processTransaction(trx, null); - trx.setTrxTrace(null); - pendingTransactions.add(trx); - Metrics.gaugeInc(MetricKeys.Gauge.MANAGER_QUEUE, 1, - MetricLabels.Gauge.QUEUE_PENDING); - tmpSession.merge(); - } - if (isShieldedTransaction(trx.getInstance())) { - shieldedTransInPendingCounts.incrementAndGet(); + try (ISession tmpSession = revokingStore.buildSession()) { + processTransaction(trx, null); + trx.setTrxTrace(null); + pendingTransactions.add(trx); + Metrics.gaugeInc(MetricKeys.Gauge.MANAGER_QUEUE, 1, + MetricLabels.Gauge.QUEUE_PENDING); + tmpSession.merge(); + } + if (isShieldedTransaction(trx.getInstance())) { + shieldedTransInPendingCounts.incrementAndGet(); + } } } } finally { @@ -825,7 +844,7 @@ public void consumeBandwidth(TransactionCapsule trx, TransactionTrace trace) /** * when switch fork need erase blocks on fork branch. */ - public synchronized void eraseBlock() { + public void eraseBlock() { session.reset(); try { BlockCapsule oldHeadBlock = chainBaseManager.getBlockById( @@ -1048,167 +1067,182 @@ public List getVerifyTxs(BlockCapsule block) { /** * save a block. */ - public synchronized void pushBlock(final BlockCapsule block) + public void pushBlock(final BlockCapsule block) throws ValidateSignatureException, ContractValidateException, ContractExeException, UnLinkedBlockException, ValidateScheduleException, AccountResourceInsufficientException, TaposException, TooBigTransactionException, TooBigTransactionResultException, DupTransactionException, TransactionExpirationException, BadNumberBlockException, BadBlockException, NonCommonBlockException, ReceiptCheckErrException, VMIllegalException, ZksnarkException, EventBloomException { - Metrics.histogramObserve(blockedTimer.get()); - blockedTimer.remove(); - long headerNumber = getDynamicPropertiesStore().getLatestBlockHeaderNumber(); - if (block.getNum() <= headerNumber && khaosDb.containBlockInMiniStore(block.getBlockId())) { - logger.info("Block {} is already exist.", block.getBlockId().getString()); - return; - } - final Histogram.Timer timer = Metrics.histogramStartTimer( - MetricKeys.Histogram.BLOCK_PUSH_LATENCY); - long start = System.currentTimeMillis(); - List txs = getVerifyTxs(block); - logger.info("Block num: {}, re-push-size: {}, pending-size: {}, " - + "block-tx-size: {}, verify-tx-size: {}", - block.getNum(), rePushTransactions.size(), pendingTransactions.size(), - block.getTransactions().size(), txs.size()); - - if (CommonParameter.getInstance().getShutdownBlockTime() != null - && CommonParameter.getInstance().getShutdownBlockTime() - .isSatisfiedBy(new Date(block.getTimeStamp()))) { - latestSolidityNumShutDown = block.getNum(); - } - - try (PendingManager pm = new PendingManager(this)) { - - if (!block.generatedByMyself) { - if (!block.calcMerkleRoot().equals(block.getMerkleRoot())) { - logger.warn( - "The merkle root doesn't match, Calc result is " - + block.calcMerkleRoot() - + " , the headers is " - + block.getMerkleRoot()); - throw new BadBlockException("The merkle hash is not validated"); - } - consensus.receiveBlock(block); - } - - if (block.getTransactions().stream().filter(tran -> isShieldedTransaction(tran.getInstance())) - .count() > SHIELDED_TRANS_IN_BLOCK_COUNTS) { - throw new BadBlockException( - "shielded transaction count > " + SHIELDED_TRANS_IN_BLOCK_COUNTS); - } - - BlockCapsule newBlock; - try { - newBlock = this.khaosDb.push(block); - } catch (UnLinkedBlockException e) { - logger.error( - "latestBlockHeaderHash:{}, latestBlockHeaderNumber:{}, latestSolidifiedBlockNum:{}", - getDynamicPropertiesStore().getLatestBlockHeaderHash(), - getDynamicPropertiesStore().getLatestBlockHeaderNumber(), - getDynamicPropertiesStore().getLatestSolidifiedBlockNum()); - throw e; - } - - // DB don't need lower block - if (getDynamicPropertiesStore().getLatestBlockHeaderHash() == null) { - if (newBlock.getNum() != 0) { + blockWaitLock.incrementAndGet(); + try { + synchronized (this) { + Metrics.histogramObserve(blockedTimer.get()); + blockedTimer.remove(); + long headerNumber = getDynamicPropertiesStore().getLatestBlockHeaderNumber(); + if (block.getNum() <= headerNumber && khaosDb.containBlockInMiniStore(block.getBlockId())) { + logger.info("Block {} is already exist.", block.getBlockId().getString()); return; } - } else { - if (newBlock.getNum() <= headerNumber) { - return; + final Histogram.Timer timer = Metrics.histogramStartTimer( + MetricKeys.Histogram.BLOCK_PUSH_LATENCY); + long start = System.currentTimeMillis(); + List txs = getVerifyTxs(block); + logger.info("Block num: {}, re-push-size: {}, pending-size: {}, " + + "block-tx-size: {}, verify-tx-size: {}", + block.getNum(), rePushTransactions.size(), pendingTransactions.size(), + block.getTransactions().size(), txs.size()); + + if (CommonParameter.getInstance().getShutdownBlockTime() != null + && CommonParameter.getInstance().getShutdownBlockTime() + .isSatisfiedBy(new Date(block.getTimeStamp()))) { + latestSolidityNumShutDown = block.getNum(); } - // switch fork - if (!newBlock - .getParentHash() - .equals(getDynamicPropertiesStore().getLatestBlockHeaderHash())) { - logger.warn( - "switch fork! new head num = {}, block id = {}", - newBlock.getNum(), - newBlock.getBlockId()); - - logger.warn( - "******** before switchFork ******* push block: " - + block.toString() - + ", new block:" - + newBlock.toString() - + ", dynamic head num: " - + chainBaseManager.getDynamicPropertiesStore().getLatestBlockHeaderNumber() - + ", dynamic head hash: " - + chainBaseManager.getDynamicPropertiesStore().getLatestBlockHeaderHash() - + ", dynamic head timestamp: " - + chainBaseManager.getDynamicPropertiesStore().getLatestBlockHeaderTimestamp() - + ", khaosDb head: " - + khaosDb.getHead() - + ", khaosDb miniStore size: " - + khaosDb.getMiniStore().size() - + ", khaosDb unlinkMiniStore size: " - + khaosDb.getMiniUnlinkedStore().size()); - - switchFork(newBlock); - logger.info(SAVE_BLOCK + newBlock); + try (PendingManager pm = new PendingManager(this)) { - logger.warn( - "******** after switchFork ******* push block: " - + block.toString() - + ", new block:" - + newBlock.toString() - + ", dynamic head num: " - + chainBaseManager.getDynamicPropertiesStore().getLatestBlockHeaderNumber() - + ", dynamic head hash: " - + chainBaseManager.getDynamicPropertiesStore().getLatestBlockHeaderHash() - + ", dynamic head timestamp: " - + chainBaseManager.getDynamicPropertiesStore().getLatestBlockHeaderTimestamp() - + ", khaosDb head: " - + khaosDb.getHead() - + ", khaosDb miniStore size: " - + khaosDb.getMiniStore().size() - + ", khaosDb unlinkMiniStore size: " - + khaosDb.getMiniUnlinkedStore().size()); + if (!block.generatedByMyself) { + if (!block.calcMerkleRoot().equals(block.getMerkleRoot())) { + logger.warn( + "The merkle root doesn't match, Calc result is " + + block.calcMerkleRoot() + + " , the headers is " + + block.getMerkleRoot()); + throw new BadBlockException("The merkle hash is not validated"); + } + consensus.receiveBlock(block); + } - return; - } - try (ISession tmpSession = revokingStore.buildSession()) { + if (block.getTransactions().stream() + .filter(tran -> isShieldedTransaction(tran.getInstance())) + .count() > SHIELDED_TRANS_IN_BLOCK_COUNTS) { + throw new BadBlockException( + "shielded transaction count > " + SHIELDED_TRANS_IN_BLOCK_COUNTS); + } - long oldSolidNum = - chainBaseManager.getDynamicPropertiesStore().getLatestSolidifiedBlockNum(); + BlockCapsule newBlock; + try { + newBlock = this.khaosDb.push(block); + } catch (UnLinkedBlockException e) { + logger.error( + "latestBlockHeaderHash:{}, latestBlockHeaderNumber:{}" + + ", latestSolidifiedBlockNum:{}", + getDynamicPropertiesStore().getLatestBlockHeaderHash(), + getDynamicPropertiesStore().getLatestBlockHeaderNumber(), + getDynamicPropertiesStore().getLatestSolidifiedBlockNum()); + throw e; + } - applyBlock(newBlock, txs); - tmpSession.commit(); - // if event subscribe is enabled, post block trigger to queue - postBlockTrigger(newBlock); - // if event subscribe is enabled, post solidity trigger to queue - postSolidityTrigger(oldSolidNum, - getDynamicPropertiesStore().getLatestSolidifiedBlockNum()); - } catch (Throwable throwable) { - logger.error(throwable.getMessage(), throwable); - khaosDb.removeBlk(block.getBlockId()); - throw throwable; + // DB don't need lower block + if (getDynamicPropertiesStore().getLatestBlockHeaderHash() == null) { + if (newBlock.getNum() != 0) { + return; + } + } else { + if (newBlock.getNum() <= headerNumber) { + return; + } + + // switch fork + if (!newBlock + .getParentHash() + .equals(getDynamicPropertiesStore().getLatestBlockHeaderHash())) { + logger.warn( + "switch fork! new head num = {}, block id = {}", + newBlock.getNum(), + newBlock.getBlockId()); + + logger.warn( + "******** before switchFork ******* push block: " + + block.toString() + + ", new block:" + + newBlock.toString() + + ", dynamic head num: " + + chainBaseManager.getDynamicPropertiesStore() + .getLatestBlockHeaderNumber() + + ", dynamic head hash: " + + chainBaseManager.getDynamicPropertiesStore() + .getLatestBlockHeaderHash() + + ", dynamic head timestamp: " + + chainBaseManager.getDynamicPropertiesStore() + .getLatestBlockHeaderTimestamp() + + ", khaosDb head: " + + khaosDb.getHead() + + ", khaosDb miniStore size: " + + khaosDb.getMiniStore().size() + + ", khaosDb unlinkMiniStore size: " + + khaosDb.getMiniUnlinkedStore().size()); + + switchFork(newBlock); + logger.info(SAVE_BLOCK + newBlock); + + logger.warn( + "******** after switchFork ******* push block: " + + block.toString() + + ", new block:" + + newBlock.toString() + + ", dynamic head num: " + + chainBaseManager.getDynamicPropertiesStore() + .getLatestBlockHeaderNumber() + + ", dynamic head hash: " + + chainBaseManager.getDynamicPropertiesStore() + .getLatestBlockHeaderHash() + + ", dynamic head timestamp: " + + chainBaseManager.getDynamicPropertiesStore() + .getLatestBlockHeaderTimestamp() + + ", khaosDb head: " + + khaosDb.getHead() + + ", khaosDb miniStore size: " + + khaosDb.getMiniStore().size() + + ", khaosDb unlinkMiniStore size: " + + khaosDb.getMiniUnlinkedStore().size()); + + return; + } + try (ISession tmpSession = revokingStore.buildSession()) { + + long oldSolidNum = + chainBaseManager.getDynamicPropertiesStore().getLatestSolidifiedBlockNum(); + + applyBlock(newBlock, txs); + tmpSession.commit(); + // if event subscribe is enabled, post block trigger to queue + postBlockTrigger(newBlock); + // if event subscribe is enabled, post solidity trigger to queue + postSolidityTrigger(oldSolidNum, + getDynamicPropertiesStore().getLatestSolidifiedBlockNum()); + } catch (Throwable throwable) { + logger.error(throwable.getMessage(), throwable); + khaosDb.removeBlk(block.getBlockId()); + throw throwable; + } + } + logger.info(SAVE_BLOCK + newBlock); + } + //clear ownerAddressSet + if (CollectionUtils.isNotEmpty(ownerAddressSet)) { + Set result = new HashSet<>(); + for (TransactionCapsule transactionCapsule : rePushTransactions) { + filterOwnerAddress(transactionCapsule, result); + } + for (TransactionCapsule transactionCapsule : pushTransactionQueue) { + filterOwnerAddress(transactionCapsule, result); + } + ownerAddressSet.clear(); + ownerAddressSet.addAll(result); } - } - logger.info(SAVE_BLOCK + newBlock); - } - //clear ownerAddressSet - if (CollectionUtils.isNotEmpty(ownerAddressSet)) { - Set result = new HashSet<>(); - for (TransactionCapsule transactionCapsule : rePushTransactions) { - filterOwnerAddress(transactionCapsule, result); - } - for (TransactionCapsule transactionCapsule : pushTransactionQueue) { - filterOwnerAddress(transactionCapsule, result); - } - ownerAddressSet.clear(); - ownerAddressSet.addAll(result); - } - long cost = System.currentTimeMillis() - start; - MetricsUtil.meterMark(MetricsKey.BLOCKCHAIN_BLOCK_PROCESS_TIME, cost); + long cost = System.currentTimeMillis() - start; + MetricsUtil.meterMark(MetricsKey.BLOCKCHAIN_BLOCK_PROCESS_TIME, cost); - logger.info("pushBlock block number:{}, cost/txs:{}/{} {}", - block.getNum(), cost, block.getTransactions().size(), cost > 1000); + logger.info("pushBlock block number:{}, cost/txs:{}/{} {}", + block.getNum(), cost, block.getTransactions().size(), cost > 1000); - Metrics.histogramObserve(timer); + Metrics.histogramObserve(timer); + } + } finally { + blockWaitLock.decrementAndGet(); + } } public void updateDynamicProperties(BlockCapsule block) { @@ -1373,7 +1407,7 @@ public TransactionInfo processTransaction(final TransactionCapsule trxCap, Block /** * Generate a block. */ - public synchronized BlockCapsule generateBlock(Miner miner, long blockTime, long timeout) { + public BlockCapsule generateBlock(Miner miner, long blockTime, long timeout) { String address = StringUtil.encode58Check(miner.getWitnessAddress().toByteArray()); final Histogram.Timer timer = Metrics.histogramStartTimer( MetricKeys.Histogram.BLOCK_GENERATE_LATENCY, address); From 9445506df442124179e9b9e74fc9e1570bdff63d Mon Sep 17 00:00:00 2001 From: Wenhua Zhang Date: Tue, 26 Jul 2022 16:24:26 +0800 Subject: [PATCH 12/51] feat: catch ClassCastException when getJSONObject --- framework/src/main/java/org/tron/core/services/http/Util.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/framework/src/main/java/org/tron/core/services/http/Util.java b/framework/src/main/java/org/tron/core/services/http/Util.java index 49b50e3356b..1a4f738a334 100644 --- a/framework/src/main/java/org/tron/core/services/http/Util.java +++ b/framework/src/main/java/org/tron/core/services/http/Util.java @@ -252,7 +252,7 @@ public static JSONObject printTransactionToJSON(Transaction transaction, boolean /** * Note: the contracts of the returned transaction may be empty - * */ + */ public static Transaction packTransaction(String strTransaction, boolean selfType) { JSONObject jsonTransaction = JSON.parseObject(strTransaction); JSONObject rawData = jsonTransaction.getJSONObject("raw_data"); @@ -290,6 +290,8 @@ public static Transaction packTransaction(String strTransaction, boolean selfTyp logger.debug("invalid contractType: {}", contractType); } catch (ParseException e) { logger.debug("ParseException: {}", e.getMessage()); + } catch (ClassCastException e) { + logger.debug("ClassCastException: {}", e.getMessage()); } catch (Exception e) { logger.error("", e); } From ba09cbdf6c64ee51e01957c7fec1d371b57e7732 Mon Sep 17 00:00:00 2001 From: chengtx01 Date: Tue, 26 Jul 2022 17:46:29 +0800 Subject: [PATCH 13/51] feat(block): remove test log --- framework/src/main/java/org/tron/core/db/Manager.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/framework/src/main/java/org/tron/core/db/Manager.java b/framework/src/main/java/org/tron/core/db/Manager.java index 59fcd6f1ee6..ad1faa2faf6 100644 --- a/framework/src/main/java/org/tron/core/db/Manager.java +++ b/framework/src/main/java/org/tron/core/db/Manager.java @@ -763,8 +763,7 @@ public boolean pushTransaction(final TransactionCapsule trx) while (true) { try { if (blockWaitLock.get() > 0) { - TimeUnit.MILLISECONDS.sleep(50); - logger.info("waiting for the block processing to complete"); + TimeUnit.MILLISECONDS.sleep(10); } else { break; } From 3889e04801678b1ee9809eb6a54407d93b08f9b5 Mon Sep 17 00:00:00 2001 From: Wenhua Zhang Date: Tue, 26 Jul 2022 18:49:38 +0800 Subject: [PATCH 14/51] feat: change implementation of web3_clientVersion, remove VERSION_NAME --- .../org/tron/core/services/jsonrpc/TronJsonRpcImpl.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/framework/src/main/java/org/tron/core/services/jsonrpc/TronJsonRpcImpl.java b/framework/src/main/java/org/tron/core/services/jsonrpc/TronJsonRpcImpl.java index 0fec85718fa..1c6debb31b7 100644 --- a/framework/src/main/java/org/tron/core/services/jsonrpc/TronJsonRpcImpl.java +++ b/framework/src/main/java/org/tron/core/services/jsonrpc/TronJsonRpcImpl.java @@ -215,10 +215,10 @@ public String web3ClientVersion() { matcher.matches(); return String.join("/", Arrays.asList( - "TRON", "v" + Version.getVersion(), + "TRON", + "v" + Version.getVersion(), System.getProperty("os.name"), - "Java" + matcher.group(1), - Version.VERSION_NAME)); + "Java" + matcher.group(1))); } @Override From e8478b1c20aac98a9f11a5c8b088010fa01a5472 Mon Sep 17 00:00:00 2001 From: halibobo1205 Date: Tue, 26 Jul 2022 17:21:04 +0800 Subject: [PATCH 15/51] api(getblock): rename params --- framework/src/main/java/org/tron/core/Wallet.java | 5 ++++- .../java/org/tron/core/services/RpcApiService.java | 6 +++--- .../tron/core/services/http/GetBlockServlet.java | 14 +++++++------- .../interfaceOnPBFT/RpcApiServiceOnPBFT.java | 2 +- .../RpcApiServiceOnSolidity.java | 2 +- protocol/src/main/protos/api/api.proto | 8 ++++---- 6 files changed, 20 insertions(+), 17 deletions(-) diff --git a/framework/src/main/java/org/tron/core/Wallet.java b/framework/src/main/java/org/tron/core/Wallet.java index 29c61082681..420e87fad40 100755 --- a/framework/src/main/java/org/tron/core/Wallet.java +++ b/framework/src/main/java/org/tron/core/Wallet.java @@ -4020,7 +4020,7 @@ public Chainbase.Cursor getCursor() { return chainBaseManager.getBlockStore().getRevokingDB().getCursor(); } - public Block getBlock(GrpcAPI.BlockMessage request) { + public Block getBlock(GrpcAPI.BlockReq request) { Block block; long head = chainBaseManager.getHeadBlockNum(); if (!request.getIdOrNum().isEmpty()) { @@ -4036,6 +4036,9 @@ public Block getBlock(GrpcAPI.BlockMessage request) { block = getBlockByNum(num); } else { RuntimeException e = new IllegalArgumentException("id must be legal block hash."); + if (request.getIdOrNum().length() != Sha256Hash.LENGTH * 2) { + throw e; + } try { ByteString id = ByteString.copyFrom(ByteArray.fromHexString(request.getIdOrNum())); if (id.size() == Sha256Hash.LENGTH) { diff --git a/framework/src/main/java/org/tron/core/services/RpcApiService.java b/framework/src/main/java/org/tron/core/services/RpcApiService.java index ac791428999..c964536aa33 100755 --- a/framework/src/main/java/org/tron/core/services/RpcApiService.java +++ b/framework/src/main/java/org/tron/core/services/RpcApiService.java @@ -909,7 +909,7 @@ public void getTransactionInfoByBlockNum(NumberMessage request, } @Override - public void getBlock(GrpcAPI.BlockMessage request, + public void getBlock(GrpcAPI.BlockReq request, StreamObserver responseObserver) { getBlockCommon(request, responseObserver); } @@ -2666,7 +2666,7 @@ public void getPendingSize(EmptyMessage request, @Override - public void getBlock(GrpcAPI.BlockMessage request, + public void getBlock(GrpcAPI.BlockReq request, StreamObserver responseObserver) { getBlockCommon(request, responseObserver); } @@ -2788,7 +2788,7 @@ public void getPendingSizeCommon(EmptyMessage request, responseObserver.onCompleted(); } - public void getBlockCommon(GrpcAPI.BlockMessage request, + public void getBlockCommon(GrpcAPI.BlockReq request, StreamObserver responseObserver) { try { responseObserver.onNext(block2Extention(wallet.getBlock(request))); diff --git a/framework/src/main/java/org/tron/core/services/http/GetBlockServlet.java b/framework/src/main/java/org/tron/core/services/http/GetBlockServlet.java index 4856946a8ed..0e0104e8014 100644 --- a/framework/src/main/java/org/tron/core/services/http/GetBlockServlet.java +++ b/framework/src/main/java/org/tron/core/services/http/GetBlockServlet.java @@ -10,7 +10,7 @@ import org.eclipse.jetty.http.HttpMethod; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; -import org.tron.api.GrpcAPI.BlockMessage; +import org.tron.api.GrpcAPI.BlockReq; import org.tron.core.Wallet; import org.tron.protos.Protocol.Block; @@ -33,7 +33,7 @@ protected void doPost(HttpServletRequest request, HttpServletResponse response) private void handle(HttpServletRequest request, HttpServletResponse response) { try { PostParams params = parseParams(request); - BlockMessage message = buildRequest(params.getParams(), params.isVisible()); + BlockReq message = buildRequest(params.getParams(), params.isVisible()); fillResponse(params.isVisible(), message, response); } catch (Exception e) { Util.processError(e, response); @@ -43,10 +43,10 @@ private void handle(HttpServletRequest request, HttpServletResponse response) { private PostParams parseParams(HttpServletRequest request) throws Exception { HttpMethod m = HttpMethod.fromString(request.getMethod()); if (HttpMethod.GET.equals(m)) { - String idOrNum = request.getParameter("idOrNum"); + String idOrNum = request.getParameter("id_or_num"); JSONObject params = new JSONObject(); if (!Strings.isNullOrEmpty(idOrNum)) { - params.put("idOrNum", idOrNum); + params.put("id_or_num", idOrNum); } params.put("detail", Boolean.parseBoolean(request.getParameter("detail"))); return new PostParams(JSON.toJSONString(params), @@ -58,16 +58,16 @@ private PostParams parseParams(HttpServletRequest request) throws Exception { throw new UnsupportedOperationException(); } - private BlockMessage buildRequest(String params, boolean visible) + private BlockReq buildRequest(String params, boolean visible) throws JsonFormat.ParseException { - BlockMessage.Builder build = BlockMessage.newBuilder(); + BlockReq.Builder build = BlockReq.newBuilder(); if (!Strings.isNullOrEmpty(params)) { JsonFormat.merge(params, build, visible); } return build.build(); } - private void fillResponse(boolean visible, BlockMessage request, HttpServletResponse response) + private void fillResponse(boolean visible, BlockReq request, HttpServletResponse response) throws IOException { try { Block reply = wallet.getBlock(request); diff --git a/framework/src/main/java/org/tron/core/services/interfaceOnPBFT/RpcApiServiceOnPBFT.java b/framework/src/main/java/org/tron/core/services/interfaceOnPBFT/RpcApiServiceOnPBFT.java index b92cd4d3041..286613b14b9 100755 --- a/framework/src/main/java/org/tron/core/services/interfaceOnPBFT/RpcApiServiceOnPBFT.java +++ b/framework/src/main/java/org/tron/core/services/interfaceOnPBFT/RpcApiServiceOnPBFT.java @@ -504,7 +504,7 @@ public void getBurnTrx(EmptyMessage request, StreamObserver respo } @Override - public void getBlock(GrpcAPI.BlockMessage request, + public void getBlock(GrpcAPI.BlockReq request, StreamObserver responseObserver) { walletOnPBFT.futureGet( () -> rpcApiService.getWalletSolidityApi().getBlock(request, responseObserver)); diff --git a/framework/src/main/java/org/tron/core/services/interfaceOnSolidity/RpcApiServiceOnSolidity.java b/framework/src/main/java/org/tron/core/services/interfaceOnSolidity/RpcApiServiceOnSolidity.java index 1e37fdb0e8f..4d9d0361d23 100755 --- a/framework/src/main/java/org/tron/core/services/interfaceOnSolidity/RpcApiServiceOnSolidity.java +++ b/framework/src/main/java/org/tron/core/services/interfaceOnSolidity/RpcApiServiceOnSolidity.java @@ -500,7 +500,7 @@ public void getBurnTrx(EmptyMessage request, StreamObserver respo } @Override - public void getBlock(GrpcAPI.BlockMessage request, + public void getBlock(GrpcAPI.BlockReq request, StreamObserver responseObserver) { walletOnSolidity.futureGet( () -> rpcApiService.getWalletSolidityApi().getBlock(request, responseObserver)); diff --git a/protocol/src/main/protos/api/api.proto b/protocol/src/main/protos/api/api.proto index 63eca1a4a29..94219db1ec2 100644 --- a/protocol/src/main/protos/api/api.proto +++ b/protocol/src/main/protos/api/api.proto @@ -790,7 +790,7 @@ service Wallet { rpc GetPendingSize (EmptyMessage) returns (NumberMessage) { } - rpc GetBlock (BlockMessage) returns (BlockExtention) { + rpc GetBlock (BlockReq) returns (BlockExtention) { } }; @@ -977,7 +977,7 @@ service WalletSolidity { rpc GetBurnTrx (EmptyMessage) returns (NumberMessage) { } - rpc GetBlock (BlockMessage) returns (BlockExtention) { + rpc GetBlock (BlockReq) returns (BlockExtention) { } }; @@ -1128,8 +1128,8 @@ message TimeMessage { int64 beginInMilliseconds = 1; int64 endInMilliseconds = 2; } -message BlockMessage { - string idOrNum = 1; +message BlockReq { + string id_or_num = 1; bool detail = 2; } message BlockLimit { From fd836ba386f7c15ef1b322c88fe25f6e55133ff4 Mon Sep 17 00:00:00 2001 From: Wenhua Zhang Date: Wed, 27 Jul 2022 17:37:00 +0800 Subject: [PATCH 16/51] feat: add api to get bandwidth price history --- .../core/store/DynamicPropertiesStore.java | 73 ++++-- .../src/main/java/org/tron/core/Wallet.java | 10 + .../tron/core/consensus/ProposalService.java | 4 + .../db/api/BandwidthPriceHistoryLoader.java | 61 +++++ .../services/http/FullNodeHttpApiService.java | 6 +- .../http/GetBandwidthPricesServlet.java | 37 ++++ .../db/BandwidthPriceHistoryLoaderTest.java | 208 ++++++++++++++++++ .../core/db/EnergyPriceHistoryLoaderTest.java | 1 - .../core/services/ProposalServiceTest.java | 21 ++ 9 files changed, 405 insertions(+), 16 deletions(-) create mode 100644 framework/src/main/java/org/tron/core/db/api/BandwidthPriceHistoryLoader.java create mode 100644 framework/src/main/java/org/tron/core/services/http/GetBandwidthPricesServlet.java create mode 100644 framework/src/test/java/org/tron/core/db/BandwidthPriceHistoryLoaderTest.java diff --git a/chainbase/src/main/java/org/tron/core/store/DynamicPropertiesStore.java b/chainbase/src/main/java/org/tron/core/store/DynamicPropertiesStore.java index e084f3ee6ed..dce79515ad4 100644 --- a/chainbase/src/main/java/org/tron/core/store/DynamicPropertiesStore.java +++ b/chainbase/src/main/java/org/tron/core/store/DynamicPropertiesStore.java @@ -77,6 +77,9 @@ public class DynamicPropertiesStore extends TronStoreWithRevoking "CREATE_NEW_ACCOUNT_BANDWIDTH_RATE" .getBytes(); private static final byte[] TRANSACTION_FEE = "TRANSACTION_FEE".getBytes(); // 1 byte + private static final long DEFAULT_TRANSACTION_FEE = 10L; + public static final String DEFAULT_BANDWIDTH_PRICE_HISTORY = "0:" + DEFAULT_TRANSACTION_FEE; + private static final byte[] ASSET_ISSUE_FEE = "ASSET_ISSUE_FEE".getBytes(); private static final byte[] UPDATE_ACCOUNT_PERMISSION_FEE = "UPDATE_ACCOUNT_PERMISSION_FEE" .getBytes(); @@ -156,7 +159,8 @@ public class DynamicPropertiesStore extends TronStoreWithRevoking private static final byte[] MAX_FEE_LIMIT = "MAX_FEE_LIMIT".getBytes(); private static final byte[] BURN_TRX_AMOUNT = "BURN_TRX_AMOUNT".getBytes(); - private static final byte[] ALLOW_BLACKHOLE_OPTIMIZATION = "ALLOW_BLACKHOLE_OPTIMIZATION".getBytes(); + private static final byte[] ALLOW_BLACKHOLE_OPTIMIZATION = + "ALLOW_BLACKHOLE_OPTIMIZATION".getBytes(); private static final byte[] ALLOW_NEW_RESOURCE_MODEL = "ALLOW_NEW_RESOURCE_MODEL".getBytes(); private static final byte[] ALLOW_TVM_FREEZE = "ALLOW_TVM_FREEZE".getBytes(); private static final byte[] ALLOW_TVM_VOTE = "ALLOW_TVM_VOTE".getBytes(); @@ -169,11 +173,15 @@ public class DynamicPropertiesStore extends TronStoreWithRevoking "ALLOW_ACCOUNT_ASSET_OPTIMIZATION".getBytes(); private static final byte[] ALLOW_ASSET_OPTIMIZATION = - "ALLOW_ASSET_OPTIMIZATION".getBytes(); + "ALLOW_ASSET_OPTIMIZATION".getBytes(); private static final byte[] ENERGY_PRICE_HISTORY = "ENERGY_PRICE_HISTORY".getBytes(); private static final byte[] ENERGY_PRICE_HISTORY_DONE = "ENERGY_PRICE_HISTORY_DONE".getBytes(); + private static final byte[] BANDWIDTH_PRICE_HISTORY = "BANDWIDTH_PRICE_HISTORY".getBytes(); + private static final byte[] BANDWIDTH_PRICE_HISTORY_DONE = + "BANDWIDTH_PRICE_HISTORY_DONE".getBytes(); + private static final byte[] SET_BLACKHOLE_ACCOUNT_PERMISSION = "SET_BLACKHOLE_ACCOUNT_PERMISSION".getBytes(); private static final byte[] ALLOW_HIGHER_LIMIT_FOR_MAX_CPU_TIME_OF_ONE_TX = @@ -378,7 +386,6 @@ private DynamicPropertiesStore(@Value("properties") String dbName) { this.saveTotalTronPowerWeight(0L); } - try { this.getAllowAdaptiveEnergy(); } catch (IllegalArgumentException e) { @@ -449,7 +456,7 @@ private DynamicPropertiesStore(@Value("properties") String dbName) { try { this.getTransactionFee(); } catch (IllegalArgumentException e) { - this.saveTransactionFee(10L); // 10sun/byte + this.saveTransactionFee(DEFAULT_TRANSACTION_FEE); // 10sun/byte } try { @@ -793,7 +800,7 @@ private DynamicPropertiesStore(@Value("properties") String dbName) { this.getAllowAssetOptimization(); } catch (IllegalArgumentException e) { this.setAllowAssetOptimization(CommonParameter - .getInstance().getAllowAssetOptimization()); + .getInstance().getAllowAssetOptimization()); } try { @@ -815,6 +822,18 @@ private DynamicPropertiesStore(@Value("properties") String dbName) { this.saveEnergyPriceHistory(DEFAULT_ENERGY_PRICE_HISTORY); } + try { + this.getBandwidthPriceHistoryDone(); + } catch (IllegalArgumentException e) { + this.saveBandwidthPriceHistoryDone(0); + } + + try { + this.getBandwidthPriceHistory(); + } catch (IllegalArgumentException e) { + this.saveBandwidthPriceHistory(DEFAULT_BANDWIDTH_PRICE_HISTORY); + } + try { this.getSetBlackholeAccountPermission(); } catch (IllegalArgumentException e) { @@ -2015,7 +2034,7 @@ public long getLatestBlockHeaderNumberFromDB() { .orElseThrow( () -> new IllegalArgumentException("not found latest block header number")); } catch (ItemNotFoundException | BadItemException e) { - logger.error("{}", e); + logger.error("{}", e); } return -1; } @@ -2385,10 +2404,10 @@ public long getNewRewardAlgorithmEffectiveCycle() { public long getAllowAccountAssetOptimizationFromRoot() { try { return Optional.ofNullable(getFromRoot(ALLOW_ASSET_OPTIMIZATION)) - .map(BytesCapsule::getData) - .map(ByteArray::toLong) - .orElseThrow( - () -> new IllegalArgumentException("not found ALLOW_ASSET_OPTIMIZATION")); + .map(BytesCapsule::getData) + .map(ByteArray::toLong) + .orElseThrow( + () -> new IllegalArgumentException("not found ALLOW_ASSET_OPTIMIZATION")); } catch (Exception e) { logger.debug("{}", e.getMessage()); return CommonParameter.getInstance().getAllowAssetOptimization(); @@ -2411,16 +2430,17 @@ public void setAllowAccountAssetOptimization(long value) { // 1: enable public long getAllowAssetOptimization() { return Optional.ofNullable(getUnchecked(ALLOW_ASSET_OPTIMIZATION)) - .map(BytesCapsule::getData) - .map(ByteArray::toLong) - .orElseThrow( - () -> new IllegalArgumentException("not found ALLOW_ASSET_OPTIMIZATION")); + .map(BytesCapsule::getData) + .map(ByteArray::toLong) + .orElseThrow( + () -> new IllegalArgumentException("not found ALLOW_ASSET_OPTIMIZATION")); } public void setAllowAssetOptimization(long value) { this.put(ALLOW_ASSET_OPTIMIZATION, new BytesCapsule(ByteArray.fromLong(value))); } + // for energy price history public void saveEnergyPriceHistoryDone(long value) { this.put(ENERGY_PRICE_HISTORY_DONE, new BytesCapsule(ByteArray.fromLong(value))); @@ -2445,6 +2465,31 @@ public void saveEnergyPriceHistory(String value) { this.put(ENERGY_PRICE_HISTORY, new BytesCapsule(ByteArray.fromString(value))); } + // for bandwidth price history + public void saveBandwidthPriceHistoryDone(long value) { + this.put(BANDWIDTH_PRICE_HISTORY_DONE, + new BytesCapsule(ByteArray.fromLong(value))); + } + + public long getBandwidthPriceHistoryDone() { + return Optional.ofNullable(getUnchecked(BANDWIDTH_PRICE_HISTORY_DONE)) + .map(BytesCapsule::getData) + .map(ByteArray::toLong) + .orElseThrow( + () -> new IllegalArgumentException("not found BANDWIDTH_PRICE_HISTORY_DONE")); + } + + public String getBandwidthPriceHistory() { + return Optional.ofNullable(getUnchecked(BANDWIDTH_PRICE_HISTORY)) + .map(BytesCapsule::getData) + .map(ByteArray::toStr) + .orElseThrow(() -> new IllegalArgumentException("not found BANDWIDTH_PRICE_HISTORY")); + } + + public void saveBandwidthPriceHistory(String value) { + this.put(BANDWIDTH_PRICE_HISTORY, new BytesCapsule(ByteArray.fromString(value))); + } + public long getSetBlackholeAccountPermission() { return Optional.of(getUnchecked(SET_BLACKHOLE_ACCOUNT_PERMISSION)) .map(BytesCapsule::getData) diff --git a/framework/src/main/java/org/tron/core/Wallet.java b/framework/src/main/java/org/tron/core/Wallet.java index 29c61082681..1c487fa5dc7 100755 --- a/framework/src/main/java/org/tron/core/Wallet.java +++ b/framework/src/main/java/org/tron/core/Wallet.java @@ -3954,6 +3954,16 @@ public String getEnergyPrices() { return null; } + public String getBandwidthPrices() { + try { + return chainBaseManager.getDynamicPropertiesStore().getBandwidthPriceHistory(); + } catch (Exception e) { + logger.error("getBandwidthPrices failed, error is {}", e.getMessage()); + } + + return null; + } + public String getCoinbase() { if (!CommonParameter.getInstance().isWitness()) { return null; diff --git a/framework/src/main/java/org/tron/core/consensus/ProposalService.java b/framework/src/main/java/org/tron/core/consensus/ProposalService.java index 21c3e70896f..f6f55b0097c 100644 --- a/framework/src/main/java/org/tron/core/consensus/ProposalService.java +++ b/framework/src/main/java/org/tron/core/consensus/ProposalService.java @@ -40,6 +40,10 @@ public static boolean process(Manager manager, ProposalCapsule proposalCapsule) } case TRANSACTION_FEE: { manager.getDynamicPropertiesStore().saveTransactionFee(entry.getValue()); + // update bandwidth price history + manager.getDynamicPropertiesStore().saveBandwidthPriceHistory( + manager.getDynamicPropertiesStore().getBandwidthPriceHistory() + + "," + proposalCapsule.getExpirationTime() + ":" + entry.getValue()); break; } case ASSET_ISSUE_FEE: { diff --git a/framework/src/main/java/org/tron/core/db/api/BandwidthPriceHistoryLoader.java b/framework/src/main/java/org/tron/core/db/api/BandwidthPriceHistoryLoader.java new file mode 100644 index 00000000000..e3df76450b1 --- /dev/null +++ b/framework/src/main/java/org/tron/core/db/api/BandwidthPriceHistoryLoader.java @@ -0,0 +1,61 @@ +package org.tron.core.db.api; + +import java.util.ArrayList; +import java.util.List; +import lombok.extern.slf4j.Slf4j; +import org.tron.core.ChainBaseManager; +import org.tron.core.capsule.ProposalCapsule; +import org.tron.core.store.DynamicPropertiesStore; +import org.tron.core.utils.ProposalUtil.ProposalType; +import org.tron.protos.Protocol.Proposal.State; + +@Slf4j(topic = "DB") +public class BandwidthPriceHistoryLoader { + + private final ChainBaseManager chainBaseManager; + private List proposalCapsuleList = new ArrayList<>(); + + public BandwidthPriceHistoryLoader(ChainBaseManager chainBaseManager) { + this.chainBaseManager = chainBaseManager; + } + + public void doWork() { + long start = System.currentTimeMillis(); + logger.info("Start to load bandwidth price"); + + getBandwidthProposals(); + if (!proposalCapsuleList.isEmpty()) { + String bandwidthPriceHistory = parseProposalsToStr(); + chainBaseManager.getDynamicPropertiesStore().saveBandwidthPriceHistory(bandwidthPriceHistory); + } + finish(); + + logger.info( + "Complete bandwidth price load, total time: {} milliseconds, total proposal count: {}", + System.currentTimeMillis() - start, proposalCapsuleList.size()); + } + + public void getBandwidthProposals() { + proposalCapsuleList = chainBaseManager.getProposalStore() + .getSpecifiedProposals(State.APPROVED, ProposalType.TRANSACTION_FEE.getCode()); + } + + public String parseProposalsToStr() { + StringBuilder builder = + new StringBuilder(DynamicPropertiesStore.DEFAULT_BANDWIDTH_PRICE_HISTORY); + + for (ProposalCapsule proposalCapsule : proposalCapsuleList) { + builder.append(",") + .append(proposalCapsule.getExpirationTime()) + .append(":") + .append(proposalCapsule.getParameters().get(ProposalType.TRANSACTION_FEE.getCode())); + } + + return builder.toString(); + } + + public void finish() { + chainBaseManager.getDynamicPropertiesStore().saveBandwidthPriceHistoryDone(1); + } + +} diff --git a/framework/src/main/java/org/tron/core/services/http/FullNodeHttpApiService.java b/framework/src/main/java/org/tron/core/services/http/FullNodeHttpApiService.java index 98e8bc88614..031bd38e50c 100644 --- a/framework/src/main/java/org/tron/core/services/http/FullNodeHttpApiService.java +++ b/framework/src/main/java/org/tron/core/services/http/FullNodeHttpApiService.java @@ -289,7 +289,8 @@ public class FullNodeHttpApiService implements Service { private GetPendingSizeServlet getPendingSizeServlet; @Autowired private GetEnergyPricesServlet getEnergyPricesServlet; - + @Autowired + private GetBandwidthPricesServlet getBandwidthPricesServlet; @Autowired private GetBlockServlet getBlockServlet; @@ -538,7 +539,10 @@ public void start() { "/wallet/gettransactionlistfrompending"); context.addServlet(new ServletHolder(getPendingSizeServlet), "/wallet/getpendingsize"); context.addServlet(new ServletHolder(getEnergyPricesServlet), "/wallet/getenergyprices"); + context.addServlet(new ServletHolder(getBandwidthPricesServlet), + "/wallet/getbandwidthprices"); context.addServlet(new ServletHolder(getBlockServlet), "/wallet/getblock"); + int maxHttpConnectNumber = Args.getInstance().getMaxHttpConnectNumber(); if (maxHttpConnectNumber > 0) { server.addBean(new ConnectionLimit(maxHttpConnectNumber, server)); diff --git a/framework/src/main/java/org/tron/core/services/http/GetBandwidthPricesServlet.java b/framework/src/main/java/org/tron/core/services/http/GetBandwidthPricesServlet.java new file mode 100644 index 00000000000..ea4b535af39 --- /dev/null +++ b/framework/src/main/java/org/tron/core/services/http/GetBandwidthPricesServlet.java @@ -0,0 +1,37 @@ +package org.tron.core.services.http; + +import com.alibaba.fastjson.JSONObject; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; +import org.tron.core.Wallet; + + +@Component +@Slf4j(topic = "API") +public class GetBandwidthPricesServlet extends RateLimiterServlet { + + @Autowired + private Wallet wallet; + + protected void doGet(HttpServletRequest request, HttpServletResponse response) { + try { + String reply = wallet.getBandwidthPrices(); + if (reply != null) { + JSONObject jsonObject = new JSONObject(); + jsonObject.put("prices", reply); + response.getWriter().println(jsonObject); + } else { + response.getWriter().println("{}"); + } + } catch (Exception e) { + Util.processError(e, response); + } + } + + protected void doPost(HttpServletRequest request, HttpServletResponse response) { + doGet(request, response); + } +} diff --git a/framework/src/test/java/org/tron/core/db/BandwidthPriceHistoryLoaderTest.java b/framework/src/test/java/org/tron/core/db/BandwidthPriceHistoryLoaderTest.java new file mode 100644 index 00000000000..2d72d36395d --- /dev/null +++ b/framework/src/test/java/org/tron/core/db/BandwidthPriceHistoryLoaderTest.java @@ -0,0 +1,208 @@ +package org.tron.core.db; + +import static org.tron.core.store.DynamicPropertiesStore.DEFAULT_BANDWIDTH_PRICE_HISTORY; +import static org.tron.core.utils.ProposalUtil.ProposalType.ALLOW_CREATION_OF_CONTRACTS; +import static org.tron.core.utils.ProposalUtil.ProposalType.ALLOW_TVM_FREEZE; +import static org.tron.core.utils.ProposalUtil.ProposalType.ALLOW_TVM_LONDON; +import static org.tron.core.utils.ProposalUtil.ProposalType.ALLOW_TVM_VOTE; +import static org.tron.core.utils.ProposalUtil.ProposalType.ASSET_ISSUE_FEE; +import static org.tron.core.utils.ProposalUtil.ProposalType.CREATE_ACCOUNT_FEE; +import static org.tron.core.utils.ProposalUtil.ProposalType.CREATE_NEW_ACCOUNT_FEE_IN_SYSTEM_CONTRACT; +import static org.tron.core.utils.ProposalUtil.ProposalType.TRANSACTION_FEE; +import static org.tron.core.utils.ProposalUtil.ProposalType.WITNESS_127_PAY_PER_BLOCK; + +import java.io.File; +import java.util.HashMap; +import java.util.Map; +import lombok.extern.slf4j.Slf4j; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.tron.common.application.TronApplicationContext; +import org.tron.common.utils.FileUtil; +import org.tron.core.ChainBaseManager; +import org.tron.core.Constant; +import org.tron.core.capsule.ProposalCapsule; +import org.tron.core.config.DefaultConfig; +import org.tron.core.config.args.Args; +import org.tron.core.db.api.BandwidthPriceHistoryLoader; +import org.tron.protos.Protocol.Proposal; +import org.tron.protos.Protocol.Proposal.State; + + +@Slf4j +public class BandwidthPriceHistoryLoaderTest { + + private static ChainBaseManager chainBaseManager; + private static TronApplicationContext context; + private static final String dbPath = "output-BandwidthPriceHistoryLoaderTest-test"; + private static long t1; + private static long price1; + private static long t2; + private static long price2; + private static long t5; + private static long price5; + + // Note, here use @Before and @After instead of @BeforeClass and @AfterClass, + // because it needs to initialize DB before the single test every time + @Before + public void init() { + Args.setParam(new String[] {"--output-directory", dbPath}, Constant.TEST_CONF); + context = new TronApplicationContext(DefaultConfig.class); + chainBaseManager = context.getBean(ChainBaseManager.class); + } + + @After + public void destroy() { + Args.clearParam(); + context.destroy(); + if (FileUtil.deleteDir(new File(dbPath))) { + logger.info("Release resources successful."); + } else { + logger.info("Release resources failure."); + } + } + + public void initDB() { + t1 = 1606240800000L; + price1 = 40; + initProposal(TRANSACTION_FEE.getCode(), t1, price1, State.APPROVED); + + t2 = 1613044800000L; + price2 = 140; + initProposal(TRANSACTION_FEE.getCode(), t2, price2, State.APPROVED); + + long t3 = 1626501600000L; + long price3 = 1000; + Map parameters = new HashMap<>(); + parameters.put(TRANSACTION_FEE.getCode(), price3); + parameters.put(CREATE_NEW_ACCOUNT_FEE_IN_SYSTEM_CONTRACT.getCode(), 1000000L); + initProposal(parameters, t3, State.CANCELED); + + long t4 = 1626501600000L; + long price4 = 1000; + parameters = new HashMap<>(); + parameters.put(TRANSACTION_FEE.getCode(), price4); + parameters.put(CREATE_NEW_ACCOUNT_FEE_IN_SYSTEM_CONTRACT.getCode(), 1000000L); + initProposal(parameters, t4, State.DISAPPROVED); + + t5 = 1627279200000L; + price5 = 1000L; + parameters = new HashMap<>(); + parameters.put(TRANSACTION_FEE.getCode(), price4); + parameters.put(CREATE_NEW_ACCOUNT_FEE_IN_SYSTEM_CONTRACT.getCode(), 1000000L); + initProposal(parameters, t5, State.APPROVED); + + long t6 = 1634299200000L; + parameters = new HashMap<>(); + parameters.put(ALLOW_TVM_FREEZE.getCode(), 1L); + parameters.put(ALLOW_TVM_VOTE.getCode(), 1L); + initProposal(parameters, t6, State.DISAPPROVED); + + initProposal(ALLOW_TVM_LONDON.getCode(), 1647604800000L, 1, State.APPROVED); + + initProposal(ALLOW_CREATION_OF_CONTRACTS.getCode(), 1539259200000L, 1, State.APPROVED); + initProposal(CREATE_ACCOUNT_FEE.getCode(), 1621468800000L, 1, State.DISAPPROVED); + + parameters = new HashMap<>(); + parameters.put(ASSET_ISSUE_FEE.getCode(), 48000000L); + parameters.put(WITNESS_127_PAY_PER_BLOCK.getCode(), 128000000L); + initProposal(parameters, 1572609600000L, State.CANCELED); + } + + private static void initProposal(long code, long timestamp, long price, State state) { + long id = chainBaseManager.getDynamicPropertiesStore().getLatestProposalNum() + 1; + + Proposal proposal = Proposal.newBuilder().putParameters(code, price) + .setExpirationTime(timestamp) + .setState(state) + .setProposalId(id) + .build(); + ProposalCapsule proposalCapsule = new ProposalCapsule(proposal); + + chainBaseManager.getProposalStore().put(proposalCapsule.createDbKey(), proposalCapsule); + chainBaseManager.getDynamicPropertiesStore().saveLatestProposalNum(id); + } + + private static void initProposal(Map parameters, long timestamp, State state) { + long id = chainBaseManager.getDynamicPropertiesStore().getLatestProposalNum() + 1; + + Proposal proposal = Proposal.newBuilder().putAllParameters(parameters) + .setExpirationTime(timestamp) + .setState(state) + .setProposalId(id) + .build(); + ProposalCapsule proposalCapsule = new ProposalCapsule(proposal); + + chainBaseManager.getProposalStore().put(proposalCapsule.createDbKey(), proposalCapsule); + chainBaseManager.getDynamicPropertiesStore().saveLatestProposalNum(id); + } + + @Test + public void testLoaderWork() { + Assert.assertEquals(0L, + chainBaseManager.getDynamicPropertiesStore().getBandwidthPriceHistoryDone()); + + initDB(); + + String preBandwidthPriceHistory = + chainBaseManager.getDynamicPropertiesStore().getBandwidthPriceHistory(); + String expectedRes = preBandwidthPriceHistory + "," + t1 + ":" + price1 + + "," + t2 + ":" + price2 + + "," + t5 + ":" + price5; + + BandwidthPriceHistoryLoader loader = new BandwidthPriceHistoryLoader(chainBaseManager); + loader.getBandwidthProposals(); + String historyStr = loader.parseProposalsToStr(); + + Assert.assertEquals(expectedRes, historyStr); + Assert.assertEquals(0L, + chainBaseManager.getDynamicPropertiesStore().getBandwidthPriceHistoryDone()); + } + + @Test + public void testProposalEmpty() { + Assert.assertEquals(0L, + chainBaseManager.getDynamicPropertiesStore().getBandwidthPriceHistoryDone()); + String preBandwidthPriceHistory = + chainBaseManager.getDynamicPropertiesStore().getBandwidthPriceHistory(); + Assert.assertEquals(DEFAULT_BANDWIDTH_PRICE_HISTORY, preBandwidthPriceHistory); + + // loader work + BandwidthPriceHistoryLoader loader = new BandwidthPriceHistoryLoader(chainBaseManager); + loader.doWork(); + + // check result + String afterBandwidthPriceHistory = + chainBaseManager.getDynamicPropertiesStore().getBandwidthPriceHistory(); + Assert.assertEquals(DEFAULT_BANDWIDTH_PRICE_HISTORY, afterBandwidthPriceHistory); + Assert.assertEquals(1L, + chainBaseManager.getDynamicPropertiesStore().getBandwidthPriceHistoryDone()); + } + + @Test + public void testLoaderWithProposals() { + String preBandwidthPriceHistory = + chainBaseManager.getDynamicPropertiesStore().getBandwidthPriceHistory(); + Assert.assertEquals(DEFAULT_BANDWIDTH_PRICE_HISTORY, preBandwidthPriceHistory); + + // init proposals + initDB(); + + // loader work + BandwidthPriceHistoryLoader loader = new BandwidthPriceHistoryLoader(chainBaseManager); + loader.doWork(); + + // check result + String afterBandwidthPriceHistory = + chainBaseManager.getDynamicPropertiesStore().getBandwidthPriceHistory(); + String expectedRes = preBandwidthPriceHistory + "," + t1 + ":" + price1 + + "," + t2 + ":" + price2 + + "," + t5 + ":" + price5; + + Assert.assertEquals(expectedRes, afterBandwidthPriceHistory); + Assert.assertEquals(1L, + chainBaseManager.getDynamicPropertiesStore().getBandwidthPriceHistoryDone()); + } +} diff --git a/framework/src/test/java/org/tron/core/db/EnergyPriceHistoryLoaderTest.java b/framework/src/test/java/org/tron/core/db/EnergyPriceHistoryLoaderTest.java index 7700dbdead2..13107480bf5 100644 --- a/framework/src/test/java/org/tron/core/db/EnergyPriceHistoryLoaderTest.java +++ b/framework/src/test/java/org/tron/core/db/EnergyPriceHistoryLoaderTest.java @@ -15,7 +15,6 @@ import org.junit.AfterClass; import org.junit.Assert; import org.junit.BeforeClass; -import org.junit.Ignore; import org.junit.Test; import org.tron.common.application.TronApplicationContext; import org.tron.common.utils.FileUtil; diff --git a/framework/src/test/java/org/tron/core/services/ProposalServiceTest.java b/framework/src/test/java/org/tron/core/services/ProposalServiceTest.java index 10faf6e19ca..7978d98a9fe 100644 --- a/framework/src/test/java/org/tron/core/services/ProposalServiceTest.java +++ b/framework/src/test/java/org/tron/core/services/ProposalServiceTest.java @@ -1,6 +1,7 @@ package org.tron.core.services; import static org.tron.core.utils.ProposalUtil.ProposalType.ENERGY_FEE; +import static org.tron.core.utils.ProposalUtil.ProposalType.TRANSACTION_FEE; import static org.tron.core.utils.ProposalUtil.ProposalType.WITNESS_127_PAY_PER_BLOCK; import java.io.File; @@ -84,6 +85,26 @@ public void testUpdateEnergyFee() { currentHistory); } + @Test + public void testUpdateTransactionFee() { + String preHistory = manager.getDynamicPropertiesStore().getBandwidthPriceHistory(); + + long newPrice = 1500; + Proposal proposal = + Proposal.newBuilder().putParameters(TRANSACTION_FEE.getCode(), newPrice).build(); + ProposalCapsule proposalCapsule = new ProposalCapsule(proposal); + proposalCapsule.setExpirationTime(1627279200000L); + boolean result = ProposalService.process(manager, proposalCapsule); + Assert.assertTrue(result); + + long currentPrice = manager.getDynamicPropertiesStore().getTransactionFee(); + Assert.assertEquals(currentPrice, newPrice); + + String expResult = preHistory + "," + proposalCapsule.getExpirationTime() + ":" + newPrice; + String currentHistory = manager.getDynamicPropertiesStore().getBandwidthPriceHistory(); + Assert.assertEquals(expResult, currentHistory); + } + @AfterClass public static void removeDb() { Args.clearParam(); From f219feef4fc7b18e8fa3c11ad2bfd2f8057ec437 Mon Sep 17 00:00:00 2001 From: Wenhua Zhang Date: Wed, 27 Jul 2022 18:42:12 +0800 Subject: [PATCH 17/51] feat(bandwidth_prices): load bandwidth prices when init --- framework/src/main/java/org/tron/core/db/Manager.java | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/framework/src/main/java/org/tron/core/db/Manager.java b/framework/src/main/java/org/tron/core/db/Manager.java index 2d0f85a94d0..d2cfb33669b 100644 --- a/framework/src/main/java/org/tron/core/db/Manager.java +++ b/framework/src/main/java/org/tron/core/db/Manager.java @@ -96,6 +96,7 @@ import org.tron.core.db.accountstate.TrieService; import org.tron.core.db.accountstate.callback.AccountStateCallBack; import org.tron.core.db.api.AssetUpdateHelper; +import org.tron.core.db.api.BandwidthPriceHistoryLoader; import org.tron.core.db.api.EnergyPriceHistoryLoader; import org.tron.core.db.api.MoveAbiHelper; import org.tron.core.db2.ISession; @@ -323,6 +324,10 @@ private boolean needToLoadEnergyPriceHistory() { return getDynamicPropertiesStore().getEnergyPriceHistoryDone() == 0L; } + private boolean needToLoadBandwidthPriceHistory() { + return getDynamicPropertiesStore().getBandwidthPriceHistoryDone() == 0L; + } + public boolean needToSetBlackholePermission() { return getDynamicPropertiesStore().getSetBlackholeAccountPermission() == 0L; } @@ -480,6 +485,10 @@ public void init() { new EnergyPriceHistoryLoader(chainBaseManager).doWork(); } + if (needToLoadBandwidthPriceHistory()) { + new BandwidthPriceHistoryLoader(chainBaseManager).doWork(); + } + if (needToSetBlackholePermission()) { resetBlackholeAccountPermission(); } From ad4bf2fe8559547812731321437125746f4e3aa0 Mon Sep 17 00:00:00 2001 From: Wenhua Zhang Date: Wed, 27 Jul 2022 21:52:44 +0800 Subject: [PATCH 18/51] feat: set saveBandwidthPriceHistoryDone as 0 before test --- .../org/tron/core/db/BandwidthPriceHistoryLoaderTest.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/framework/src/test/java/org/tron/core/db/BandwidthPriceHistoryLoaderTest.java b/framework/src/test/java/org/tron/core/db/BandwidthPriceHistoryLoaderTest.java index 2d72d36395d..a91c3f96c70 100644 --- a/framework/src/test/java/org/tron/core/db/BandwidthPriceHistoryLoaderTest.java +++ b/framework/src/test/java/org/tron/core/db/BandwidthPriceHistoryLoaderTest.java @@ -141,8 +141,6 @@ private static void initProposal(Map parameters, long timestamp, Sta @Test public void testLoaderWork() { - Assert.assertEquals(0L, - chainBaseManager.getDynamicPropertiesStore().getBandwidthPriceHistoryDone()); initDB(); @@ -163,12 +161,12 @@ public void testLoaderWork() { @Test public void testProposalEmpty() { - Assert.assertEquals(0L, - chainBaseManager.getDynamicPropertiesStore().getBandwidthPriceHistoryDone()); String preBandwidthPriceHistory = chainBaseManager.getDynamicPropertiesStore().getBandwidthPriceHistory(); Assert.assertEquals(DEFAULT_BANDWIDTH_PRICE_HISTORY, preBandwidthPriceHistory); + chainBaseManager.getDynamicPropertiesStore().saveBandwidthPriceHistoryDone(0); + // loader work BandwidthPriceHistoryLoader loader = new BandwidthPriceHistoryLoader(chainBaseManager); loader.doWork(); @@ -187,6 +185,8 @@ public void testLoaderWithProposals() { chainBaseManager.getDynamicPropertiesStore().getBandwidthPriceHistory(); Assert.assertEquals(DEFAULT_BANDWIDTH_PRICE_HISTORY, preBandwidthPriceHistory); + chainBaseManager.getDynamicPropertiesStore().saveBandwidthPriceHistoryDone(0); + // init proposals initDB(); From d8a09e1ca8b1d7f95526688d56a659d8b996aef1 Mon Sep 17 00:00:00 2001 From: Wenhua Zhang Date: Thu, 28 Jul 2022 10:56:50 +0800 Subject: [PATCH 19/51] feat: remove check andwidthPriceHistoryDone --- .../java/org/tron/core/db/BandwidthPriceHistoryLoaderTest.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/framework/src/test/java/org/tron/core/db/BandwidthPriceHistoryLoaderTest.java b/framework/src/test/java/org/tron/core/db/BandwidthPriceHistoryLoaderTest.java index a91c3f96c70..4c7af2b31fe 100644 --- a/framework/src/test/java/org/tron/core/db/BandwidthPriceHistoryLoaderTest.java +++ b/framework/src/test/java/org/tron/core/db/BandwidthPriceHistoryLoaderTest.java @@ -155,8 +155,6 @@ public void testLoaderWork() { String historyStr = loader.parseProposalsToStr(); Assert.assertEquals(expectedRes, historyStr); - Assert.assertEquals(0L, - chainBaseManager.getDynamicPropertiesStore().getBandwidthPriceHistoryDone()); } @Test From 25ab12561e47d9b8c26138c778bdab1fbe2a5caa Mon Sep 17 00:00:00 2001 From: chengtx01 Date: Fri, 29 Jul 2022 14:32:08 +0800 Subject: [PATCH 20/51] perf(block): optimize block processing thread acquires the lock code --- .../tron/core/consensus/BlockHandleImpl.java | 6 +---- .../main/java/org/tron/core/db/Manager.java | 23 +++++++++++++++---- 2 files changed, 19 insertions(+), 10 deletions(-) diff --git a/framework/src/main/java/org/tron/core/consensus/BlockHandleImpl.java b/framework/src/main/java/org/tron/core/consensus/BlockHandleImpl.java index 03452546a9b..fcfb801d1ce 100644 --- a/framework/src/main/java/org/tron/core/consensus/BlockHandleImpl.java +++ b/framework/src/main/java/org/tron/core/consensus/BlockHandleImpl.java @@ -60,10 +60,6 @@ public BlockCapsule produce(Miner miner, long blockTime, long timeout) { } public void setBlockWaitLock(boolean flag) { - if (flag) { - manager.getBlockWaitLock().incrementAndGet(); - } else { - manager.getBlockWaitLock().decrementAndGet(); - } + manager.setBlockWaitLock(flag); } } diff --git a/framework/src/main/java/org/tron/core/db/Manager.java b/framework/src/main/java/org/tron/core/db/Manager.java index 539c6d4c700..da8b7491248 100644 --- a/framework/src/main/java/org/tron/core/db/Manager.java +++ b/framework/src/main/java/org/tron/core/db/Manager.java @@ -169,6 +169,8 @@ public class Manager { private static final String SAVE_BLOCK = "save block: "; private static final int SLEEP_TIME_OUT = 50; private static final int TX_ID_CACHE_SIZE = 100_000; + private static final int SLEEP_FOR_WAIT_LOCK = 10; + private static final int NO_BLOCK_WAITING_LOCK = 0; private final int shieldedTransInPendingMaxCounts = Args.getInstance().getShieldedTransInPendingMaxCounts(); @Getter @@ -240,7 +242,6 @@ public class Manager { @Getter private final ThreadLocal blockedTimer = new ThreadLocal<>(); - @Getter private AtomicInteger blockWaitLock = new AtomicInteger(0); private Object transactionLock = new Object(); @@ -773,8 +774,8 @@ public boolean pushTransaction(final TransactionCapsule trx) synchronized (transactionLock) { while (true) { try { - if (blockWaitLock.get() > 0) { - TimeUnit.MILLISECONDS.sleep(10); + if (isBlockWaitingLock()) { + TimeUnit.MILLISECONDS.sleep(SLEEP_FOR_WAIT_LOCK); } else { break; } @@ -1084,7 +1085,7 @@ public void pushBlock(final BlockCapsule block) DupTransactionException, TransactionExpirationException, BadNumberBlockException, BadBlockException, NonCommonBlockException, ReceiptCheckErrException, VMIllegalException, ZksnarkException, EventBloomException { - blockWaitLock.incrementAndGet(); + setBlockWaitLock(true); try { synchronized (this) { Metrics.histogramObserve(blockedTimer.get()); @@ -1251,7 +1252,7 @@ public void pushBlock(final BlockCapsule block) Metrics.histogramObserve(timer); } } finally { - blockWaitLock.decrementAndGet(); + setBlockWaitLock(false); } } @@ -2270,6 +2271,18 @@ public long getPendingSize() { return value; } + public void setBlockWaitLock(boolean waitFlag) { + if (waitFlag) { + blockWaitLock.incrementAndGet(); + } else { + blockWaitLock.decrementAndGet(); + } + } + + private boolean isBlockWaitingLock() { + return blockWaitLock.get() > NO_BLOCK_WAITING_LOCK; + } + private static class ValidateSignTask implements Callable { private TransactionCapsule trx; From d56d5b5261364bc16fa618852a53e2657193fa33 Mon Sep 17 00:00:00 2001 From: Wenhua Zhang Date: Fri, 29 Jul 2022 19:15:52 +0800 Subject: [PATCH 21/51] feat: catch DecoderException when address is not base58 in GetRewardServlet --- .../org/tron/core/services/http/GetRewardServlet.java | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/framework/src/main/java/org/tron/core/services/http/GetRewardServlet.java b/framework/src/main/java/org/tron/core/services/http/GetRewardServlet.java index d1903a0fb2c..390f7973bc3 100644 --- a/framework/src/main/java/org/tron/core/services/http/GetRewardServlet.java +++ b/framework/src/main/java/org/tron/core/services/http/GetRewardServlet.java @@ -4,6 +4,7 @@ import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; import lombok.extern.slf4j.Slf4j; +import org.bouncycastle.util.encoders.DecoderException; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; import org.tron.core.db.Manager; @@ -24,6 +25,13 @@ protected void doGet(HttpServletRequest request, HttpServletResponse response) { value = manager.getMortgageService().queryReward(address); } response.getWriter().println("{\"reward\": " + value + "}"); + } catch (DecoderException e) { + try { + response.getWriter() + .println("{\"Error\": " + "\"INVALID base58 String, " + e.getMessage() + "\"}"); + } catch (IOException ioe) { + logger.debug("IOException: {}", ioe.getMessage()); + } } catch (Exception e) { logger.error("", e); try { From 294b63218f88837bd02468a1494b1b62a596cd67 Mon Sep 17 00:00:00 2001 From: Wenhua Zhang Date: Fri, 29 Jul 2022 19:19:42 +0800 Subject: [PATCH 22/51] feat: update error msg --- .../main/java/org/tron/core/services/http/GetRewardServlet.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/framework/src/main/java/org/tron/core/services/http/GetRewardServlet.java b/framework/src/main/java/org/tron/core/services/http/GetRewardServlet.java index 390f7973bc3..14d6f11770d 100644 --- a/framework/src/main/java/org/tron/core/services/http/GetRewardServlet.java +++ b/framework/src/main/java/org/tron/core/services/http/GetRewardServlet.java @@ -28,7 +28,7 @@ protected void doGet(HttpServletRequest request, HttpServletResponse response) { } catch (DecoderException e) { try { response.getWriter() - .println("{\"Error\": " + "\"INVALID base58 String, " + e.getMessage() + "\"}"); + .println("{\"Error\": " + "\"INVALID address, " + e.getMessage() + "\"}"); } catch (IOException ioe) { logger.debug("IOException: {}", ioe.getMessage()); } From 0e74efc5a3c8228da02834b8072d51f495d856e9 Mon Sep 17 00:00:00 2001 From: halibobo1205 Date: Fri, 29 Jul 2022 17:56:32 +0800 Subject: [PATCH 23/51] tool(db):db path move function. 1. print help if no params. 2. remove toolkit name. 3. if config or path not exist,print error --- .../src/main/java/org/tron/plugins/Db.java | 26 +++++++++++++++-- .../main/java/org/tron/plugins/DbMove.java | 29 ++++++++++++++----- .../main/java/org/tron/plugins/Toolkit.java | 11 +++++-- 3 files changed, 53 insertions(+), 13 deletions(-) diff --git a/plugins/src/main/java/org/tron/plugins/Db.java b/plugins/src/main/java/org/tron/plugins/Db.java index a886d717c6d..e03a412c1a9 100644 --- a/plugins/src/main/java/org/tron/plugins/Db.java +++ b/plugins/src/main/java/org/tron/plugins/Db.java @@ -2,6 +2,9 @@ import com.typesafe.config.Config; import com.typesafe.config.ConfigFactory; +import java.io.File; +import java.io.IOException; +import java.nio.file.Path; import java.nio.file.Paths; import picocli.CommandLine; @@ -18,8 +21,27 @@ static class ConfigConverter implements CommandLine.ITypeConverter { ConfigConverter() { } - public Config convert(String value) { - return ConfigFactory.parseFile(Paths.get(value).toFile()); + public Config convert(String value) throws IOException { + File file = Paths.get(value).toFile(); + if (file.exists() && file.isFile()) { + return ConfigFactory.parseFile(Paths.get(value).toFile()); + } else { + throw new IOException("DB config [" + value + "] not exist!"); + } + } + } + + static class PathConverter implements CommandLine.ITypeConverter { + PathConverter() { + } + + public Path convert(String value) throws IOException { + File file = Paths.get(value).toFile(); + if (file.exists() && file.isDirectory()) { + return file.toPath(); + } else { + throw new IOException("DB path [" + value + "] not exist!"); + } } } } diff --git a/plugins/src/main/java/org/tron/plugins/DbMove.java b/plugins/src/main/java/org/tron/plugins/DbMove.java index 67bc3a57706..d9a2a8b4831 100644 --- a/plugins/src/main/java/org/tron/plugins/DbMove.java +++ b/plugins/src/main/java/org/tron/plugins/DbMove.java @@ -25,15 +25,16 @@ public class DbMove implements Callable { private static final String DEFAULT_DB_DIRECTORY = "database"; private static final String NAME_CONFIG_KEY = "name"; private static final String PATH_CONFIG_KEY = "path"; - private static final String NOT_FIND = "The database to be moved cannot be found."; + private static final String NOT_FIND = "There is no database to be moved, exist."; @CommandLine.Spec CommandLine.Model.CommandSpec spec; @CommandLine.Option(names = {"-d", "--database-directory"}, defaultValue = "output-directory", + converter = Db.PathConverter.class, description = "database directory path. Default: ${DEFAULT-VALUE}") - String database; + Path database; @CommandLine.Option(names = {"-c", "--config"}, defaultValue = "config.conf", @@ -50,10 +51,11 @@ public Integer call() throws Exception { spec.commandLine().usage(System.out); return 0; } + if (config.hasPath(PROPERTIES_CONFIG_KEY)) { List dbs = config.getConfigList(PROPERTIES_CONFIG_KEY); if (dbs.isEmpty()) { - spec.commandLine().getOut().println(NOT_FIND); + printNotExist(); return 0; } String dbPath = config.hasPath(DB_DIRECTORY_CONFIG_KEY) @@ -64,14 +66,14 @@ public Integer call() throws Exception { .collect(Collectors.toList()); if (dbs.isEmpty()) { - spec.commandLine().getOut().println(NOT_FIND); + printNotExist(); return 0; } List toBeMove = dbs.stream() .map(c -> { try { return new Property(c.getString(NAME_CONFIG_KEY), - Paths.get(database, dbPath, c.getString(NAME_CONFIG_KEY)), + Paths.get(database.toString(), dbPath, c.getString(NAME_CONFIG_KEY)), Paths.get(c.getString(PATH_CONFIG_KEY), dbPath, c.getString(NAME_CONFIG_KEY))); } catch (IOException e) { spec.commandLine().getErr().println(e); @@ -81,7 +83,7 @@ public Integer call() throws Exception { .filter(p -> !p.destination.equals(p.original)).collect(Collectors.toList()); if (toBeMove.isEmpty()) { - spec.commandLine().getOut().println(NOT_FIND); + printNotExist(); return 0; } toBeMove = toBeMove.stream() @@ -96,14 +98,15 @@ public Integer call() throws Exception { }).collect(Collectors.toList()); if (toBeMove.isEmpty()) { - spec.commandLine().getOut().println(NOT_FIND); + printNotExist(); return 0; } ProgressBar.wrap(toBeMove.stream(), "mv task").forEach(this::run); spec.commandLine().getOut().println("move db done."); } else { - spec.commandLine().getOut().println(NOT_FIND); + printNotExist(); + return 0; } return 0; } @@ -133,6 +136,10 @@ private void run(Property p) { } } + private void printNotExist() { + spec.commandLine().getErr().println(NOT_FIND); + } + /** * delete directory. */ @@ -157,6 +164,12 @@ static class Property { public Property(String name, Path original, Path destination) throws IOException { this.name = name; this.original = original.toFile().getCanonicalFile().toPath(); + if (!this.original.toFile().exists()) { + throw new IOException(this.original + " not exist!"); + } + if (this.original.toFile().isFile()) { + throw new IOException(this.original + " is a file!"); + } this.destination = destination.toFile().getCanonicalFile().toPath(); } } diff --git a/plugins/src/main/java/org/tron/plugins/Toolkit.java b/plugins/src/main/java/org/tron/plugins/Toolkit.java index 4a087e40b34..3b9972de1c5 100644 --- a/plugins/src/main/java/org/tron/plugins/Toolkit.java +++ b/plugins/src/main/java/org/tron/plugins/Toolkit.java @@ -3,13 +3,18 @@ import java.util.concurrent.Callable; import picocli.CommandLine; -@CommandLine.Command(name = "tron", subcommands = { CommandLine.HelpCommand.class, Db.class}) +@CommandLine.Command(subcommands = { CommandLine.HelpCommand.class, Db.class}) public class Toolkit implements Callable { public static void main(String[] args) { - int exitCode = new CommandLine(new Toolkit()).execute(args); - System.exit(exitCode); + CommandLine cli = new CommandLine(new Toolkit()); + if (args == null || args.length == 0) { + cli.usage(System.out); + } else { + int exitCode = cli.execute(args); + System.exit(exitCode); + } } @Override From cac3e60a556c77fdc8aed2386a30bf2d0492a3e2 Mon Sep 17 00:00:00 2001 From: wubin01 Date: Mon, 1 Aug 2022 18:09:57 +0800 Subject: [PATCH 24/51] fix(net): solve the null pointer problem caused by the change of peer ID --- .../org/tron/core/net/service/AdvService.java | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/framework/src/main/java/org/tron/core/net/service/AdvService.java b/framework/src/main/java/org/tron/core/net/service/AdvService.java index 611e0f47a1f..2df3598f611 100644 --- a/framework/src/main/java/org/tron/core/net/service/AdvService.java +++ b/framework/src/main/java/org/tron/core/net/service/AdvService.java @@ -333,13 +333,19 @@ public void add(Entry id, PeerConnection peer) { } public void add(Item id, PeerConnection peer) { - if (send.containsKey(peer) && !send.get(peer).containsKey(id.getType())) { - send.get(peer).put(id.getType(), new LinkedList<>()); - } else if (!send.containsKey(peer)) { - send.put(peer, new HashMap<>()); - send.get(peer).put(id.getType(), new LinkedList<>()); + HashMap> map = send.get(peer); + if (map == null) { + map = new HashMap<>(); + send.put(peer, map); } - send.get(peer).get(id.getType()).offer(id.getHash()); + + LinkedList list = map.get(id.getType()); + if (list == null) { + list = new LinkedList<>(); + map.put(id.getType(), list); + } + + list.offer(id.getHash()); } public int getSize(PeerConnection peer) { From 44a5ce275dda08f2da751ed7310a43f44374e158 Mon Sep 17 00:00:00 2001 From: jiangyuanshu <317787106@qq.com> Date: Mon, 1 Aug 2022 18:48:02 +0800 Subject: [PATCH 25/51] merge pr 4536 4537 --- .../common/overlay/server/ChannelManager.java | 16 ++++++++-------- .../tron/common/overlay/server/MessageQueue.java | 11 +++++++++++ .../org/tron/core/net/service/SyncService.java | 8 ++++---- 3 files changed, 23 insertions(+), 12 deletions(-) diff --git a/framework/src/main/java/org/tron/common/overlay/server/ChannelManager.java b/framework/src/main/java/org/tron/common/overlay/server/ChannelManager.java index 114df13d847..1bed907c270 100644 --- a/framework/src/main/java/org/tron/common/overlay/server/ChannelManager.java +++ b/framework/src/main/java/org/tron/common/overlay/server/ChannelManager.java @@ -34,7 +34,7 @@ @Component public class ChannelManager { - private final Map activePeers = new ConcurrentHashMap<>(); + private final Map activeChannels = new ConcurrentHashMap<>(); @Autowired private PeerServer peerServer; @Autowired @@ -124,7 +124,7 @@ public void processDisconnect(Channel channel, ReasonCode reason) { public void notifyDisconnect(Channel channel) { syncPool.onDisconnect(channel); - activePeers.values().remove(channel); + activeChannels.values().remove(channel); if (channel != null) { if (channel.getNodeStatistics() != null) { channel.getNodeStatistics().notifyDisconnect(); @@ -149,7 +149,7 @@ public synchronized boolean processPeer(Channel peer) { return false; } - if (!peer.isActive() && activePeers.size() >= maxConnections) { + if (!peer.isActive() && activeChannels.size() >= maxConnections) { peer.disconnect(TOO_MANY_PEERS); return false; } @@ -160,7 +160,7 @@ public synchronized boolean processPeer(Channel peer) { } } - Channel channel = activePeers.get(peer.getNodeIdWrapper()); + Channel channel = activeChannels.get(peer.getNodeIdWrapper()); if (channel != null) { if (channel.getStartTime() > peer.getStartTime()) { logger.info("Disconnect connection established later, {}", channel.getNode()); @@ -170,14 +170,14 @@ public synchronized boolean processPeer(Channel peer) { return false; } } - activePeers.put(peer.getNodeIdWrapper(), peer); - logger.info("Add active peer {}, total active peers: {}", peer, activePeers.size()); + activeChannels.put(peer.getNodeIdWrapper(), peer); + logger.info("Add active peer {}, total active peers: {}", peer, activeChannels.size()); return true; } public int getConnectionNum(InetAddress inetAddress) { int cnt = 0; - for (Channel channel : activePeers.values()) { + for (Channel channel : activeChannels.values()) { if (channel.getInetAddress().equals(inetAddress)) { cnt++; } @@ -186,7 +186,7 @@ public int getConnectionNum(InetAddress inetAddress) { } public Collection getActivePeers() { - return activePeers.values(); + return activeChannels.values(); } public Cache getRecentlyDisconnected() { diff --git a/framework/src/main/java/org/tron/common/overlay/server/MessageQueue.java b/framework/src/main/java/org/tron/common/overlay/server/MessageQueue.java index 1919042232f..a8db0225be4 100644 --- a/framework/src/main/java/org/tron/common/overlay/server/MessageQueue.java +++ b/framework/src/main/java/org/tron/common/overlay/server/MessageQueue.java @@ -69,6 +69,12 @@ public void activate(ChannelHandlerContext ctx) { continue; } Message msg = msgQueue.take(); + if (channel.isDisconnect()) { + logger.warn("Failed to send to {} as channel has closed, {}", + ctx.channel().remoteAddress(), msg); + msgQueue.clear(); + return; + } ctx.writeAndFlush(msg.getSendData()).addListener((ChannelFutureListener) future -> { if (!future.isSuccess() && !channel.isDisconnect()) { logger.warn("Failed to send to {}, {}", ctx.channel().remoteAddress(), msg); @@ -92,6 +98,11 @@ public void setChannel(Channel channel) { } public void fastSend(Message msg) { + if (channel.isDisconnect()) { + logger.warn("Fast send to {} failed as channel has closed, {} ", + ctx.channel().remoteAddress(), msg); + return; + } logger.info("Fast send to {}, {} ", ctx.channel().remoteAddress(), msg); ctx.writeAndFlush(msg.getSendData()).addListener((ChannelFutureListener) future -> { if (!future.isSuccess() && !channel.isDisconnect()) { diff --git a/framework/src/main/java/org/tron/core/net/service/SyncService.java b/framework/src/main/java/org/tron/core/net/service/SyncService.java index 49dc9ce2d7a..a845636784f 100644 --- a/framework/src/main/java/org/tron/core/net/service/SyncService.java +++ b/framework/src/main/java/org/tron/core/net/service/SyncService.java @@ -238,8 +238,8 @@ private synchronized void handleSyncBlock() { isProcessed[0] = false; - synchronized (tronNetDelegate.getBlockLock()) { - blockWaitToProcess.forEach((msg, peerConnection) -> { + blockWaitToProcess.forEach((msg, peerConnection) -> { + synchronized (tronNetDelegate.getBlockLock()) { if (peerConnection.isDisconnect()) { blockWaitToProcess.remove(msg); invalid(msg.getBlockId()); @@ -258,8 +258,8 @@ private synchronized void handleSyncBlock() { isProcessed[0] = true; processSyncBlock(msg.getBlockCapsule()); } - }); - } + } + }); } } From e5f84fc0a3204d2f0446cf883b4a774816e081cc Mon Sep 17 00:00:00 2001 From: halibobo1205 Date: Tue, 2 Aug 2022 17:46:04 +0800 Subject: [PATCH 26/51] tool(db):db path move, print error and exit if path or config error. --- .../main/java/org/tron/plugins/DbMove.java | 79 ++++++++++++++++++- 1 file changed, 77 insertions(+), 2 deletions(-) diff --git a/plugins/src/main/java/org/tron/plugins/DbMove.java b/plugins/src/main/java/org/tron/plugins/DbMove.java index d9a2a8b4831..77c3dca26f8 100644 --- a/plugins/src/main/java/org/tron/plugins/DbMove.java +++ b/plugins/src/main/java/org/tron/plugins/DbMove.java @@ -1,6 +1,7 @@ package org.tron.plugins; import com.typesafe.config.Config; +import com.typesafe.config.ConfigFactory; import java.io.File; import java.io.IOException; import java.nio.file.Files; @@ -8,8 +9,10 @@ import java.nio.file.Paths; import java.nio.file.StandardCopyOption; import java.util.Arrays; +import java.util.HashSet; import java.util.List; import java.util.Objects; +import java.util.Set; import java.util.concurrent.Callable; import java.util.stream.Collectors; import me.tongfei.progressbar.ProgressBar; @@ -34,11 +37,12 @@ public class DbMove implements Callable { defaultValue = "output-directory", converter = Db.PathConverter.class, description = "database directory path. Default: ${DEFAULT-VALUE}") - Path database; + static Path database; @CommandLine.Option(names = {"-c", "--config"}, defaultValue = "config.conf", - converter = Db.ConfigConverter.class, + converter = ConfigConverter.class, + order = Integer.MAX_VALUE, description = " config file. Default: ${DEFAULT-VALUE}") Config config; @@ -170,7 +174,78 @@ public Property(String name, Path original, Path destination) throws IOException if (this.original.toFile().isFile()) { throw new IOException(this.original + " is a file!"); } + if (isSymbolicLink(original.toFile())) { + throw new IOException(original + " is symbolicLink!"); + } this.destination = destination.toFile().getCanonicalFile().toPath(); + if (this.destination.toFile().exists()) { + throw new IOException(this.destination + " already exist!"); + } + if (this.destination.equals(this.original)) { + throw new IOException("destination and original can not be same:[" + this.original + "]!"); + } + } + + public boolean isSymbolicLink(File file) throws IOException { + if (file == null) { + throw new NullPointerException("File must not be null"); + } + + File canon; + if (file.getParent() == null) { + canon = file; + } else { + File canonDir = file.getParentFile().getCanonicalFile(); + canon = new File(canonDir, file.getName()); + } + return !canon.getCanonicalFile().equals(canon.getAbsoluteFile()); + } + } + + static class ConfigConverter implements CommandLine.ITypeConverter { + private final Exception notFind = + new IllegalArgumentException("There is no database to be moved,please check."); + + ConfigConverter() { + } + + public Config convert(String value) throws Exception { + File file = Paths.get(value).toFile(); + if (file.exists() && file.isFile()) { + Config config = ConfigFactory.parseFile(Paths.get(value).toFile()); + if (config.hasPath(PROPERTIES_CONFIG_KEY)) { + List dbs = config.getConfigList(PROPERTIES_CONFIG_KEY); + if (dbs.isEmpty()) { + throw notFind; + } + String dbPath = config.hasPath(DB_DIRECTORY_CONFIG_KEY) + ? config.getString(DB_DIRECTORY_CONFIG_KEY) : DEFAULT_DB_DIRECTORY; + + dbs = dbs.stream() + .filter(c -> c.hasPath(NAME_CONFIG_KEY) && c.hasPath(PATH_CONFIG_KEY)) + .collect(Collectors.toList()); + + if (dbs.isEmpty()) { + throw notFind; + } + Set toBeMove = new HashSet<>(); + for (Config c : dbs) { + if (!toBeMove.add(new Property(c.getString(NAME_CONFIG_KEY), + Paths.get(database.toString(), dbPath, c.getString(NAME_CONFIG_KEY)), + Paths.get(c.getString(PATH_CONFIG_KEY), dbPath, + c.getString(NAME_CONFIG_KEY))).name)) { + throw new IllegalArgumentException( + "DB config has duplicate key:[" + c.getString(NAME_CONFIG_KEY) + + "],please check! "); + } + } + } else { + throw notFind; + } + return config; + } else { + throw new IOException("DB config [" + value + "] not exist!"); + } } } } From 2b7db1b714e17ce9d83892256997ebf10e8c6267 Mon Sep 17 00:00:00 2001 From: zhang0125 Date: Thu, 14 Jul 2022 18:46:57 +0800 Subject: [PATCH 27/51] refactor(tx-cache): use Bloom filters to refactor txCacheDB to reduce memory usage --- .../org/tron/core/db2/common/TxCacheDB.java | 192 +++++++++--------- .../tron/common/prometheus/MetricKeys.java | 1 + .../tron/common/prometheus/MetricsGauge.java | 1 + .../org/tron/core/config/args/Storage.java | 13 ++ .../java/org/tron/core/config/args/Args.java | 3 + .../main/java/org/tron/core/db/Manager.java | 5 +- framework/src/main/resources/config.conf | 9 +- .../java/org/tron/core/db/TxCacheDBTest.java | 74 +++++++ 8 files changed, 203 insertions(+), 95 deletions(-) create mode 100644 framework/src/test/java/org/tron/core/db/TxCacheDBTest.java diff --git a/chainbase/src/main/java/org/tron/core/db2/common/TxCacheDB.java b/chainbase/src/main/java/org/tron/core/db2/common/TxCacheDB.java index 88adfaab00f..e72a5fc898e 100644 --- a/chainbase/src/main/java/org/tron/core/db2/common/TxCacheDB.java +++ b/chainbase/src/main/java/org/tron/core/db2/common/TxCacheDB.java @@ -1,25 +1,19 @@ package org.tron.core.db2.common; -import com.google.common.collect.ArrayListMultimap; -import com.google.common.collect.Iterators; -import com.google.common.collect.Maps; -import com.google.common.collect.Multimap; +import com.google.common.hash.BloomFilter; +import com.google.common.hash.Funnels; import com.google.common.primitives.Longs; - import java.nio.file.Paths; -import java.util.ArrayList; -import java.util.Collection; import java.util.Iterator; -import java.util.List; import java.util.Map; import java.util.Map.Entry; -import java.util.Set; -import java.util.WeakHashMap; - import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.ArrayUtils; import org.bouncycastle.util.encoders.Hex; import org.iq80.leveldb.WriteOptions; import org.tron.common.parameter.CommonParameter; +import org.tron.common.prometheus.MetricKeys; +import org.tron.common.prometheus.Metrics; import org.tron.common.storage.leveldb.LevelDbDataSourceImpl; import org.tron.common.storage.rocksdb.RocksDbDataSourceImpl; import org.tron.common.utils.ByteArray; @@ -34,44 +28,68 @@ public class TxCacheDB implements DB, Flusher { // > 65_536(= 2^16) blocks, that is the number of the reference block - private final int BLOCK_COUNT = 70_000; - private final long MAX_SIZE = 65536; - private Map db = new WeakHashMap<>(); - private Multimap blockNumMap = ArrayListMultimap.create(); - private String name; - private RecentTransactionStore recentTransactionStore; + private static final long MAX_BLOCK_SIZE = 65536; + // estimated number transactions in one block + private final int TRANSACTION_COUNT; + + // Since the filter cannot query for specific record information, + // FAKE_TRANSACTION represent the record presence. + private final byte[] FAKE_TRANSACTION = ByteArray.fromLong(0); + + // a pair of bloom filters record the recent transactions + private BloomFilter[] bloomFilters = new BloomFilter[2]; + // filterStartBlock record the start block of the active filter + private long filterStartBlock = 0; + // currentFilterIndex records the index of the active filter + private int currentFilterIndex = 0; + + // record the last metric block to avoid duplication + private long lastMetricBlock = 0; + + private final String name; // add a persistent storage, the store name is: trans-cache // when fullnode startup, transactionCache initializes transactions from this store private DB persistentStore; + // replace persistentStore and optimizes startup performance + private RecentTransactionStore recentTransactionStore; + public TxCacheDB(String name, RecentTransactionStore recentTransactionStore) { this.name = name; + this.TRANSACTION_COUNT = + CommonParameter.getInstance().getStorage().getEstimatedBlockTransactions(); this.recentTransactionStore = recentTransactionStore; int dbVersion = CommonParameter.getInstance().getStorage().getDbVersion(); String dbEngine = CommonParameter.getInstance().getStorage().getDbEngine(); if (dbVersion == 2) { if ("LEVELDB".equals(dbEngine.toUpperCase())) { this.persistentStore = new LevelDB( - new LevelDbDataSourceImpl(StorageUtils.getOutputDirectoryByDbName(name), - name, StorageUtils.getOptionsByDbName(name), - new WriteOptions().sync(CommonParameter.getInstance() - .getStorage().isDbSync()))); + new LevelDbDataSourceImpl(StorageUtils.getOutputDirectoryByDbName(name), + name, StorageUtils.getOptionsByDbName(name), + new WriteOptions().sync(CommonParameter.getInstance() + .getStorage().isDbSync()))); } else if ("ROCKSDB".equals(dbEngine.toUpperCase())) { String parentPath = Paths - .get(StorageUtils.getOutputDirectoryByDbName(name), CommonParameter - .getInstance().getStorage().getDbDirectory()).toString(); + .get(StorageUtils.getOutputDirectoryByDbName(name), CommonParameter + .getInstance().getStorage().getDbDirectory()).toString(); this.persistentStore = new RocksDB( - new RocksDbDataSourceImpl(parentPath, - name, CommonParameter.getInstance() - .getRocksDBCustomSettings())); + new RocksDbDataSourceImpl(parentPath, + name, CommonParameter.getInstance() + .getRocksDBCustomSettings())); } else { throw new RuntimeException("db type is not supported."); } } else { throw new RuntimeException("db version is not supported."); } + + this.bloomFilters[0] = BloomFilter.create(Funnels.byteArrayFunnel(), + MAX_BLOCK_SIZE * TRANSACTION_COUNT); + this.bloomFilters[1] = BloomFilter.create(Funnels.byteArrayFunnel(), + MAX_BLOCK_SIZE * TRANSACTION_COUNT); + init(); } @@ -81,57 +99,52 @@ public TxCacheDB(String name, RecentTransactionStore recentTransactionStore) { private void initCache() { long start = System.currentTimeMillis(); DBIterator iterator = (DBIterator) persistentStore.iterator(); + long persistentSize = 0; while (iterator.hasNext()) { Entry entry = iterator.next(); - byte[] key = entry.getKey(); - byte[] value = entry.getValue(); - if (key == null || value == null) { + if (ArrayUtils.isEmpty(entry.getKey()) || ArrayUtils.isEmpty(entry.getValue())) { return; } - Key k = Key.copyOf(key); - Long v = Longs.fromByteArray(value); - blockNumMap.put(v, k); - db.put(k, v); + bloomFilters[1].put(entry.getKey()); + persistentSize++; } - logger.info("Transaction cache init-1 db-size:{}, blockNumMap-size:{}, cost:{}ms", - db.size(), blockNumMap.size(), System.currentTimeMillis() - start); + logger.info("load transaction cache from persistentStore " + + "db-size:{}, filter-size:{}, filter-fpp:{}, cost:{}ms", + persistentSize, + bloomFilters[1].approximateElementCount(), bloomFilters[1].expectedFpp(), + System.currentTimeMillis() - start); } private void init() { long size = recentTransactionStore.size(); - if (size != MAX_SIZE) { + if (size != MAX_BLOCK_SIZE) { + // 0. load from persistentStore initCache(); - return; } + // 1. load from recentTransactionStore long start = System.currentTimeMillis(); - List l1 = new ArrayList<>(); - List l2 = new ArrayList<>(); - Iterator> iterator = recentTransactionStore.iterator(); - while (iterator.hasNext()) { - l1.add(iterator.next().getValue().getData()); - } - - l1.forEach(v -> l2.add(JsonUtil.json2Obj(new String(v), RecentTransactionItem.class))); + for (Entry bytesCapsuleEntry : recentTransactionStore) { + byte[] data = bytesCapsuleEntry.getValue().getData(); + RecentTransactionItem trx = + JsonUtil.json2Obj(new String(data), RecentTransactionItem.class); - l2.forEach(v -> v.getTransactionIds().forEach(tid -> putTransaction(tid, v.getNum()))); - - logger.info("Transaction cache init-2 db-size:{}, blockNumMap-size:{}, cost:{}ms", - db.size(), blockNumMap.size(), System.currentTimeMillis() - start); - } + trx.getTransactionIds().forEach(tid -> bloomFilters[1].put(Hex.decode(tid))); + } - private void putTransaction(String key, long value) { - Key k = Key.copyOf(Hex.decode(key)); - Long v = Longs.fromByteArray(ByteArray.fromLong(value)); - blockNumMap.put(v, k); - db.put(k, v); + logger.info("load transaction cache from recentTransactionStore" + + " filter-size:{}, filter-fpp:{}, cost:{}ms", + bloomFilters[1].approximateElementCount(), bloomFilters[1].expectedFpp(), + System.currentTimeMillis() - start); } - @Override public byte[] get(byte[] key) { - Long v = db.get(Key.of(key)); - return v == null ? null : Longs.toByteArray(v); + if (!bloomFilters[0].mightContain(key) && !bloomFilters[1].mightContain(key)) { + return null; + } + // this means exist + return FAKE_TRANSACTION; } @Override @@ -140,46 +153,46 @@ public void put(byte[] key, byte[] value) { return; } - Key k = Key.copyOf(key); - Long v = Longs.fromByteArray(value); - blockNumMap.put(v, k); - db.put(k, v); - // put the data into persistent storage - persistentStore.put(key, value); - removeEldest(); - } - - private void removeEldest() { - Set keys = blockNumMap.keySet(); - if (keys.size() > BLOCK_COUNT) { - keys.stream() - .min(Long::compareTo) - .ifPresent(k -> { - Collection trxHashs = blockNumMap.get(k); - // remove transaction from persistentStore, - // if foreach is inefficient, change remove-foreach to remove-batch - trxHashs.forEach(key -> persistentStore.remove(key.getBytes())); - blockNumMap.removeAll(k); - logger.debug("******removeEldest block number:{}, block count:{}", k, keys.size()); - }); + long blockNum = Longs.fromByteArray(value); + if (filterStartBlock == 0) { + // init active filter start block + filterStartBlock = blockNum; + currentFilterIndex = 0; + } else if (blockNum - filterStartBlock > MAX_BLOCK_SIZE) { + // active filter is full + logger.info("active bloomFilters is full (size={} fpp={}), create a new one", + bloomFilters[currentFilterIndex].approximateElementCount(), + bloomFilters[currentFilterIndex].expectedFpp()); + currentFilterIndex ^= 1; + filterStartBlock = blockNum; + bloomFilters[currentFilterIndex] = + BloomFilter.create(Funnels.byteArrayFunnel(), + MAX_BLOCK_SIZE * TRANSACTION_COUNT); + } + bloomFilters[currentFilterIndex].put(key); + + if (lastMetricBlock != blockNum) { + lastMetricBlock = blockNum; + Metrics.gaugeSet(MetricKeys.Gauge.TX_CACHE, + bloomFilters[currentFilterIndex].approximateElementCount(), "count"); + Metrics.gaugeSet(MetricKeys.Gauge.TX_CACHE, + bloomFilters[currentFilterIndex].expectedFpp(), "fpp"); } } @Override public long size() { - return db.size(); + throw new UnsupportedOperationException("TxCacheDB size"); } @Override public boolean isEmpty() { - return db.isEmpty(); + throw new UnsupportedOperationException("TxCacheDB isEmpty"); } @Override public void remove(byte[] key) { - if (key != null) { - db.remove(Key.of(key)); - } + throw new UnsupportedOperationException("TxCacheDB remove"); } @Override @@ -189,8 +202,7 @@ public String getDbName() { @Override public Iterator> iterator() { - return Iterators.transform(db.entrySet().iterator(), - e -> Maps.immutableEntry(e.getKey().getBytes(), Longs.toByteArray(e.getValue()))); + throw new UnsupportedOperationException("TxCacheDB iterator"); } @Override @@ -201,15 +213,13 @@ public void flush(Map batch) { @Override public void close() { reset(); - db = null; - blockNumMap = null; + bloomFilters[0] = null; + bloomFilters[1] = null; persistentStore.close(); } @Override public void reset() { - db.clear(); - blockNumMap.clear(); } @Override diff --git a/common/src/main/java/org/tron/common/prometheus/MetricKeys.java b/common/src/main/java/org/tron/common/prometheus/MetricKeys.java index 8172eaffd8b..a68f379f394 100644 --- a/common/src/main/java/org/tron/common/prometheus/MetricKeys.java +++ b/common/src/main/java/org/tron/common/prometheus/MetricKeys.java @@ -32,6 +32,7 @@ public static class Gauge { public static final String DB_SIZE_BYTES = "tron:db_size_bytes"; public static final String DB_SST_LEVEL = "tron:db_sst_level"; public static final String MANAGER_QUEUE = "tron:manager_queue_size"; + public static final String TX_CACHE = "tron:tx_cache"; private Gauge() { throw new IllegalStateException("Gauge"); diff --git a/common/src/main/java/org/tron/common/prometheus/MetricsGauge.java b/common/src/main/java/org/tron/common/prometheus/MetricsGauge.java index 57461375b75..dc1e5447540 100644 --- a/common/src/main/java/org/tron/common/prometheus/MetricsGauge.java +++ b/common/src/main/java/org/tron/common/prometheus/MetricsGauge.java @@ -18,6 +18,7 @@ class MetricsGauge { init(MetricKeys.Gauge.PEERS, "tron peers.size .", "type"); init(MetricKeys.Gauge.DB_SIZE_BYTES, "tron db size .", "type", "db", "level"); init(MetricKeys.Gauge.DB_SST_LEVEL, "tron db files .", "type", "db", "level"); + init(MetricKeys.Gauge.TX_CACHE, "tron tx cache info.", "type"); } private MetricsGauge() { diff --git a/common/src/main/java/org/tron/core/config/args/Storage.java b/common/src/main/java/org/tron/core/config/args/Storage.java index 021accb808a..75f8d9b0ddb 100644 --- a/common/src/main/java/org/tron/core/config/args/Storage.java +++ b/common/src/main/java/org/tron/core/config/args/Storage.java @@ -48,6 +48,8 @@ public class Storage { private static final String INDEX_DIRECTORY_CONFIG_KEY = "storage.index.directory"; private static final String INDEX_SWITCH_CONFIG_KEY = "storage.index.switch"; private static final String TRANSACTIONHISTORY_SWITCH_CONFIG_KEY = "storage.transHistory.switch"; + private static final String ESTIMATED_TRANSACTIONS_CONFIG_KEY = + "storage.txCache.estimatedTransactions"; private static final String PROPERTIES_CONFIG_KEY = "storage.properties"; private static final String PROPERTIES_CONFIG_DB_KEY = "storage"; private static final String PROPERTIES_CONFIG_DEFAULT_KEY = "default"; @@ -77,6 +79,7 @@ public class Storage { private static final String DEFAULT_DB_DIRECTORY = "database"; private static final String DEFAULT_INDEX_DIRECTORY = "index"; private static final String DEFAULT_INDEX_SWITCH = "on"; + private static final int DEFAULT_ESTIMATED_TRANSACTIONS = 500; private Config storage; /** @@ -119,6 +122,10 @@ public class Storage { private Options defaultDbOptions; + @Getter + @Setter + private int estimatedBlockTransactions; + /** * Key: dbName, Value: Property object of that database */ @@ -168,6 +175,12 @@ public static String getTransactionHistorySwitchFromConfig(final Config config) : DEFAULT_TRANSACTIONHISTORY_SWITCH; } + public static int getEstimatedTransactionsFromConfig(final Config config) { + return config.hasPath(ESTIMATED_TRANSACTIONS_CONFIG_KEY) + ? config.getInt(ESTIMATED_TRANSACTIONS_CONFIG_KEY) + : DEFAULT_ESTIMATED_TRANSACTIONS; + } + private Property createProperty(final ConfigObject conf) { Property property = new Property(); diff --git a/framework/src/main/java/org/tron/core/config/args/Args.java b/framework/src/main/java/org/tron/core/config/args/Args.java index 0e2482e7f37..a66f56e6067 100644 --- a/framework/src/main/java/org/tron/core/config/args/Args.java +++ b/framework/src/main/java/org/tron/core/config/args/Args.java @@ -406,6 +406,9 @@ public static void setParam(final String[] args, final String confFileName) { .filter(StringUtils::isNotEmpty) .orElse(Storage.getTransactionHistorySwitchFromConfig(config))); + PARAMETER.storage.setEstimatedBlockTransactions( + Storage.getEstimatedTransactionsFromConfig(config)); + PARAMETER.storage.setDefaultDbOptions(config); PARAMETER.storage.setPropertyMapFromConfig(config); diff --git a/framework/src/main/java/org/tron/core/db/Manager.java b/framework/src/main/java/org/tron/core/db/Manager.java index da8b7491248..06fe9c62d4e 100644 --- a/framework/src/main/java/org/tron/core/db/Manager.java +++ b/framework/src/main/java/org/tron/core/db/Manager.java @@ -740,8 +740,9 @@ private boolean containsTransaction(TransactionCapsule transactionCapsule) { private boolean containsTransaction(byte[] transactionId) { - if (transactionCache != null) { - return transactionCache.has(transactionId); + if (transactionCache != null && !transactionCache.has(transactionId)) { + // using the bloom filter only determines non-existent transaction + return false; } return chainBaseManager.getTransactionStore() diff --git a/framework/src/main/resources/config.conf b/framework/src/main/resources/config.conf index 74f148e1405..c12a2740b76 100644 --- a/framework/src/main/resources/config.conf +++ b/framework/src/main/resources/config.conf @@ -89,6 +89,11 @@ storage { } balance.history.lookup = false + + # the estimated number of block transactions (default 500). + # so the total number of cached transactions is 65536 * txCache.estimatedTransactions + # txCache.estimatedTransactions = 500 + } node.discovery = { @@ -124,12 +129,12 @@ crypto { engine = "eckey" } # prometheus metrics start -#node.metrics = { +# node.metrics = { # prometheus{ # enable=true # port="9527" # } -#} +# } # prometheus metrics end diff --git a/framework/src/test/java/org/tron/core/db/TxCacheDBTest.java b/framework/src/test/java/org/tron/core/db/TxCacheDBTest.java new file mode 100644 index 00000000000..6f627ab535e --- /dev/null +++ b/framework/src/test/java/org/tron/core/db/TxCacheDBTest.java @@ -0,0 +1,74 @@ +package org.tron.core.db; + +import java.io.File; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; +import org.tron.common.application.Application; +import org.tron.common.application.ApplicationFactory; +import org.tron.common.application.TronApplicationContext; +import org.tron.common.utils.ByteArray; +import org.tron.common.utils.FileUtil; +import org.tron.core.Constant; +import org.tron.core.capsule.BytesCapsule; +import org.tron.core.config.DefaultConfig; +import org.tron.core.config.args.Args; +import org.tron.keystore.Wallet; + +public class TxCacheDBTest { + private static final String dbPath = "output_TransactionCache_test"; + + private static TronApplicationContext context; + private static Manager dbManager; + + /** + * Init data. + */ + @BeforeClass + public static void init() { + String dbDirectory = "db_TransactionCache_test"; + String indexDirectory = "index_TransactionCache_test"; + Args.setParam(new String[]{"--output-directory", dbPath, "--storage-db-directory", + dbDirectory, "--storage-index-directory", indexDirectory, "-w"}, Constant.TEST_CONF); + context = new TronApplicationContext(DefaultConfig.class); + Application appT = ApplicationFactory.create(context); + dbManager = context.getBean(Manager.class); + } + + /** + * release resources. + */ + @AfterClass + public static void destroy() { + Args.clearParam(); + context.destroy(); + FileUtil.deleteDir(new File(dbPath)); + } + + @Test + public void putTransactionTest() { + TransactionCache db = dbManager.getTransactionCache(); + byte[][] hash = new byte[140000][64]; + for (int i = 1; i < 140000; i++) { + hash[i] = Wallet.generateRandomBytes(64); + db.put(hash[i], new BytesCapsule(ByteArray.fromLong(i))); + } + // [1,63357] are expired + for (int i = 1; i < 65538; i++) { + try { + Assert.assertFalse("index = " + i, db.has(hash[i])); + } catch (Exception e) { + Assert.fail("transaction should be expired index = " + i); + } + } + // [63358,140000] are in cache + for (int i = 65538; i < 140000; i++) { + try { + Assert.assertTrue("index = " + i, db.has(hash[i])); + } catch (Exception e) { + Assert.fail("transaction should not be expired index = " + i); + } + } + } +} \ No newline at end of file From 8f3410da114c8be89a1efc7e35edc379f6d642ec Mon Sep 17 00:00:00 2001 From: zhang0125 Date: Thu, 21 Jul 2022 14:08:57 +0800 Subject: [PATCH 28/51] refactor(tx-cache): add log information and modify default parameters --- .../java/org/tron/core/db2/common/TxCacheDB.java | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/chainbase/src/main/java/org/tron/core/db2/common/TxCacheDB.java b/chainbase/src/main/java/org/tron/core/db2/common/TxCacheDB.java index e72a5fc898e..fb38208d2ac 100644 --- a/chainbase/src/main/java/org/tron/core/db2/common/TxCacheDB.java +++ b/chainbase/src/main/java/org/tron/core/db2/common/TxCacheDB.java @@ -32,6 +32,8 @@ public class TxCacheDB implements DB, Flusher { // estimated number transactions in one block private final int TRANSACTION_COUNT; + private static final long INVALID_BLOCK = -1; + // Since the filter cannot query for specific record information, // FAKE_TRANSACTION represent the record presence. private final byte[] FAKE_TRANSACTION = ByteArray.fromLong(0); @@ -39,7 +41,7 @@ public class TxCacheDB implements DB, Flusher { // a pair of bloom filters record the recent transactions private BloomFilter[] bloomFilters = new BloomFilter[2]; // filterStartBlock record the start block of the active filter - private long filterStartBlock = 0; + private long filterStartBlock = INVALID_BLOCK; // currentFilterIndex records the index of the active filter private int currentFilterIndex = 0; @@ -154,15 +156,17 @@ public void put(byte[] key, byte[] value) { } long blockNum = Longs.fromByteArray(value); - if (filterStartBlock == 0) { + if (filterStartBlock == INVALID_BLOCK) { // init active filter start block filterStartBlock = blockNum; currentFilterIndex = 0; + logger.info("init tx cache bloomFilters at {}",blockNum); } else if (blockNum - filterStartBlock > MAX_BLOCK_SIZE) { // active filter is full - logger.info("active bloomFilters is full (size={} fpp={}), create a new one", + logger.info("active bloomFilters is full (size={} fpp={}), create a new one (start={})", bloomFilters[currentFilterIndex].approximateElementCount(), - bloomFilters[currentFilterIndex].expectedFpp()); + bloomFilters[currentFilterIndex].expectedFpp(), + blockNum); currentFilterIndex ^= 1; filterStartBlock = blockNum; bloomFilters[currentFilterIndex] = @@ -228,8 +232,6 @@ public TxCacheDB newInstance() { } @Override - public void stat() { - this.persistentStore.stat(); - } + public void stat() {} } From e0be5999c7c6f387b7f4860e47c3046aa1704dd1 Mon Sep 17 00:00:00 2001 From: zhang0125 Date: Thu, 21 Jul 2022 17:44:31 +0800 Subject: [PATCH 29/51] refactor(tx-cache): recover partial transaction db for light nodes --- .../org/tron/core/db/TransactionStore.java | 3 ++ .../main/java/org/tron/core/db/Manager.java | 40 +++++++++++++++++++ 2 files changed, 43 insertions(+) diff --git a/chainbase/src/main/java/org/tron/core/db/TransactionStore.java b/chainbase/src/main/java/org/tron/core/db/TransactionStore.java index 342550ecb4a..c0067b77934 100644 --- a/chainbase/src/main/java/org/tron/core/db/TransactionStore.java +++ b/chainbase/src/main/java/org/tron/core/db/TransactionStore.java @@ -52,6 +52,9 @@ private TransactionCapsule getTransactionFromBlockStore(byte[] key, long blockNu private TransactionCapsule getTransactionFromKhaosDatabase(byte[] key, long high) { List khaosBlocks = khaosDatabase.getMiniStore().getBlockByNum(high); + if (khaosBlocks == null) { + return null; + } for (KhaosBlock bl : khaosBlocks) { for (TransactionCapsule e : bl.getBlk().getTransactions()) { if (e.getTransactionId().equals(Sha256Hash.wrap(key))) { diff --git a/framework/src/main/java/org/tron/core/db/Manager.java b/framework/src/main/java/org/tron/core/db/Manager.java index 06fe9c62d4e..57adaa14df2 100644 --- a/framework/src/main/java/org/tron/core/db/Manager.java +++ b/framework/src/main/java/org/tron/core/db/Manager.java @@ -19,6 +19,7 @@ import java.util.HashSet; import java.util.LinkedList; import java.util.List; +import java.util.Map; import java.util.Objects; import java.util.Optional; import java.util.Set; @@ -501,6 +502,9 @@ public void init() { //for test only chainBaseManager.getDynamicPropertiesStore().updateDynamicStoreByConfig(); + // init liteFullNode + initLiteNode(); + long headNum = chainBaseManager.getDynamicPropertiesStore().getLatestBlockHeaderNumber(); logger.info("current headNum is: {}", headNum); int nodeType = chainBaseManager.getCommonStore().getNodeType(); @@ -2272,6 +2276,42 @@ public long getPendingSize() { return value; } + private void initLiteNode() { + // When using bloom filter for transaction de-duplication, + // it is possible to use trans for secondary confirmation. + // Init trans db for liteNode, + long headNum = chainBaseManager.getDynamicPropertiesStore().getLatestBlockHeaderNumber(); + long recentBlockCount = chainBaseManager.getRecentBlockStore().size(); + long recentBlockStart = headNum - recentBlockCount + 1; + try { + chainBaseManager.getBlockByNum(recentBlockStart); + } catch (ItemNotFoundException | BadItemException e) { + // copy transaction from recent-transaction to trans + logger.info("load trans for lite node. from={},to={}", recentBlockStart, headNum); + + TransactionCapsule item = new TransactionCapsule(Transaction.newBuilder().build()); + // add a fake number, tx Cache check if transaction exist + item.setBlockNum(0); + + long transactionCount = 0; + for (Map.Entry entry : + chainBaseManager.getRecentTransactionStore()) { + byte[] data = entry.getValue().getData(); + RecentTransactionItem trx = + JsonUtil.json2Obj(new String(data), RecentTransactionItem.class); + if (trx == null) { + continue; + } + transactionCount += trx.getTransactionIds().size(); + + trx.getTransactionIds().forEach( + tid -> chainBaseManager.getTransactionStore().put(Hex.decode(tid), item)); + } + logger.info("load trans complete, trans:{}.", transactionCount); + } + } + + public void setBlockWaitLock(boolean waitFlag) { if (waitFlag) { blockWaitLock.incrementAndGet(); From 8198fb9434d288e6424ce9a78f81c6b54ef69caa Mon Sep 17 00:00:00 2001 From: zhang0125 Date: Mon, 1 Aug 2022 18:00:21 +0800 Subject: [PATCH 30/51] refactor(tx-cache): add volatile to increase multi-threaded safety --- .../org/tron/core/db2/common/TxCacheDB.java | 18 ++++++++++++------ .../java/org/tron/core/db/TxCacheDBTest.java | 4 ++-- 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/chainbase/src/main/java/org/tron/core/db2/common/TxCacheDB.java b/chainbase/src/main/java/org/tron/core/db2/common/TxCacheDB.java index fb38208d2ac..02f8f06fd2e 100644 --- a/chainbase/src/main/java/org/tron/core/db2/common/TxCacheDB.java +++ b/chainbase/src/main/java/org/tron/core/db2/common/TxCacheDB.java @@ -41,9 +41,9 @@ public class TxCacheDB implements DB, Flusher { // a pair of bloom filters record the recent transactions private BloomFilter[] bloomFilters = new BloomFilter[2]; // filterStartBlock record the start block of the active filter - private long filterStartBlock = INVALID_BLOCK; + private volatile long filterStartBlock = INVALID_BLOCK; // currentFilterIndex records the index of the active filter - private int currentFilterIndex = 0; + private volatile int currentFilterIndex = 0; // record the last metric block to avoid duplication private long lastMetricBlock = 0; @@ -86,7 +86,6 @@ public TxCacheDB(String name, RecentTransactionStore recentTransactionStore) { } else { throw new RuntimeException("db version is not supported."); } - this.bloomFilters[0] = BloomFilter.create(Funnels.byteArrayFunnel(), MAX_BLOCK_SIZE * TRANSACTION_COUNT); this.bloomFilters[1] = BloomFilter.create(Funnels.byteArrayFunnel(), @@ -160,14 +159,20 @@ public void put(byte[] key, byte[] value) { // init active filter start block filterStartBlock = blockNum; currentFilterIndex = 0; - logger.info("init tx cache bloomFilters at {}",blockNum); + logger.info("init tx cache bloomFilters at {}", blockNum); } else if (blockNum - filterStartBlock > MAX_BLOCK_SIZE) { // active filter is full logger.info("active bloomFilters is full (size={} fpp={}), create a new one (start={})", bloomFilters[currentFilterIndex].approximateElementCount(), bloomFilters[currentFilterIndex].expectedFpp(), blockNum); - currentFilterIndex ^= 1; + + if (currentFilterIndex == 0) { + currentFilterIndex = 1; + } else { + currentFilterIndex = 0; + } + filterStartBlock = blockNum; bloomFilters[currentFilterIndex] = BloomFilter.create(Funnels.byteArrayFunnel(), @@ -232,6 +237,7 @@ public TxCacheDB newInstance() { } @Override - public void stat() {} + public void stat() { + } } diff --git a/framework/src/test/java/org/tron/core/db/TxCacheDBTest.java b/framework/src/test/java/org/tron/core/db/TxCacheDBTest.java index 6f627ab535e..7d9bbcd5ad2 100644 --- a/framework/src/test/java/org/tron/core/db/TxCacheDBTest.java +++ b/framework/src/test/java/org/tron/core/db/TxCacheDBTest.java @@ -54,7 +54,7 @@ public void putTransactionTest() { hash[i] = Wallet.generateRandomBytes(64); db.put(hash[i], new BytesCapsule(ByteArray.fromLong(i))); } - // [1,63357] are expired + // [1,65537] are expired for (int i = 1; i < 65538; i++) { try { Assert.assertFalse("index = " + i, db.has(hash[i])); @@ -62,7 +62,7 @@ public void putTransactionTest() { Assert.fail("transaction should be expired index = " + i); } } - // [63358,140000] are in cache + // [65538,140000] are in cache for (int i = 65538; i < 140000; i++) { try { Assert.assertTrue("index = " + i, db.has(hash[i])); From 7d17172761fb1507dbf86151a36f4b407926c863 Mon Sep 17 00:00:00 2001 From: zhang0125 Date: Tue, 2 Aug 2022 19:30:21 +0800 Subject: [PATCH 31/51] refactor(tx-cache): limit param estimatedBlockTransactions --- .../java/org/tron/core/config/args/Storage.java | 13 ++++++++++--- .../src/main/java/org/tron/core/db/Manager.java | 14 +++++++++----- framework/src/main/resources/config.conf | 2 +- 3 files changed, 20 insertions(+), 9 deletions(-) diff --git a/common/src/main/java/org/tron/core/config/args/Storage.java b/common/src/main/java/org/tron/core/config/args/Storage.java index 75f8d9b0ddb..60e3cd9ba6d 100644 --- a/common/src/main/java/org/tron/core/config/args/Storage.java +++ b/common/src/main/java/org/tron/core/config/args/Storage.java @@ -176,9 +176,16 @@ public static String getTransactionHistorySwitchFromConfig(final Config config) } public static int getEstimatedTransactionsFromConfig(final Config config) { - return config.hasPath(ESTIMATED_TRANSACTIONS_CONFIG_KEY) - ? config.getInt(ESTIMATED_TRANSACTIONS_CONFIG_KEY) - : DEFAULT_ESTIMATED_TRANSACTIONS; + if (!config.hasPath(ESTIMATED_TRANSACTIONS_CONFIG_KEY)) { + return DEFAULT_ESTIMATED_TRANSACTIONS; + } + int estimatedTransactions = config.getInt(ESTIMATED_TRANSACTIONS_CONFIG_KEY); + if (estimatedTransactions > 10000) { + estimatedTransactions = 10000; + } else if (estimatedTransactions < 100) { + estimatedTransactions = 100; + } + return estimatedTransactions; } private Property createProperty(final ConfigObject conf) { diff --git a/framework/src/main/java/org/tron/core/db/Manager.java b/framework/src/main/java/org/tron/core/db/Manager.java index 57adaa14df2..0d25dadf48e 100644 --- a/framework/src/main/java/org/tron/core/db/Manager.java +++ b/framework/src/main/java/org/tron/core/db/Manager.java @@ -2287,13 +2287,13 @@ private void initLiteNode() { chainBaseManager.getBlockByNum(recentBlockStart); } catch (ItemNotFoundException | BadItemException e) { // copy transaction from recent-transaction to trans - logger.info("load trans for lite node. from={},to={}", recentBlockStart, headNum); + logger.info("load trans for lite node."); TransactionCapsule item = new TransactionCapsule(Transaction.newBuilder().build()); - // add a fake number, tx Cache check if transaction exist - item.setBlockNum(0); long transactionCount = 0; + long minBlock = Long.MAX_VALUE; + long maxBlock = Long.MIN_VALUE; for (Map.Entry entry : chainBaseManager.getRecentTransactionStore()) { byte[] data = entry.getValue().getData(); @@ -2303,11 +2303,15 @@ private void initLiteNode() { continue; } transactionCount += trx.getTransactionIds().size(); - + long blockNum = trx.getNum(); + maxBlock = Math.max(maxBlock, blockNum); + minBlock = Math.min(minBlock, blockNum); + item.setBlockNum(blockNum); trx.getTransactionIds().forEach( tid -> chainBaseManager.getTransactionStore().put(Hex.decode(tid), item)); } - logger.info("load trans complete, trans:{}.", transactionCount); + logger.info("load trans complete, trans:{},from={},to={}", + transactionCount, minBlock, maxBlock); } } diff --git a/framework/src/main/resources/config.conf b/framework/src/main/resources/config.conf index c12a2740b76..93259dfffa6 100644 --- a/framework/src/main/resources/config.conf +++ b/framework/src/main/resources/config.conf @@ -90,7 +90,7 @@ storage { balance.history.lookup = false - # the estimated number of block transactions (default 500). + # the estimated number of block transactions (default 500, min 100, max 10000). # so the total number of cached transactions is 65536 * txCache.estimatedTransactions # txCache.estimatedTransactions = 500 From 299d68573d0a3d4fc3433b79795bdbb0dd07166f Mon Sep 17 00:00:00 2001 From: wubin01 Date: Wed, 3 Aug 2022 11:00:57 +0800 Subject: [PATCH 32/51] perf(net): compatible network configuration --- .../src/main/java/org/tron/core/Constant.java | 7 ++- .../java/org/tron/core/config/args/Args.java | 45 ++++++++++++++----- framework/src/main/resources/config.conf | 2 +- 3 files changed, 40 insertions(+), 14 deletions(-) diff --git a/common/src/main/java/org/tron/core/Constant.java b/common/src/main/java/org/tron/core/Constant.java index 1eb10792600..883e4cd3f6a 100644 --- a/common/src/main/java/org/tron/core/Constant.java +++ b/common/src/main/java/org/tron/core/Constant.java @@ -87,7 +87,12 @@ public class Constant { public static final String NODE_MIN_CONNECTIONS = "node.minConnections"; public static final String NODE_MIN_ACTIVE_CONNECTIONS = "node.minActiveConnections"; - public static final String NODE_MAX_ACTIVE_NODES_WITH_SAMEIP = "node.maxConnectionsWithSameIp"; + public static final String NODE_MAX_ACTIVE_NODES = "node.maxActiveNodes"; + public static final String NODE_MAX_ACTIVE_NODES_WITH_SAME_IP = "node.maxActiveNodesWithSameIp"; + public static final String NODE_CONNECT_FACTOR = "node.connectFactor"; + public static final String NODE_ACTIVE_CONNECT_FACTOR = "node.activeConnectFactor"; + + public static final String NODE_MAX_CONNECTIONS_WITH_SAME_IP = "node.maxConnectionsWithSameIp"; public static final String NODE_MIN_PARTICIPATION_RATE = "node.minParticipationRate"; public static final String NODE_LISTEN_PORT = "node.listen.port"; public static final String NODE_DISCOVERY_PUBLIC_HOME_NODE = "node.discovery.public.home.node"; diff --git a/framework/src/main/java/org/tron/core/config/args/Args.java b/framework/src/main/java/org/tron/core/config/args/Args.java index 0e2482e7f37..dd53ac7e220 100644 --- a/framework/src/main/java/org/tron/core/config/args/Args.java +++ b/framework/src/main/java/org/tron/core/config/args/Args.java @@ -463,21 +463,42 @@ public static void setParam(final String[] args, final String confFileName) { ? config.getInt(Constant.NODE_CHANNEL_READ_TIMEOUT) : 0; - PARAMETER.maxConnections = - config.hasPath(Constant.NODE_MAX_CONNECTIONS) - ? config.getInt(Constant.NODE_MAX_CONNECTIONS) : 30; + if (config.hasPath(Constant.NODE_MAX_ACTIVE_NODES)) { + PARAMETER.maxConnections = config.getInt(Constant.NODE_MAX_ACTIVE_NODES); + } else { + PARAMETER.maxConnections = + config.hasPath(Constant.NODE_MAX_CONNECTIONS) + ? config.getInt(Constant.NODE_MAX_CONNECTIONS) : 30; + } - PARAMETER.minConnections = - config.hasPath(Constant.NODE_MIN_CONNECTIONS) - ? config.getInt(Constant.NODE_MIN_CONNECTIONS) : 8; + if (config.hasPath(Constant.NODE_MAX_ACTIVE_NODES) + && config.hasPath(Constant.NODE_CONNECT_FACTOR)) { + PARAMETER.minConnections = (int) (PARAMETER.maxConnections + * config.getDouble(Constant.NODE_CONNECT_FACTOR)); + } else { + PARAMETER.minConnections = + config.hasPath(Constant.NODE_MIN_CONNECTIONS) + ? config.getInt(Constant.NODE_MIN_CONNECTIONS) : 8; + } - PARAMETER.minActiveConnections = - config.hasPath(Constant.NODE_MIN_ACTIVE_CONNECTIONS) - ? config.getInt(Constant.NODE_MIN_ACTIVE_CONNECTIONS) : 3; + if (config.hasPath(Constant.NODE_MAX_ACTIVE_NODES) + && config.hasPath(Constant.NODE_ACTIVE_CONNECT_FACTOR)) { + PARAMETER.minActiveConnections = (int) (PARAMETER.maxConnections + * config.getDouble(Constant.NODE_ACTIVE_CONNECT_FACTOR)); + } else { + PARAMETER.minActiveConnections = + config.hasPath(Constant.NODE_MIN_ACTIVE_CONNECTIONS) + ? config.getInt(Constant.NODE_MIN_ACTIVE_CONNECTIONS) : 3; + } - PARAMETER.maxConnectionsWithSameIp = - config.hasPath(Constant.NODE_MAX_ACTIVE_NODES_WITH_SAMEIP) ? config - .getInt(Constant.NODE_MAX_ACTIVE_NODES_WITH_SAMEIP) : 2; + if (config.hasPath(Constant.NODE_MAX_ACTIVE_NODES_WITH_SAME_IP)) { + PARAMETER.maxConnectionsWithSameIp = + config.getInt(Constant.NODE_MAX_ACTIVE_NODES_WITH_SAME_IP); + } else { + PARAMETER.maxConnectionsWithSameIp = + config.hasPath(Constant.NODE_MAX_CONNECTIONS_WITH_SAME_IP) ? config + .getInt(Constant.NODE_MAX_CONNECTIONS_WITH_SAME_IP) : 2; + } PARAMETER.minParticipationRate = config.hasPath(Constant.NODE_MIN_PARTICIPATION_RATE) diff --git a/framework/src/main/resources/config.conf b/framework/src/main/resources/config.conf index 74f148e1405..e5534594a2b 100644 --- a/framework/src/main/resources/config.conf +++ b/framework/src/main/resources/config.conf @@ -164,7 +164,7 @@ node { minParticipationRate = 15 - isOpenFullTcpDisconnect = true + isOpenFullTcpDisconnect = false p2p { version = 11111 # 11111: mainnet; 20180622: testnet From 45f01193197906241bf23c16e00297767e6765ac Mon Sep 17 00:00:00 2001 From: liukai Date: Thu, 4 Aug 2022 14:45:27 +0800 Subject: [PATCH 33/51] feat(log): modify log output level 1. modify unnecessary error log output --- .../java/org/tron/common/overlay/server/MessageQueue.java | 2 +- framework/src/main/java/org/tron/core/Wallet.java | 8 ++++---- .../src/main/java/org/tron/core/net/TronNetService.java | 4 ++-- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/framework/src/main/java/org/tron/common/overlay/server/MessageQueue.java b/framework/src/main/java/org/tron/common/overlay/server/MessageQueue.java index a8db0225be4..9b702dac915 100644 --- a/framework/src/main/java/org/tron/common/overlay/server/MessageQueue.java +++ b/framework/src/main/java/org/tron/common/overlay/server/MessageQueue.java @@ -106,7 +106,7 @@ public void fastSend(Message msg) { logger.info("Fast send to {}, {} ", ctx.channel().remoteAddress(), msg); ctx.writeAndFlush(msg.getSendData()).addListener((ChannelFutureListener) future -> { if (!future.isSuccess() && !channel.isDisconnect()) { - logger.error("Fast send to {} failed, {}", ctx.channel().remoteAddress(), msg); + logger.warn("Fast send to {} failed, {}", ctx.channel().remoteAddress(), msg); } }); } diff --git a/framework/src/main/java/org/tron/core/Wallet.java b/framework/src/main/java/org/tron/core/Wallet.java index 4f473f79d24..ac88a898ed9 100755 --- a/framework/src/main/java/org/tron/core/Wallet.java +++ b/framework/src/main/java/org/tron/core/Wallet.java @@ -536,17 +536,17 @@ public GrpcAPI.Return broadcastTransaction(Transaction signedTransaction) { return builder.setResult(true).setCode(response_code.SUCCESS).build(); } } catch (ValidateSignatureException e) { - logger.error(BROADCAST_TRANS_FAILED, txID, e.getMessage()); + logger.warn(BROADCAST_TRANS_FAILED, txID, e.getMessage()); return builder.setResult(false).setCode(response_code.SIGERROR) .setMessage(ByteString.copyFromUtf8("Validate signature error: " + e.getMessage())) .build(); } catch (ContractValidateException e) { - logger.error(BROADCAST_TRANS_FAILED, txID, e.getMessage()); + logger.warn(BROADCAST_TRANS_FAILED, txID, e.getMessage()); return builder.setResult(false).setCode(response_code.CONTRACT_VALIDATE_ERROR) .setMessage(ByteString.copyFromUtf8(CONTRACT_VALIDATE_ERROR + e.getMessage())) .build(); } catch (ContractExeException e) { - logger.error(BROADCAST_TRANS_FAILED, txID, e.getMessage()); + logger.warn(BROADCAST_TRANS_FAILED, txID, e.getMessage()); return builder.setResult(false).setCode(response_code.CONTRACT_EXE_ERROR) .setMessage(ByteString.copyFromUtf8("Contract execute error : " + e.getMessage())) .build(); @@ -556,7 +556,7 @@ public GrpcAPI.Return broadcastTransaction(Transaction signedTransaction) { .setMessage(ByteString.copyFromUtf8("Account resource insufficient error.")) .build(); } catch (DupTransactionException e) { - logger.error(BROADCAST_TRANS_FAILED, txID, e.getMessage()); + logger.warn(BROADCAST_TRANS_FAILED, txID, e.getMessage()); return builder.setResult(false).setCode(response_code.DUP_TRANSACTION_ERROR) .setMessage(ByteString.copyFromUtf8("Dup transaction.")) .build(); diff --git a/framework/src/main/java/org/tron/core/net/TronNetService.java b/framework/src/main/java/org/tron/core/net/TronNetService.java index 250a9a444b0..69d1bc0cebe 100644 --- a/framework/src/main/java/org/tron/core/net/TronNetService.java +++ b/framework/src/main/java/org/tron/core/net/TronNetService.java @@ -153,11 +153,11 @@ private void processException(PeerConnection peer, TronMessage msg, Exception ex code = ReasonCode.UNKNOWN; break; } - logger.error("Message from {} process failed, {} \n type: {}, detail: {}.", + logger.warn("Message from {} process failed, {} \n type: {}, detail: {}.", peer.getInetAddress(), msg, type, ex.getMessage()); } else { code = ReasonCode.UNKNOWN; - logger.error("Message from {} process failed, {}", + logger.warn("Message from {} process failed, {}", peer.getInetAddress(), msg, ex); } From a077c5daa43dbba162ea290f3d4f5c7b176be10d Mon Sep 17 00:00:00 2001 From: wubin01 Date: Thu, 4 Aug 2022 16:03:13 +0800 Subject: [PATCH 34/51] perf(net): optimize disconnect logic --- .../common/overlay/server/PeerConnectionCheckService.java | 6 +----- framework/src/main/java/org/tron/core/config/args/Args.java | 2 ++ 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/framework/src/main/java/org/tron/common/overlay/server/PeerConnectionCheckService.java b/framework/src/main/java/org/tron/common/overlay/server/PeerConnectionCheckService.java index edd18bd9ed0..3aafa4a8e95 100644 --- a/framework/src/main/java/org/tron/common/overlay/server/PeerConnectionCheckService.java +++ b/framework/src/main/java/org/tron/common/overlay/server/PeerConnectionCheckService.java @@ -18,9 +18,7 @@ @Slf4j(topic = "net") @Service public class PeerConnectionCheckService { - private int p2pVersion = Args.getInstance().getNodeP2pVersion(); private int maxConnections = Args.getInstance().getMaxConnections(); - private int minActiveConnections = Args.getInstance().getMinActiveConnections(); private boolean isFastForward = Args.getInstance().isFastForward(); private boolean isOpenFullTcpDisconnect = Args.getInstance().isOpenFullTcpDisconnect(); private ScheduledExecutorService poolLoopExecutor = Executors.newSingleThreadScheduledExecutor(); @@ -51,12 +49,10 @@ public void check() { if (syncPool.getActivePeers().size() < maxConnections) { return; } - boolean flag = syncPool.getActivePeersCount().get() > minActiveConnections * 2; Collection peers = syncPool.getActivePeers().stream() .filter(peer -> peer.isIdle()) .filter(peer -> !peer.isTrustPeer()) - .filter(peer -> !peer.getNode().isConnectible(p2pVersion)) - .filter(peer -> peer.isActive() == flag) + .filter(peer -> !peer.isActive()) .collect(Collectors.toList()); if (peers.size() == 0) { return; diff --git a/framework/src/main/java/org/tron/core/config/args/Args.java b/framework/src/main/java/org/tron/core/config/args/Args.java index fa00f42fc32..38ae934877e 100644 --- a/framework/src/main/java/org/tron/core/config/args/Args.java +++ b/framework/src/main/java/org/tron/core/config/args/Args.java @@ -1256,6 +1256,8 @@ public static void logConfig() { logger.info("FastForward node number: {}", parameter.getMaxFastForwardNum()); logger.info("Seed node size: {}", parameter.getSeedNode().getIpList().size()); logger.info("Max connection: {}", parameter.getMaxConnections()); + logger.info("Min connection: {}", parameter.getMinConnections()); + logger.info("Min active connection: {}", parameter.getMinActiveConnections()); logger.info("Max connection with same IP: {}", parameter.getMaxConnectionsWithSameIp()); logger.info("Solidity threads: {}", parameter.getSolidityThreads()); logger.info("Trx reference block: {}", parameter.getTrxReferenceBlock()); From 078943e6c544c5256016a33ead0e0a0b21ffe619 Mon Sep 17 00:00:00 2001 From: Wenhua Zhang Date: Fri, 5 Aug 2022 17:07:01 +0800 Subject: [PATCH 35/51] feat(jsonrpc): if revert happens when executing contract in eth_estimateGas and eth_call, return err msg --- .../logsfilter/ContractEventParser.java | 2 +- .../core/services/jsonrpc/TronJsonRpc.java | 4 +- .../services/jsonrpc/TronJsonRpcImpl.java | 40 ++++++++++++++++--- .../common/logsfilter/EventParserTest.java | 15 +++++++ 4 files changed, 52 insertions(+), 9 deletions(-) diff --git a/framework/src/main/java/org/tron/common/logsfilter/ContractEventParser.java b/framework/src/main/java/org/tron/common/logsfilter/ContractEventParser.java index e0e7961ebf0..b2d8a0d8d3a 100644 --- a/framework/src/main/java/org/tron/common/logsfilter/ContractEventParser.java +++ b/framework/src/main/java/org/tron/common/logsfilter/ContractEventParser.java @@ -17,7 +17,7 @@ public class ContractEventParser { private static final int DATAWORD_UNIT_SIZE = 32; - protected static String parseDataBytes(byte[] data, String typeStr, int index) { + public static String parseDataBytes(byte[] data, String typeStr, int index) { try { byte[] startBytes = subBytes(data, index * DATAWORD_UNIT_SIZE, DATAWORD_UNIT_SIZE); Type type = basicType(typeStr); diff --git a/framework/src/main/java/org/tron/core/services/jsonrpc/TronJsonRpc.java b/framework/src/main/java/org/tron/core/services/jsonrpc/TronJsonRpc.java index d082c7152ea..edbb9377988 100644 --- a/framework/src/main/java/org/tron/core/services/jsonrpc/TronJsonRpc.java +++ b/framework/src/main/java/org/tron/core/services/jsonrpc/TronJsonRpc.java @@ -13,7 +13,6 @@ import lombok.NoArgsConstructor; import lombok.Setter; import lombok.ToString; -import lombok.Value; import org.springframework.stereotype.Component; import org.tron.common.runtime.vm.DataWord; import org.tron.common.utils.ByteArray; @@ -150,9 +149,10 @@ TransactionResult getTransactionByBlockNumberAndIndex(String blockNumOrTag, Stri @JsonRpcMethod("eth_call") @JsonRpcErrors({ @JsonRpcError(exception = JsonRpcInvalidParamsException.class, code = -32602, data = "{}"), + @JsonRpcError(exception = JsonRpcInternalException.class, code = -32000, data = "{}"), }) String getCall(CallArguments transactionCall, String blockNumOrTag) - throws JsonRpcInvalidParamsException; + throws JsonRpcInvalidParamsException, JsonRpcInternalException; @JsonRpcMethod("net_peerCount") String getPeerCount(); diff --git a/framework/src/main/java/org/tron/core/services/jsonrpc/TronJsonRpcImpl.java b/framework/src/main/java/org/tron/core/services/jsonrpc/TronJsonRpcImpl.java index 1c6debb31b7..503d42b295d 100644 --- a/framework/src/main/java/org/tron/core/services/jsonrpc/TronJsonRpcImpl.java +++ b/framework/src/main/java/org/tron/core/services/jsonrpc/TronJsonRpcImpl.java @@ -29,11 +29,13 @@ import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.StringUtils; +import org.bouncycastle.util.encoders.Hex; import org.tron.api.GrpcAPI.BytesMessage; import org.tron.api.GrpcAPI.Return; import org.tron.api.GrpcAPI.Return.response_code; import org.tron.api.GrpcAPI.TransactionExtention; import org.tron.common.crypto.Hash; +import org.tron.common.logsfilter.ContractEventParser; import org.tron.common.logsfilter.capsule.BlockFilterCapsule; import org.tron.common.logsfilter.capsule.LogsFilterCapsule; import org.tron.common.runtime.vm.DataWord; @@ -76,6 +78,7 @@ import org.tron.protos.Protocol.Block; import org.tron.protos.Protocol.Transaction; import org.tron.protos.Protocol.Transaction.Contract.ContractType; +import org.tron.protos.Protocol.Transaction.Result.code; import org.tron.protos.Protocol.TransactionInfo; import org.tron.protos.contract.AssetIssueContractOuterClass.TransferAssetContract; import org.tron.protos.contract.BalanceContract.TransferContract; @@ -389,7 +392,7 @@ private void callTriggerConstantContract(byte[] ownerAddressByte, byte[] contrac * getMethodSign(methodName(uint256,uint256)) || data1 || data2 */ private String call(byte[] ownerAddressByte, byte[] contractAddressByte, long value, - byte[] data) { + byte[] data) throws JsonRpcInternalException { TransactionExtention.Builder trxExtBuilder = TransactionExtention.newBuilder(); Return.Builder retBuilder = Return.newBuilder(); @@ -418,9 +421,8 @@ private String call(byte[] ownerAddressByte, byte[] contractAddressByte, long va trxExt = trxExtBuilder.build(); } - String result = "0x"; - String code = trxExt.getResult().getCode().toString(); - if ("SUCCESS".equals(code)) { + String result; + if (trxExtBuilder.getTransaction().getRet(0).getRet().equals(code.SUCESS)) { List list = trxExt.getConstantResultList(); byte[] listBytes = new byte[0]; for (ByteString bs : list) { @@ -429,6 +431,16 @@ private String call(byte[] ownerAddressByte, byte[] contractAddressByte, long va result = ByteArray.toJsonHex(listBytes); } else { logger.error("trigger contract failed."); + String errMsg = retBuilder.getMessage().toStringUtf8(); + byte[] resData = trxExtBuilder.getConstantResult(0).toByteArray(); + if (resData.length > 4 && Hex.toHexString(resData).startsWith("08c379a0")) { // Error(string) + String msg = ContractEventParser + .parseDataBytes(org.bouncycastle.util.Arrays.copyOfRange(resData, 4, resData.length), + "string", 0); + errMsg += ": " + msg; + } + + throw new JsonRpcInternalException(errMsg); } return result; @@ -545,7 +557,7 @@ public String estimateGas(CallArguments args) throws JsonRpcInvalidRequestExcept trxExtBuilder, retBuilder); - return ByteArray.toJsonHex(trxExtBuilder.getEnergyUsed()); + } catch (ContractValidateException e) { String errString = "invalid contract"; if (e.getMessage() != null) { @@ -561,6 +573,22 @@ public String estimateGas(CallArguments args) throws JsonRpcInvalidRequestExcept throw new JsonRpcInternalException(errString); } + + if (trxExtBuilder.getTransaction().getRet(0).getRet().equals(code.FAILED)) { + String errMsg = retBuilder.getMessage().toStringUtf8(); + + byte[] data = trxExtBuilder.getConstantResult(0).toByteArray(); + if (data.length > 4 && Hex.toHexString(data).startsWith("08c379a0")) { // Error(string) + String msg = ContractEventParser + .parseDataBytes(org.bouncycastle.util.Arrays.copyOfRange(data, 4, data.length), + "string", 0); + errMsg += ": " + msg; + } + + throw new JsonRpcInternalException(errMsg); + } else { + return ByteArray.toJsonHex(trxExtBuilder.getEnergyUsed()); + } } @Override @@ -689,7 +717,7 @@ public TransactionReceipt getTransactionReceipt(String txId) @Override public String getCall(CallArguments transactionCall, String blockNumOrTag) - throws JsonRpcInvalidParamsException { + throws JsonRpcInvalidParamsException, JsonRpcInternalException { if (EARLIEST_STR.equalsIgnoreCase(blockNumOrTag) || PENDING_STR.equalsIgnoreCase(blockNumOrTag)) { throw new JsonRpcInvalidParamsException(TAG_NOT_SUPPORT_ERROR); diff --git a/framework/src/test/java/org/tron/common/logsfilter/EventParserTest.java b/framework/src/test/java/org/tron/common/logsfilter/EventParserTest.java index 5745102156b..a9b06c0e039 100644 --- a/framework/src/test/java/org/tron/common/logsfilter/EventParserTest.java +++ b/framework/src/test/java/org/tron/common/logsfilter/EventParserTest.java @@ -5,6 +5,7 @@ import java.util.LinkedList; import java.util.List; import java.util.Map; +import org.bouncycastle.util.Arrays; import org.junit.Test; import org.testng.Assert; import org.tron.common.crypto.Hash; @@ -96,4 +97,18 @@ public synchronized void testEventParser() { Assert.assertEquals(dataMap.get("str"), "abcdefg123"); } + + @Test + public void testParseRevert() { + String dataHex = "08c379a0" + + "0000000000000000000000000000000000000000000000000000000000000020" + + "0000000000000000000000000000000000000000000000000000000000000016" + + "6e6f7420656e6f75676820696e7075742076616c756500000000000000000000"; + + byte[] data = ByteArray.fromHexString(dataHex); + String msg = ContractEventParser.parseDataBytes(Arrays.copyOfRange(data, 4, data.length), + "string", 0); + Assert.assertEquals(msg, "not enough input value"); + + } } From 8afd14dab08551385c6df6cafe723eb1727fad72 Mon Sep 17 00:00:00 2001 From: Wenhua Zhang Date: Fri, 5 Aug 2022 17:44:23 +0800 Subject: [PATCH 36/51] feat(jsonrpc): catch exception when call and return err msg --- .../core/services/jsonrpc/TronJsonRpc.java | 4 +- .../services/jsonrpc/TronJsonRpcImpl.java | 37 ++++++++++--------- 2 files changed, 22 insertions(+), 19 deletions(-) diff --git a/framework/src/main/java/org/tron/core/services/jsonrpc/TronJsonRpc.java b/framework/src/main/java/org/tron/core/services/jsonrpc/TronJsonRpc.java index edbb9377988..caf0292a35c 100644 --- a/framework/src/main/java/org/tron/core/services/jsonrpc/TronJsonRpc.java +++ b/framework/src/main/java/org/tron/core/services/jsonrpc/TronJsonRpc.java @@ -148,11 +148,13 @@ TransactionResult getTransactionByBlockNumberAndIndex(String blockNumOrTag, Stri @JsonRpcMethod("eth_call") @JsonRpcErrors({ + @JsonRpcError(exception = JsonRpcInvalidRequestException.class, code = -32600, data = "{}"), @JsonRpcError(exception = JsonRpcInvalidParamsException.class, code = -32602, data = "{}"), @JsonRpcError(exception = JsonRpcInternalException.class, code = -32000, data = "{}"), }) String getCall(CallArguments transactionCall, String blockNumOrTag) - throws JsonRpcInvalidParamsException, JsonRpcInternalException; + throws JsonRpcInvalidParamsException, JsonRpcInvalidRequestException, + JsonRpcInternalException; @JsonRpcMethod("net_peerCount") String getPeerCount(); diff --git a/framework/src/main/java/org/tron/core/services/jsonrpc/TronJsonRpcImpl.java b/framework/src/main/java/org/tron/core/services/jsonrpc/TronJsonRpcImpl.java index 503d42b295d..82568f09acd 100644 --- a/framework/src/main/java/org/tron/core/services/jsonrpc/TronJsonRpcImpl.java +++ b/framework/src/main/java/org/tron/core/services/jsonrpc/TronJsonRpcImpl.java @@ -1,7 +1,6 @@ package org.tron.core.services.jsonrpc; import static org.tron.core.Wallet.CONTRACT_VALIDATE_ERROR; -import static org.tron.core.Wallet.CONTRACT_VALIDATE_EXCEPTION; import static org.tron.core.services.http.Util.setTransactionExtraData; import static org.tron.core.services.http.Util.setTransactionPermissionId; import static org.tron.core.services.jsonrpc.JsonRpcApiUtil.addressCompatibleToByteArray; @@ -136,6 +135,7 @@ public enum RequestSource { private static final String QUANTITY_NOT_SUPPORT_ERROR = "QUANTITY not supported, just support TAG as latest"; + private static final String ERROR_SELECTOR = "08c379a0"; // Function selector for Error(string) /** * thread pool of query section bloom store */ @@ -392,7 +392,7 @@ private void callTriggerConstantContract(byte[] ownerAddressByte, byte[] contrac * getMethodSign(methodName(uint256,uint256)) || data1 || data2 */ private String call(byte[] ownerAddressByte, byte[] contractAddressByte, long value, - byte[] data) throws JsonRpcInternalException { + byte[] data) throws JsonRpcInvalidRequestException, JsonRpcInternalException { TransactionExtention.Builder trxExtBuilder = TransactionExtention.newBuilder(); Return.Builder retBuilder = Return.newBuilder(); @@ -403,20 +403,20 @@ private String call(byte[] ownerAddressByte, byte[] contractAddressByte, long va trxExtBuilder, retBuilder); } catch (ContractValidateException | VMIllegalException e) { - retBuilder.setResult(false).setCode(response_code.CONTRACT_VALIDATE_ERROR) - .setMessage(ByteString.copyFromUtf8(CONTRACT_VALIDATE_ERROR + e.getMessage())); - trxExtBuilder.setResult(retBuilder); - logger.warn(CONTRACT_VALIDATE_EXCEPTION, e.getMessage()); - } catch (RuntimeException e) { - retBuilder.setResult(false).setCode(response_code.CONTRACT_EXE_ERROR) - .setMessage(ByteString.copyFromUtf8(e.getClass() + " : " + e.getMessage())); - trxExtBuilder.setResult(retBuilder); - logger.warn("When run constant call in VM, have RuntimeException: " + e.getMessage()); + String errString = CONTRACT_VALIDATE_ERROR; + if (e.getMessage() != null) { + errString = e.getMessage(); + } + + throw new JsonRpcInvalidRequestException(errString); } catch (Exception e) { - retBuilder.setResult(false).setCode(response_code.OTHER_ERROR) - .setMessage(ByteString.copyFromUtf8(e.getClass() + " : " + e.getMessage())); - trxExtBuilder.setResult(retBuilder); - logger.warn("Unknown exception caught: " + e.getMessage(), e); + String errString = JSON_ERROR; + if (e.getMessage() != null) { + errString = e.getMessage().replaceAll("[\"]", "'"); + } + + throw new JsonRpcInternalException(errString); + } finally { trxExt = trxExtBuilder.build(); } @@ -433,7 +433,7 @@ private String call(byte[] ownerAddressByte, byte[] contractAddressByte, long va logger.error("trigger contract failed."); String errMsg = retBuilder.getMessage().toStringUtf8(); byte[] resData = trxExtBuilder.getConstantResult(0).toByteArray(); - if (resData.length > 4 && Hex.toHexString(resData).startsWith("08c379a0")) { // Error(string) + if (resData.length > 4 && Hex.toHexString(resData).startsWith(ERROR_SELECTOR)) { String msg = ContractEventParser .parseDataBytes(org.bouncycastle.util.Arrays.copyOfRange(resData, 4, resData.length), "string", 0); @@ -578,7 +578,7 @@ public String estimateGas(CallArguments args) throws JsonRpcInvalidRequestExcept String errMsg = retBuilder.getMessage().toStringUtf8(); byte[] data = trxExtBuilder.getConstantResult(0).toByteArray(); - if (data.length > 4 && Hex.toHexString(data).startsWith("08c379a0")) { // Error(string) + if (data.length > 4 && Hex.toHexString(data).startsWith(ERROR_SELECTOR)) { String msg = ContractEventParser .parseDataBytes(org.bouncycastle.util.Arrays.copyOfRange(data, 4, data.length), "string", 0); @@ -717,7 +717,8 @@ public TransactionReceipt getTransactionReceipt(String txId) @Override public String getCall(CallArguments transactionCall, String blockNumOrTag) - throws JsonRpcInvalidParamsException, JsonRpcInternalException { + throws JsonRpcInvalidParamsException, JsonRpcInvalidRequestException, + JsonRpcInternalException { if (EARLIEST_STR.equalsIgnoreCase(blockNumOrTag) || PENDING_STR.equalsIgnoreCase(blockNumOrTag)) { throw new JsonRpcInvalidParamsException(TAG_NOT_SUPPORT_ERROR); From 77fb40fe80c921e230a54b69e9cf62335a8e0f0c Mon Sep 17 00:00:00 2001 From: wubin01 Date: Mon, 8 Aug 2022 11:53:54 +0800 Subject: [PATCH 37/51] perf(net): configurable block cache validity period --- .../java/org/tron/common/parameter/CommonParameter.java | 4 ++++ common/src/main/java/org/tron/core/Constant.java | 2 ++ framework/src/main/java/org/tron/core/config/args/Args.java | 6 +++++- .../src/main/java/org/tron/core/net/service/AdvService.java | 4 +++- .../main/java/org/tron/core/net/service/SyncService.java | 4 +++- .../src/test/java/org/tron/common/config/args/ArgsTest.java | 1 + 6 files changed, 18 insertions(+), 3 deletions(-) diff --git a/common/src/main/java/org/tron/common/parameter/CommonParameter.java b/common/src/main/java/org/tron/common/parameter/CommonParameter.java index 6daf213e672..9a8117c4364 100644 --- a/common/src/main/java/org/tron/common/parameter/CommonParameter.java +++ b/common/src/main/java/org/tron/common/parameter/CommonParameter.java @@ -544,6 +544,10 @@ public class CommonParameter { @Setter public long shutdownBlockCount = -1; + @Getter + @Setter + public long blockCacheTimeout = 60; + private static double calcMaxTimeRatio() { //return max(2.0, min(5.0, 5 * 4.0 / max(Runtime.getRuntime().availableProcessors(), 1))); return 5.0; diff --git a/common/src/main/java/org/tron/core/Constant.java b/common/src/main/java/org/tron/core/Constant.java index 883e4cd3f6a..94eed9f9941 100644 --- a/common/src/main/java/org/tron/core/Constant.java +++ b/common/src/main/java/org/tron/core/Constant.java @@ -311,4 +311,6 @@ public class Constant { public static final String NODE_SHUTDOWN_BLOCK_TIME = "node.shutdown.BlockTime"; public static final String NODE_SHUTDOWN_BLOCK_HEIGHT = "node.shutdown.BlockHeight"; public static final String NODE_SHUTDOWN_BLOCK_COUNT = "node.shutdown.BlockCount"; + + public static final String BLOCK_CACHE_TIMEOUT = "node.blockCacheTimeout"; } diff --git a/framework/src/main/java/org/tron/core/config/args/Args.java b/framework/src/main/java/org/tron/core/config/args/Args.java index 38ae934877e..16ede8cdbaa 100644 --- a/framework/src/main/java/org/tron/core/config/args/Args.java +++ b/framework/src/main/java/org/tron/core/config/args/Args.java @@ -207,7 +207,7 @@ public static void clearParam() { PARAMETER.shutdownBlockTime = null; PARAMETER.shutdownBlockHeight = -1; PARAMETER.shutdownBlockCount = -1; - + PARAMETER.blockCacheTimeout = 60; } /** @@ -925,6 +925,10 @@ public static void setParam(final String[] args, final String confFileName) { PARAMETER.shutdownBlockCount = config.getLong(Constant.NODE_SHUTDOWN_BLOCK_COUNT); } + if (config.hasPath(Constant.BLOCK_CACHE_TIMEOUT)) { + PARAMETER.blockCacheTimeout = config.getLong(Constant.BLOCK_CACHE_TIMEOUT); + } + logConfig(); } diff --git a/framework/src/main/java/org/tron/core/net/service/AdvService.java b/framework/src/main/java/org/tron/core/net/service/AdvService.java index 2df3598f611..e0dc4238ab7 100644 --- a/framework/src/main/java/org/tron/core/net/service/AdvService.java +++ b/framework/src/main/java/org/tron/core/net/service/AdvService.java @@ -57,8 +57,10 @@ public class AdvService { private ConcurrentHashMap invToSpread = new ConcurrentHashMap<>(); + private long blockCacheTimeout = Args.getInstance().getBlockCacheTimeout(); private Cache invToFetchCache = CacheBuilder.newBuilder() - .maximumSize(MAX_INV_TO_FETCH_CACHE_SIZE).expireAfterWrite(1, TimeUnit.HOURS) + .maximumSize(MAX_INV_TO_FETCH_CACHE_SIZE) + .expireAfterWrite(blockCacheTimeout, TimeUnit.MINUTES) .recordStats().build(); private Cache trxCache = CacheBuilder.newBuilder() diff --git a/framework/src/main/java/org/tron/core/net/service/SyncService.java b/framework/src/main/java/org/tron/core/net/service/SyncService.java index a845636784f..8e4d7a49b3d 100644 --- a/framework/src/main/java/org/tron/core/net/service/SyncService.java +++ b/framework/src/main/java/org/tron/core/net/service/SyncService.java @@ -23,6 +23,7 @@ import org.tron.core.capsule.BlockCapsule; import org.tron.core.capsule.BlockCapsule.BlockId; import org.tron.core.config.Parameter.NetConstants; +import org.tron.core.config.args.Args; import org.tron.core.exception.P2pException; import org.tron.core.exception.P2pException.TypeEnum; import org.tron.core.net.TronNetDelegate; @@ -48,8 +49,9 @@ public class SyncService { private Map blockJustReceived = new ConcurrentHashMap<>(); + private long blockCacheTimeout = Args.getInstance().getBlockCacheTimeout(); private Cache requestBlockIds = CacheBuilder.newBuilder().maximumSize(10_000) - .expireAfterWrite(1, TimeUnit.HOURS).initialCapacity(10_000) + .expireAfterWrite(blockCacheTimeout, TimeUnit.MINUTES).initialCapacity(10_000) .recordStats().build(); private ScheduledExecutorService fetchExecutor = Executors.newSingleThreadScheduledExecutor(); diff --git a/framework/src/test/java/org/tron/common/config/args/ArgsTest.java b/framework/src/test/java/org/tron/common/config/args/ArgsTest.java index 2732824779e..5b391f0d38d 100644 --- a/framework/src/test/java/org/tron/common/config/args/ArgsTest.java +++ b/framework/src/test/java/org/tron/common/config/args/ArgsTest.java @@ -29,5 +29,6 @@ public void testConfig() { Assert.assertEquals(Args.getInstance().getPendingTransactionTimeout(), 60_000); Assert.assertEquals(Args.getInstance().getNodeDiscoveryPingTimeout(), 15_000); Assert.assertEquals(Args.getInstance().getMaxFastForwardNum(), 3); + Assert.assertEquals(Args.getInstance().getBlockCacheTimeout(), 60); } } \ No newline at end of file From b6a6b8303c21aa05c63b35c766ccfe97abe4e51f Mon Sep 17 00:00:00 2001 From: halibobo1205 Date: Mon, 8 Aug 2022 19:01:16 +0800 Subject: [PATCH 38/51] test(api):add test for query block header. --- .../test/java/org/tron/core/WalletTest.java | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/framework/src/test/java/org/tron/core/WalletTest.java b/framework/src/test/java/org/tron/core/WalletTest.java index 4c0ff5ed6c4..1b35a78bf82 100644 --- a/framework/src/test/java/org/tron/core/WalletTest.java +++ b/framework/src/test/java/org/tron/core/WalletTest.java @@ -21,6 +21,7 @@ import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; import static stest.tron.wallet.common.client.utils.PublicMethed.decode58Check; import com.google.protobuf.Any; @@ -36,6 +37,7 @@ import org.junit.BeforeClass; import org.junit.Ignore; import org.junit.Test; +import org.tron.api.GrpcAPI; import org.tron.api.GrpcAPI.AssetIssueList; import org.tron.api.GrpcAPI.BlockList; import org.tron.api.GrpcAPI.ExchangeList; @@ -503,6 +505,32 @@ public void getPaginatedExchangeList() { exchangeList.getExchangesList().get(1).getCreatorAddress().toStringUtf8()); } + @Test + public void getBlock() { + GrpcAPI.BlockReq req = GrpcAPI.BlockReq.getDefaultInstance(); + Block block = wallet.getBlock(req); + assertNotNull(block); + try { + req = req.toBuilder().setIdOrNum("-1").build(); + wallet.getBlock(req); + } catch (Exception e) { + Assert.assertTrue(e instanceof IllegalArgumentException); + } + try { + req = req.toBuilder().setIdOrNum("hash000001").build(); + wallet.getBlock(req); + } catch (Exception e) { + Assert.assertTrue(e instanceof IllegalArgumentException); + } + req = GrpcAPI.BlockReq.newBuilder().setIdOrNum("0").build(); + block = wallet.getBlock(req); + req = req.toBuilder().setDetail(true).build(); + assertEquals(block, wallet.getBlock(req).toBuilder().clearTransactions().build()); + req = req.toBuilder().clearDetail() + .setIdOrNum(new BlockCapsule(block).getBlockId().toString()).build(); + assertEquals(block, wallet.getBlock(req)); + } + //@Test public void testChainParameters() { From 8c91dd4d8b4cc6300c7fe68db415444580426532 Mon Sep 17 00:00:00 2001 From: halibobo1205 Date: Mon, 8 Aug 2022 18:43:36 +0800 Subject: [PATCH 39/51] test(db-mv):add test for db mv func. --- .../java/org/tron/plugins/DbMoveTest.java | 157 ++++++++++++++++++ .../src/test/resources/config-duplicate.conf | 22 +++ plugins/src/test/resources/config.conf | 22 +++ 3 files changed, 201 insertions(+) create mode 100644 plugins/src/test/java/org/tron/plugins/DbMoveTest.java create mode 100644 plugins/src/test/resources/config-duplicate.conf create mode 100644 plugins/src/test/resources/config.conf diff --git a/plugins/src/test/java/org/tron/plugins/DbMoveTest.java b/plugins/src/test/java/org/tron/plugins/DbMoveTest.java new file mode 100644 index 00000000000..6bbac663c2c --- /dev/null +++ b/plugins/src/test/java/org/tron/plugins/DbMoveTest.java @@ -0,0 +1,157 @@ +package org.tron.plugins; + +import static org.iq80.leveldb.impl.Iq80DBFactory.factory; + +import java.io.BufferedReader; +import java.io.BufferedWriter; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStreamReader; +import java.io.OutputStream; +import java.io.OutputStreamWriter; +import java.net.URL; +import java.nio.charset.StandardCharsets; +import java.nio.file.Paths; +import java.util.Properties; +import java.util.UUID; +import lombok.extern.slf4j.Slf4j; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; +import picocli.CommandLine; + +@Slf4j +public class DbMoveTest { + + private static final String OUTPUT_DIRECTORY = "output-directory-toolkit"; + private static final String OUTPUT_DIRECTORY_DATABASE = + Paths.get(OUTPUT_DIRECTORY,"ori","database").toString(); + private static final String ENGINE = "ENGINE"; + private static final String LEVELDB = "LEVELDB"; + private static final String ACCOUNT = "account"; + private static final String TRANS = "trans"; + private static final String MARKET = "market_pair_price_to_order"; + private static final String ENGINE_FILE = "engine.properties"; + + + @BeforeClass + public static void init() throws IOException { + File file = new File(OUTPUT_DIRECTORY_DATABASE, ACCOUNT); + factory.open(file, ArchiveManifest.newDefaultLevelDbOptions()).close(); + writeProperty(file + File.separator + ENGINE_FILE, ENGINE, LEVELDB); + + file = new File(OUTPUT_DIRECTORY_DATABASE, MARKET); + factory.open(file, ArchiveManifest.newDefaultLevelDbOptions()).close(); + writeProperty(file + File.separator + ENGINE_FILE, ENGINE, LEVELDB); + + file = new File(OUTPUT_DIRECTORY_DATABASE, TRANS); + factory.open(file, ArchiveManifest.newDefaultLevelDbOptions()).close(); + writeProperty(file + File.separator + ENGINE_FILE, ENGINE, LEVELDB); + + } + + @AfterClass + public static void destroy() { + deleteDir(new File(OUTPUT_DIRECTORY)); + } + + private static void writeProperty(String filename, String key, String value) throws IOException { + File file = new File(filename); + if (!file.exists()) { + file.createNewFile(); + } + + try (FileInputStream fis = new FileInputStream(file); + OutputStream out = new FileOutputStream(file); + BufferedWriter bw = new BufferedWriter(new OutputStreamWriter(out, + StandardCharsets.UTF_8))) { + BufferedReader bf = new BufferedReader(new InputStreamReader(fis, StandardCharsets.UTF_8)); + Properties properties = new Properties(); + properties.load(bf); + properties.setProperty(key, value); + properties.store(bw, "Generated by the application. PLEASE DO NOT EDIT! "); + } catch (Exception e) { + logger.warn("{}", e); + } + } + + /** + * delete directory. + */ + private static boolean deleteDir(File dir) { + if (dir.isDirectory()) { + String[] children = dir.list(); + assert children != null; + for (String child : children) { + boolean success = deleteDir(new File(dir, child)); + if (!success) { + logger.warn("can't delete dir:" + dir); + return false; + } + } + } + return dir.delete(); + } + + private static String getConfig(String config) { + URL path = DbMoveTest.class.getClassLoader().getResource(config); + return path == null ? null : path.getPath(); + } + + @Test + public void testMv() { + String[] args = new String[] {"db", "mv", "-d", + Paths.get(OUTPUT_DIRECTORY,"ori").toString(), "-c", + getConfig("config.conf")}; + CommandLine cli = new CommandLine(new Toolkit()); + Assert.assertEquals(0, cli.execute(args)); + Assert.assertEquals(2, cli.execute(args)); + } + + @Test + public void testDuplicate() { + String[] args = new String[] {"db", "mv", "-d", + Paths.get(OUTPUT_DIRECTORY,"ori").toString(), "-c", + getConfig("config-duplicate.conf")}; + CommandLine cli = new CommandLine(new Toolkit()); + Assert.assertEquals(2, cli.execute(args)); + } + + @Test + public void testHelp() { + String[] args = new String[] {}; + CommandLine cli = new CommandLine(new Toolkit()); + Assert.assertEquals(0, cli.execute(args)); + } + + @Test + public void testDicNotExist() { + String[] args = new String[] {"db", "mv", "-d", "dicNotExist"}; + CommandLine cli = new CommandLine(new Toolkit()); + Assert.assertEquals(2, cli.execute(args)); + } + + @Test + public void testConfNotExist() { + String[] args = new String[] {"db", "mv", "-d", + Paths.get(OUTPUT_DIRECTORY,"ori").toString(), "-c", + "config.conf"}; + CommandLine cli = new CommandLine(new Toolkit()); + Assert.assertEquals(2, cli.execute(args)); + } + + @Test + public void testEmpty() { + File file = new File(OUTPUT_DIRECTORY_DATABASE + File.separator + UUID.randomUUID()); + file.mkdirs(); + file.deleteOnExit(); + String[] args = new String[] {"db", "mv", "-d", file.toString(), "-c", + getConfig("config.conf")}; + CommandLine cli = new CommandLine(new Toolkit()); + + Assert.assertEquals(2, cli.execute(args)); + } +} diff --git a/plugins/src/test/resources/config-duplicate.conf b/plugins/src/test/resources/config-duplicate.conf new file mode 100644 index 00000000000..758312ad05e --- /dev/null +++ b/plugins/src/test/resources/config-duplicate.conf @@ -0,0 +1,22 @@ + +storage { + # Directory for storing persistent data + db.version = 2, + db.engine = "LEVELDB", + db.sync = false, + db.directory = "database", + index.directory = "index", + transHistory.switch = "on", + properties = [ + { + name = "trans", + path = "output-directory-toolkit/dest", + }, + { + name = "trans", + path = "output-directory-toolkit/dest2", + }, + ] +} + + diff --git a/plugins/src/test/resources/config.conf b/plugins/src/test/resources/config.conf new file mode 100644 index 00000000000..be16d44c420 --- /dev/null +++ b/plugins/src/test/resources/config.conf @@ -0,0 +1,22 @@ + +storage { + # Directory for storing persistent data + db.version = 2, + db.engine = "LEVELDB", + db.sync = false, + db.directory = "database", + index.directory = "index", + transHistory.switch = "on", + properties = [ + { + name = "account", + path = "output-directory-toolkit/dest", + }, + { + name = "market_pair_price_to_order", + path = "output-directory-toolkit/dest", + }, + ] +} + + From e4175e5cfdc014e1162e962d448cec39d7435649 Mon Sep 17 00:00:00 2001 From: chengtx01 Date: Tue, 9 Aug 2022 17:56:03 +0800 Subject: [PATCH 40/51] fix(solidityNode): fix solidity node startup issue --- .../java/org/tron/common/overlay/server/ChannelManager.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/framework/src/main/java/org/tron/common/overlay/server/ChannelManager.java b/framework/src/main/java/org/tron/common/overlay/server/ChannelManager.java index 1bed907c270..d9da821f495 100644 --- a/framework/src/main/java/org/tron/common/overlay/server/ChannelManager.java +++ b/framework/src/main/java/org/tron/common/overlay/server/ChannelManager.java @@ -70,7 +70,7 @@ public class ChannelManager { private int maxConnectionsWithSameIp = parameter.getMaxConnectionsWithSameIp(); public void init() { - if (this.parameter.getNodeListenPort() > 0) { + if (this.parameter.getNodeListenPort() > 0 && !this.parameter.isSolidityNode()) { new Thread(() -> peerServer.start(Args.getInstance().getNodeListenPort()), "PeerServerThread").start(); } From 42eec0d0ee86278f86023fce68fef0111b2f9833 Mon Sep 17 00:00:00 2001 From: chengtx01 Date: Tue, 9 Aug 2022 18:36:13 +0800 Subject: [PATCH 41/51] perf(solidityNode): optimize solidity node startup process --- .../java/org/tron/common/overlay/discover/DiscoverServer.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/framework/src/main/java/org/tron/common/overlay/discover/DiscoverServer.java b/framework/src/main/java/org/tron/common/overlay/discover/DiscoverServer.java index 4a0d49a9cb6..983be8543ef 100755 --- a/framework/src/main/java/org/tron/common/overlay/discover/DiscoverServer.java +++ b/framework/src/main/java/org/tron/common/overlay/discover/DiscoverServer.java @@ -59,7 +59,8 @@ public class DiscoverServer { @Autowired public DiscoverServer(final NodeManager nodeManager) { this.nodeManager = nodeManager; - if (parameter.isNodeDiscoveryEnable() && !parameter.isFastForward()) { + if (parameter.isNodeDiscoveryEnable() && !parameter.isFastForward() + && !parameter.isSolidityNode()) { if (port == 0) { logger.error("Discovery can't be started while listen port == 0"); } else { From 36305069cfd9980b1e76df27daaded41ed30e8e7 Mon Sep 17 00:00:00 2001 From: halibobo1205 Date: Wed, 10 Aug 2022 12:04:22 +0800 Subject: [PATCH 42/51] fix(lite):fix merge function when split and merge base on liteNode. --- .../tron/tool/litefullnode/LiteFullNodeTool.java | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/framework/src/main/java/org/tron/tool/litefullnode/LiteFullNodeTool.java b/framework/src/main/java/org/tron/tool/litefullnode/LiteFullNodeTool.java index 17f4a4d81e4..2de9972f73c 100644 --- a/framework/src/main/java/org/tron/tool/litefullnode/LiteFullNodeTool.java +++ b/framework/src/main/java/org/tron/tool/litefullnode/LiteFullNodeTool.java @@ -489,12 +489,15 @@ private static boolean isEmptyBytes(byte[] b) { private void deleteSnapshotFlag(String databaseDir) throws IOException, RocksDBException { logger.info("-- delete the info file to identify this node is a real fullnode."); Files.delete(Paths.get(databaseDir, INFO_FILE_NAME)); + DBInterface destBlockIndexDb = DbTool.getDB(databaseDir, BLOCK_INDEX_DB_NAME); + if (destBlockIndexDb.get(ByteArray.fromLong(1)) != null) { + DBInterface destCommonDb = DbTool.getDB(databaseDir, COMMON_DB_NAME); + destCommonDb.delete(DB_KEY_NODE_TYPE); + destCommonDb.delete(DB_KEY_LOWEST_BLOCK_NUM); + logger.info("-- deleted node_type and lowest_block_num from " + + "common to identify this node is a real fullnode."); + } - DBInterface destCommonDb = DbTool.getDB(databaseDir, COMMON_DB_NAME); - destCommonDb.delete(DB_KEY_NODE_TYPE); - destCommonDb.delete(DB_KEY_LOWEST_BLOCK_NUM); - logger.info("-- deleted node_type and lowest_block_num from " - + "common to identify this node is a real fullnode."); } private void run(Args argv) { From 40b250cba9bc6d57a4786ba8cc780038476aa712 Mon Sep 17 00:00:00 2001 From: zhang0125 Date: Wed, 10 Aug 2022 15:42:18 +0800 Subject: [PATCH 43/51] fix(tx-cache): fix lite node init error. --- .../src/main/java/org/tron/core/db/Manager.java | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/framework/src/main/java/org/tron/core/db/Manager.java b/framework/src/main/java/org/tron/core/db/Manager.java index 0d25dadf48e..6152fa526da 100644 --- a/framework/src/main/java/org/tron/core/db/Manager.java +++ b/framework/src/main/java/org/tron/core/db/Manager.java @@ -2279,13 +2279,22 @@ public long getPendingSize() { private void initLiteNode() { // When using bloom filter for transaction de-duplication, // it is possible to use trans for secondary confirmation. - // Init trans db for liteNode, + // Init trans db for liteNode if needed. long headNum = chainBaseManager.getDynamicPropertiesStore().getLatestBlockHeaderNumber(); long recentBlockCount = chainBaseManager.getRecentBlockStore().size(); long recentBlockStart = headNum - recentBlockCount + 1; - try { - chainBaseManager.getBlockByNum(recentBlockStart); - } catch (ItemNotFoundException | BadItemException e) { + boolean needInit = false; + if (recentBlockStart == 0) { + needInit = true; + } else { + try { + chainBaseManager.getBlockByNum(recentBlockStart); + } catch (ItemNotFoundException | BadItemException e) { + needInit = true; + } + } + + if (needInit) { // copy transaction from recent-transaction to trans logger.info("load trans for lite node."); From afbd927388e77a93476f58a9da45272dd0ce62e0 Mon Sep 17 00:00:00 2001 From: halibobo1205 Date: Wed, 10 Aug 2022 16:51:29 +0800 Subject: [PATCH 44/51] log(lite-tool):merge function. --- .../main/java/org/tron/tool/litefullnode/LiteFullNodeTool.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/framework/src/main/java/org/tron/tool/litefullnode/LiteFullNodeTool.java b/framework/src/main/java/org/tron/tool/litefullnode/LiteFullNodeTool.java index 2de9972f73c..037760e201a 100644 --- a/framework/src/main/java/org/tron/tool/litefullnode/LiteFullNodeTool.java +++ b/framework/src/main/java/org/tron/tool/litefullnode/LiteFullNodeTool.java @@ -487,7 +487,7 @@ private static boolean isEmptyBytes(byte[] b) { } private void deleteSnapshotFlag(String databaseDir) throws IOException, RocksDBException { - logger.info("-- delete the info file to identify this node is a real fullnode."); + logger.info("-- delete the info file."); Files.delete(Paths.get(databaseDir, INFO_FILE_NAME)); DBInterface destBlockIndexDb = DbTool.getDB(databaseDir, BLOCK_INDEX_DB_NAME); if (destBlockIndexDb.get(ByteArray.fromLong(1)) != null) { From f53fee84f2ed8acaa5828081ed484f1e96d58997 Mon Sep 17 00:00:00 2001 From: zhang0125 Date: Wed, 10 Aug 2022 18:39:21 +0800 Subject: [PATCH 45/51] feat(tx-cache): change the default value of the txCache.estimatedTransactions param. --- common/src/main/java/org/tron/core/config/args/Storage.java | 2 +- framework/src/main/resources/config.conf | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/common/src/main/java/org/tron/core/config/args/Storage.java b/common/src/main/java/org/tron/core/config/args/Storage.java index 60e3cd9ba6d..1ea7e00eb1f 100644 --- a/common/src/main/java/org/tron/core/config/args/Storage.java +++ b/common/src/main/java/org/tron/core/config/args/Storage.java @@ -79,7 +79,7 @@ public class Storage { private static final String DEFAULT_DB_DIRECTORY = "database"; private static final String DEFAULT_INDEX_DIRECTORY = "index"; private static final String DEFAULT_INDEX_SWITCH = "on"; - private static final int DEFAULT_ESTIMATED_TRANSACTIONS = 500; + private static final int DEFAULT_ESTIMATED_TRANSACTIONS = 1000; private Config storage; /** diff --git a/framework/src/main/resources/config.conf b/framework/src/main/resources/config.conf index eeca1172a3e..aa1f51507bf 100644 --- a/framework/src/main/resources/config.conf +++ b/framework/src/main/resources/config.conf @@ -90,9 +90,9 @@ storage { balance.history.lookup = false - # the estimated number of block transactions (default 500, min 100, max 10000). + # the estimated number of block transactions (default 1000, min 100, max 10000). # so the total number of cached transactions is 65536 * txCache.estimatedTransactions - # txCache.estimatedTransactions = 500 + # txCache.estimatedTransactions = 1000 } From f7f88b0aa7f79a02048e62b588f25f6c4bca8064 Mon Sep 17 00:00:00 2001 From: halibobo1205 Date: Thu, 11 Aug 2022 11:43:05 +0800 Subject: [PATCH 46/51] fix(block): fix different result code when fork. 1.use deep copy for trx when put poppedTransactions. --- framework/src/main/java/org/tron/core/db/Manager.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/framework/src/main/java/org/tron/core/db/Manager.java b/framework/src/main/java/org/tron/core/db/Manager.java index 6152fa526da..e5eee4eda7e 100644 --- a/framework/src/main/java/org/tron/core/db/Manager.java +++ b/framework/src/main/java/org/tron/core/db/Manager.java @@ -869,7 +869,8 @@ public void eraseBlock() { khaosDb.pop(); revokingStore.fastPop(); logger.info("end to erase block:" + oldHeadBlock); - poppedTransactions.addAll(oldHeadBlock.getTransactions()); + oldHeadBlock.getTransactions().forEach(tc -> + poppedTransactions.add(new TransactionCapsule(tc.getInstance()))); Metrics.gaugeInc(MetricKeys.Gauge.MANAGER_QUEUE, oldHeadBlock.getTransactions().size(), MetricLabels.Gauge.QUEUE_POPPED); From 8f1b4c2d1bc9da197fae717e2005d83511537457 Mon Sep 17 00:00:00 2001 From: wubin01 Date: Fri, 12 Aug 2022 11:12:34 +0800 Subject: [PATCH 47/51] close the peer connection when the service is down --- .../main/java/org/tron/common/overlay/server/SyncPool.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/framework/src/main/java/org/tron/common/overlay/server/SyncPool.java b/framework/src/main/java/org/tron/common/overlay/server/SyncPool.java index 4c35f6b628d..d850a387d98 100644 --- a/framework/src/main/java/org/tron/common/overlay/server/SyncPool.java +++ b/framework/src/main/java/org/tron/common/overlay/server/SyncPool.java @@ -217,6 +217,11 @@ public boolean isCanConnect() { public void close() { try { + activePeers.forEach(p -> { + if (!p.isDisconnect()) { + p.close(); + } + }); poolLoopExecutor.shutdownNow(); logExecutor.shutdownNow(); } catch (Exception e) { From 378dd3eb393e197775ecc1ee68164c06a7b50c2d Mon Sep 17 00:00:00 2001 From: Asuka Date: Fri, 12 Aug 2022 16:01:38 +0800 Subject: [PATCH 48/51] fix(vm, hard_fork): init energy limit hard fork while init db manager --- framework/src/main/java/org/tron/core/db/Manager.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/framework/src/main/java/org/tron/core/db/Manager.java b/framework/src/main/java/org/tron/core/db/Manager.java index 6152fa526da..50e81ea602f 100644 --- a/framework/src/main/java/org/tron/core/db/Manager.java +++ b/framework/src/main/java/org/tron/core/db/Manager.java @@ -85,6 +85,7 @@ import org.tron.core.capsule.BlockCapsule; import org.tron.core.capsule.BlockCapsule.BlockId; import org.tron.core.capsule.BytesCapsule; +import org.tron.core.capsule.ReceiptCapsule; import org.tron.core.capsule.TransactionCapsule; import org.tron.core.capsule.TransactionInfoCapsule; import org.tron.core.capsule.TransactionRetCapsule; @@ -154,6 +155,7 @@ import org.tron.core.store.WitnessScheduleStore; import org.tron.core.store.WitnessStore; import org.tron.core.utils.TransactionRegister; +import org.tron.core.vm.config.VMConfig; import org.tron.protos.Protocol.AccountType; import org.tron.protos.Protocol.Permission; import org.tron.protos.Protocol.Transaction; @@ -534,7 +536,9 @@ public void init() { //initActuatorCreator ActuatorCreator.init(); TransactionRegister.registerActuator(); - + //initEnergyLimitHardFork + VMConfig.initVmHardFork(ReceiptCapsule.checkForEnergyLimit( + chainBaseManager.getDynamicPropertiesStore())); long exitHeight = CommonParameter.getInstance().getShutdownBlockHeight(); long exitCount = CommonParameter.getInstance().getShutdownBlockCount(); From 6d8cea70677b875d7034130bbeb88894da8fba11 Mon Sep 17 00:00:00 2001 From: wubin01 Date: Fri, 12 Aug 2022 17:33:35 +0800 Subject: [PATCH 49/51] perf(net): optimize message sending thread shutdown logic --- .../main/java/org/tron/common/overlay/server/MessageQueue.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/framework/src/main/java/org/tron/common/overlay/server/MessageQueue.java b/framework/src/main/java/org/tron/common/overlay/server/MessageQueue.java index 9b702dac915..2edf84c8e32 100644 --- a/framework/src/main/java/org/tron/common/overlay/server/MessageQueue.java +++ b/framework/src/main/java/org/tron/common/overlay/server/MessageQueue.java @@ -155,7 +155,7 @@ public void receivedMessage(Message msg) { public void close() { sendMsgFlag = false; if (sendTask != null && !sendTask.isCancelled()) { - sendTask.cancel(false); + sendTask.cancel(true); sendTask = null; } if (sendMsgThread != null) { From 2cb7b7b4e9a084de28b0284ce0fc846976d1fedd Mon Sep 17 00:00:00 2001 From: liukai Date: Mon, 15 Aug 2022 10:51:07 +0800 Subject: [PATCH 50/51] feat(version): update Version 1. update version to 4.5.2 --- framework/src/main/java/org/tron/program/Version.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/framework/src/main/java/org/tron/program/Version.java b/framework/src/main/java/org/tron/program/Version.java index 944511cbd4e..704ee0f2806 100644 --- a/framework/src/main/java/org/tron/program/Version.java +++ b/framework/src/main/java/org/tron/program/Version.java @@ -4,7 +4,7 @@ public class Version { public static final String VERSION_NAME = "GreatVoyage-v4.4.6-257-ge06abd15c"; public static final String VERSION_CODE = "16944"; - private static final String VERSION = "4.5.1"; + private static final String VERSION = "4.5.2"; public static String getVersion() { return VERSION; From 4d8cb29ad39a71b0a12f47e9859bf28e94fbf49c Mon Sep 17 00:00:00 2001 From: lvs007 Date: Thu, 18 Aug 2022 14:50:17 +0800 Subject: [PATCH 51/51] update a new version. version name:GreatVoyage-v4.5.1-125-ga9809b5bb,version code:17071 --- framework/src/main/java/org/tron/program/Version.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/framework/src/main/java/org/tron/program/Version.java b/framework/src/main/java/org/tron/program/Version.java index 704ee0f2806..4296f5daf1a 100644 --- a/framework/src/main/java/org/tron/program/Version.java +++ b/framework/src/main/java/org/tron/program/Version.java @@ -2,8 +2,8 @@ public class Version { - public static final String VERSION_NAME = "GreatVoyage-v4.4.6-257-ge06abd15c"; - public static final String VERSION_CODE = "16944"; + public static final String VERSION_NAME = "GreatVoyage-v4.5.1-125-ga9809b5bb"; + public static final String VERSION_CODE = "17071"; private static final String VERSION = "4.5.2"; public static String getVersion() {