merge Release 4.1.3 to master

chainbase/src/main/java/org/tron/core/capsule/TransactionCapsule.java

@@ -102,6 +102,9 @@ public class TransactionCapsule implements ProtoCapsule<Transaction> {
   @Getter
   @Setter
   private long time;
+  @Getter
+  @Setter
+  private long order;
 
   /**
    * constructor TransactionCapsule.

chainbase/src/main/java/org/tron/core/db/TransactionTrace.java

@@ -181,16 +181,16 @@ public void exec()
     runtime.execute(transactionContext);
     setBill(transactionContext.getProgramResult().getEnergyUsed());
 
-    if (TrxType.TRX_PRECOMPILED_TYPE != trxType) {
-      if (contractResult.OUT_OF_TIME
-          .equals(receipt.getResult())) {
-        setTimeResultType(TimeResultType.OUT_OF_TIME);
-      } else if (System.currentTimeMillis() - txStartTimeInMs
-          > CommonParameter.getInstance()
-          .getLongRunningTime()) {
-        setTimeResultType(TimeResultType.LONG_RUNNING);
-      }
-    }
+//    if (TrxType.TRX_PRECOMPILED_TYPE != trxType) {
+//      if (contractResult.OUT_OF_TIME
+//          .equals(receipt.getResult())) {
+//        setTimeResultType(TimeResultType.OUT_OF_TIME);
+//      } else if (System.currentTimeMillis() - txStartTimeInMs
+//          > CommonParameter.getInstance()
+//          .getLongRunningTime()) {
+//        setTimeResultType(TimeResultType.LONG_RUNNING);
+//      }
+//    }
   }
 
   public void finalization() throws ContractExeException {

common/src/main/java/org/tron/common/parameter/CommonParameter.java

@@ -461,6 +461,13 @@ public class CommonParameter {
   @Parameter(names = {"--history-balance-lookup"})
   public boolean historyBalanceLookup = false;
 
+  @Getter
+  @Setter
+  public boolean openPrintLog = true;
+  @Getter
+  @Setter
+  public boolean openTransactionSort = false;
+
   private static double calcMaxTimeRatio() {
     //return max(2.0, min(5.0, 5 * 4.0 / max(Runtime.getRuntime().availableProcessors(), 1)));
     return 5.0;

common/src/main/java/org/tron/core/Constant.java

@@ -273,6 +273,8 @@ public class Constant {
   public static final String METRICS_REPORT_INTERVAL = "node.metrics.influxdb.metricsReportInterval";
 
   public static final String HISTORY_BALANCE_LOOKUP = "storage.balance.history.lookup";
+  public static final String OPEN_PRINT_LOG = "node.openPrintLog";
+  public static final String OPEN_TRANSACTION_SORT = "node.openTransactionSort";
 
   public static final String LOCAL_HOST = "127.0.0.1";
 }

framework/src/main/java/org/tron/core/config/args/Args.java

@@ -185,6 +185,8 @@ public static void clearParam() {
     PARAMETER.allowTvmStake = 0;
     PARAMETER.allowTvmAssetIssue = 0;
     PARAMETER.historyBalanceLookup = false;
+    PARAMETER.openPrintLog = true;
+    PARAMETER.openTransactionSort = false;
   }
 
   /**
@@ -744,6 +746,11 @@ public static void setParam(final String[] args, final String confFileName) {
     PARAMETER.historyBalanceLookup = config.hasPath(Constant.HISTORY_BALANCE_LOOKUP) && config
         .getBoolean(Constant.HISTORY_BALANCE_LOOKUP);
 
+    PARAMETER.openPrintLog = config.hasPath(Constant.OPEN_PRINT_LOG) && config
+        .getBoolean(Constant.OPEN_PRINT_LOG);
+    PARAMETER.openTransactionSort = config.hasPath(Constant.OPEN_TRANSACTION_SORT) && config
+        .getBoolean(Constant.OPEN_TRANSACTION_SORT);
+
     logConfig();
   }
 

framework/src/main/java/org/tron/core/db/Manager.java

@@ -16,9 +16,10 @@
 import com.google.protobuf.ByteString;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Collection;
 import java.util.Collections;
+import java.util.Comparator;
 import java.util.HashSet;
-import java.util.Iterator;
 import java.util.LinkedList;
 import java.util.List;
 import java.util.Objects;
@@ -32,9 +33,11 @@
 import java.util.concurrent.Executors;
 import java.util.concurrent.Future;
 import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.PriorityBlockingQueue;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.atomic.AtomicReference;
 import java.util.stream.Collectors;
 import java.util.stream.LongStream;
 import javax.annotation.PostConstruct;
@@ -205,7 +208,7 @@ public class Manager {
   @Getter
   private ChainBaseManager chainBaseManager;
   // transactions cache
-  private List<TransactionCapsule> pendingTransactions;
+  private BlockingQueue<TransactionCapsule> pendingTransactions;
   @Getter
   private AtomicInteger shieldedTransInPendingCounts = new AtomicInteger(0);
   // transactions popped
@@ -224,7 +227,10 @@ public class Manager {
           TransactionCapsule tx = null;
           try {
             tx = getRePushTransactions().peek();
-            if (tx != null) {
+            if (tx != null && System.currentTimeMillis() - tx.getTime() >= Args.getInstance()
+                .getPendingTransactionTimeout()) {
+              logger.warn("[timeout] remove tx from rePush, txId:{}", tx.getTransactionId());
+            } else if (tx != null) {
               this.rePush(tx);
             } else {
               TimeUnit.MILLISECONDS.sleep(SLEEP_TIME_OUT);
@@ -319,7 +325,7 @@ public BlockIndexStore getBlockIndexStore() {
     return chainBaseManager.getBlockIndexStore();
   }
 
-  public List<TransactionCapsule> getPendingTransactions() {
+  public BlockingQueue<TransactionCapsule> getPendingTransactions() {
     return this.pendingTransactions;
   }
 
@@ -339,6 +345,9 @@ public void stopRePushTriggerThread() {
     isRunTriggerCapsuleProcessThread = false;
   }
 
+  private Comparator downComparator = (Comparator<TransactionCapsule>) (o1, o2) -> Long
+      .compare(o2.getOrder(), o1.getOrder());
+
   @PostConstruct
   public void init() {
     Message.setDynamicPropertiesStore(this.getDynamicPropertiesStore());
@@ -353,8 +362,13 @@ public void init() {
     this.setMerkleContainer(
         merkleContainer.createInstance(chainBaseManager.getMerkleTreeStore(),
             chainBaseManager.getMerkleTreeIndexStore()));
-    this.pendingTransactions = Collections.synchronizedList(Lists.newArrayList());
-    this.rePushTransactions = new LinkedBlockingQueue<>();
+    if (Args.getInstance().isOpenTransactionSort()) {
+      this.pendingTransactions = new PriorityBlockingQueue(2000, downComparator);
+      this.rePushTransactions = new PriorityBlockingQueue<>(2000, downComparator);
+    } else {
+      this.pendingTransactions = new LinkedBlockingQueue<>();
+      this.rePushTransactions = new LinkedBlockingQueue<>();
+    }
     this.triggerCapsuleQueue = new LinkedBlockingQueue<>();
     chainBaseManager.setMerkleContainer(getMerkleContainer());
     chainBaseManager.setMortgageService(mortgageService);
@@ -1182,7 +1196,8 @@ public TransactionInfo processTransaction(final TransactionCapsule trxCap, Block
               trace.getRuntimeResult().getResultCode().name());
       chainBaseManager.getBalanceTraceStore().resetCurrentTransactionTrace();
     }
-
+    //set the sort order
+    trxCap.setOrder(transactionInfo.getFee());
     return transactionInfo.getInstance();
   }
 
@@ -1216,13 +1231,21 @@ public synchronized BlockCapsule generateBlock(Miner miner, long blockTime, long
 
     Set<String> accountSet = new HashSet<>();
     AtomicInteger shieldedTransCounts = new AtomicInteger(0);
-    Iterator<TransactionCapsule> iterator = pendingTransactions.iterator();
-    while (iterator.hasNext() || rePushTransactions.size() > 0) {
+    while (pendingTransactions.size() > 0 || rePushTransactions.size() > 0) {
       boolean fromPending = false;
       TransactionCapsule trx;
-      if (iterator.hasNext()) {
-        fromPending = true;
-        trx = iterator.next();
+      if (pendingTransactions.size() > 0) {
+        trx = pendingTransactions.peek();
+        if (Args.getInstance().isOpenTransactionSort()) {
+          TransactionCapsule trxRepush = rePushTransactions.peek();
+          if (trxRepush == null || trx.getOrder() >= trxRepush.getOrder()) {
+            fromPending = true;
+          } else {
+            trx = rePushTransactions.poll();
+          }
+        } else {
+          fromPending = true;
+        }
       } else {
         trx = rePushTransactions.poll();
       }
@@ -1268,10 +1291,11 @@ public synchronized BlockCapsule generateBlock(Miner miner, long blockTime, long
           transactionRetCapsule.addTransactionInfo(result);
         }
         if (fromPending) {
-          iterator.remove();
+          pendingTransactions.poll();
         }
       } catch (Exception e) {
-        logger.error("Process trx failed when generating block: {}", e.getMessage());
+        logger.error("Process trx {} failed when generating block: {}", trx.getTransactionId(),
+            e.getMessage());
       }
     }
 
@@ -1778,4 +1802,42 @@ public Boolean call() throws ValidateSignatureException {
       return true;
     }
   }
+
+  public TransactionCapsule getTxFromPending(String txId) {
+    AtomicReference<TransactionCapsule> transactionCapsule = new AtomicReference<>();
+    Sha256Hash txHash = Sha256Hash.wrap(ByteArray.fromHexString(txId));
+    pendingTransactions.forEach(tx -> {
+      if (tx.getTransactionId().equals(txHash)) {
+        transactionCapsule.set(tx);
+        return;
+      }
+    });
+    if (transactionCapsule.get() != null) {
+      return transactionCapsule.get();
+    }
+    rePushTransactions.forEach(tx -> {
+      if (tx.getTransactionId().equals(txHash)) {
+        transactionCapsule.set(tx);
+        return;
+      }
+    });
+    return transactionCapsule.get();
+  }
+
+  public Collection<String> getTxListFromPending() {
+    Set<String> result = new HashSet<>();
+    pendingTransactions.forEach(tx -> {
+      result.add(tx.getTransactionId().toString());
+    });
+    rePushTransactions.forEach(tx -> {
+      result.add(tx.getTransactionId().toString());
+    });
+    return result;
+  }
+
+  public long getPendingSize() {
+    long value = getPendingTransactions().size() + getRePushTransactions().size()
+        + getPoppedTransactions().size();
+    return value;
+  }
 }

framework/src/main/java/org/tron/core/db/PendingManager.java

@@ -1,60 +1,44 @@
 package org.tron.core.db;
 
-import java.util.ArrayList;
-import java.util.List;
-import lombok.Getter;
 import lombok.extern.slf4j.Slf4j;
 import org.tron.core.capsule.TransactionCapsule;
 import org.tron.core.config.args.Args;
-import org.tron.core.db.TransactionTrace.TimeResultType;
-import org.tron.core.metrics.MetricsKey;
-import org.tron.core.metrics.MetricsUtil;
 
 @Slf4j(topic = "DB")
 public class PendingManager implements AutoCloseable {
 
-  @Getter
-  private List<TransactionCapsule> tmpTransactions = new ArrayList<>();
   private Manager dbManager;
   private long timeout = Args.getInstance().getPendingTransactionTimeout();
 
   public PendingManager(Manager db) {
     this.dbManager = db;
-    db.getPendingTransactions().forEach(transactionCapsule -> {
-      if (System.currentTimeMillis() - transactionCapsule.getTime() < timeout) {
-        tmpTransactions.add(transactionCapsule);
-      }
-    });
-
-    if (db.getPendingTransactions().size() > tmpTransactions.size()) {
-      MetricsUtil.meterMark(MetricsKey.BLOCKCHAIN_MISSED_TRANSACTION,
-          db.getPendingTransactions().size() - tmpTransactions.size());
-    }
-
-    db.getPendingTransactions().clear();
     db.getSession().reset();
     db.getShieldedTransInPendingCounts().set(0);
   }
 
   @Override
   public void close() {
 
-    for (TransactionCapsule tx : tmpTransactions) {
+    for (TransactionCapsule tx : dbManager.getPendingTransactions()) {
       txIteration(tx);
     }
-    tmpTransactions.clear();
-
+    dbManager.getPendingTransactions().clear();
     for (TransactionCapsule tx : dbManager.getPoppedTransactions()) {
+      tx.setTime(System.currentTimeMillis());
       txIteration(tx);
     }
     dbManager.getPoppedTransactions().clear();
+    if (Args.getInstance().isOpenPrintLog()) {
+      logger.warn("pending tx size:{}", dbManager.getRePushTransactions().size());
+    }
   }
 
   private void txIteration(TransactionCapsule tx) {
     try {
-      if (tx.getTrxTrace() != null
-          && tx.getTrxTrace().getTimeResultType().equals(TimeResultType.NORMAL)) {
+      if (System.currentTimeMillis() - tx.getTime() < timeout) {
         dbManager.getRePushTransactions().put(tx);
+      } else if (Args.getInstance().isOpenPrintLog()) {
+        logger.warn("[timeout] remove tx from pending, txId:{}", tx.getTransactionId());
       }
     } catch (InterruptedException e) {
       logger.error(e.getMessage());

framework/src/main/java/org/tron/core/net/messagehandler/InventoryMsgHandler.java

@@ -4,6 +4,7 @@
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Component;
 import org.tron.common.utils.Sha256Hash;
+import org.tron.core.config.args.Args;
 import org.tron.core.net.TronNetDelegate;
 import org.tron.core.net.message.InventoryMessage;
 import org.tron.core.net.message.TronMessage;
@@ -58,12 +59,18 @@ private boolean check(PeerConnection peer, InventoryMessage inventoryMessage) {
       if (count > maxCountIn10s) {
         logger.warn("Drop inv: {} size: {} from Peer {}, Inv count: {} is overload.",
             type, size, peer.getInetAddress(), count);
+        if (Args.getInstance().isOpenPrintLog()) {
+          logger.warn("[overload]Drop tx list is: {}", inventoryMessage.getHashList());
+        }
         return false;
       }
 
       if (transactionsMsgHandler.isBusy()) {
         logger.warn("Drop inv: {} size: {} from Peer {}, transactionsMsgHandler is busy.",
             type, size, peer.getInetAddress());
+        if (Args.getInstance().isOpenPrintLog()) {
+          logger.warn("[isBusy]Drop tx list is: {}", inventoryMessage.getHashList());
+        }
         return false;
       }
     }