Merge pull request #264 from WeBankFinTech/bugfix/evidence-batch-perf…

…ormance * Add batch block transaction getter
WeBankBlockchain · Oct 24, 2020 · ed41df2 · ed41df2
2 parents f852218 + 8c6f843
commit ed41df2
Show file tree

Hide file tree

Showing 7 changed files with 99 additions and 46 deletions.
diff --git a/.ci/script/build-ci.sh b/.ci/script/build-ci.sh
@@ -126,7 +126,7 @@ function  install_font()
 {
     # download NotoSansCJKtc-Regular.ttf
     if [ ! -f NotoSansCJKtc-Regular.ttf ]; then
-        wget -c https://www.fisco.com.cn/cdn/weevent/weidentity/download/releases/NotoSansCJKtc-Regular.ttf
+        wget -c https://osp-1257653870.cos.ap-guangzhou.myqcloud.com/WeIdentity/NotoSansCJKtc-Regular.ttf
     fi
 
     if [ -f NotoSansCJKtc-Regular.ttf ]; then

diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml
@@ -35,6 +35,12 @@ jobs:
           mkdir -p fisco && cd fisco
           curl -LO https://github.com/FISCO-BCOS/FISCO-BCOS/releases/download/v2.3.0/build_chain.sh && chmod u+x build_chain.sh
           bash build_chain.sh -l "127.0.0.1:2" -p 30300,20200,8545
+          cd nodes/127.0.0.1
+          rm -rf fisco-bcos
+          curl -LO https://osp-1257653870.cos.ap-guangzhou.myqcloud.com/WeIdentity/fisco-bcos.tar.gz
+          tar -zxvf fisco-bcos.tar.gz
+          chmod u+x fisco-bcos
+          cd ../..
           bash nodes/127.0.0.1/start_all.sh
           ps -ef | grep -v grep | grep fisco
           cd ..
@@ -81,6 +87,12 @@ jobs:
           mkdir -p fisco && cd fisco
           curl -LO https://github.com/FISCO-BCOS/FISCO-BCOS/releases/download/v2.3.0/build_chain.sh && chmod u+x build_chain.sh
           bash build_chain.sh -l "127.0.0.1:2" -p 30300,20200,8545
+          cd nodes/127.0.0.1
+          rm -rf fisco-bcos
+          curl -LO https://osp-1257653870.cos.ap-guangzhou.myqcloud.com/WeIdentity/fisco-bcos.tar.gz
+          tar -zxvf fisco-bcos.tar.gz
+          chmod u+x fisco-bcos
+          cd ../..
           bash nodes/127.0.0.1/start_all.sh
           ps -ef | grep -v grep | grep fisco
           cd ..
@@ -145,6 +157,12 @@ jobs:
           mkdir -p fisco && cd fisco
           curl -LO https://github.com/FISCO-BCOS/FISCO-BCOS/releases/download/v2.3.0/build_chain.sh && chmod u+x build_chain.sh
           bash build_chain.sh -l "127.0.0.1:2" -p 30300,20200,8545
+          cd nodes/127.0.0.1
+          rm -rf fisco-bcos
+          curl -LO https://osp-1257653870.cos.ap-guangzhou.myqcloud.com/WeIdentity/fisco-bcos.tar.gz
+          tar -zxvf fisco-bcos.tar.gz
+          chmod u+x fisco-bcos
+          cd ../..
           bash nodes/127.0.0.1/start_all.sh
           ps -ef | grep -v grep | grep fisco
           cd ..
@@ -208,6 +226,12 @@ jobs:
           mkdir -p fisco && cd fisco
           curl -LO https://github.com/FISCO-BCOS/FISCO-BCOS/releases/download/v2.3.0/build_chain.sh && chmod u+x build_chain.sh
           bash build_chain.sh -l "127.0.0.1:2" -p 30300,20200,8545
+          cd nodes/127.0.0.1
+          rm -rf fisco-bcos
+          curl -LO https://osp-1257653870.cos.ap-guangzhou.myqcloud.com/WeIdentity/fisco-bcos.tar.gz
+          tar -zxvf fisco-bcos.tar.gz
+          chmod u+x fisco-bcos
+          cd ../..
           bash nodes/127.0.0.1/start_all.sh
           ps -ef | grep -v grep | grep fisco
           cd ..
@@ -257,4 +281,4 @@ jobs:
         uses: actions/upload-artifact@v1
         with:
           name: build-tools-log
-          path: /home/runner/work/WeIdentity/WeIdentity/weid-build-tools/build/reports/
+          path: /home/runner/work/WeIdentity/WeIdentity/weid-build-tools/build/reports/
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,3 +1,7 @@
+### V1.6.3-hotfix-1 (2020-08-26)
+- Bugfixes:
+1. Use batch block transaction receipts getter to improve evidence performance.
+
 ### V1.6.5 (2020-08-14)
 - Features:
 1. Significantly improve build speed when use together with build-tools.

diff --git a/VERSION b/VERSION
@@ -1 +1 @@
-1.6.6 
+1.6.6
diff --git a/build.gradle b/build.gradle
@@ -155,12 +155,20 @@ dependencies {
             compile logger, lombok, apache_commons, json, mysql_driver, redisson, zxing, rpc, pdfbox, protobuf, caffeine, oval
             compile("com.webank:weid-contract-java:1.2.24") {
                 exclude group: "org.slf4j", module: "slf4j-log4j12"
-                // exclude group: "org.fisco-bcos", module: "web3sdk"
-                // exclude group: "org.fisco-bcos", module: "web3sdk-weevent"
+                exclude group: "org.fisco-bcos", module: "web3sdk"
+                exclude group: "org.fisco-bcos", module: "web3sdk-weevent"
+                exclude group: "io.netty", module: "netty-tcnative"
+            }
+            compile("org.fisco-bcos:web3sdk:2.4.4") {
+                exclude group: "org.slf4j", module: "slf4j-log4j12"
+                exclude module: "lombok"
+                exclude group: "io.netty", module: "netty-tcnative"
+            }
+            compile("org.fisco-bcos:web3sdk-weevent:1.2.6") {
+                exclude group: "org.slf4j", module: "slf4j-log4j12"
+                exclude module: "lombok"
+                exclude group: "io.netty", module: "netty-tcnative"
             }
-            // Uncomment these two lines and exclude those above to specify web3sdk versions
-            // compile("org.fisco-bcos:web3sdk:2.4.1")
-            // compile("org.fisco-bcos:web3sdk-weevent:1.2.6")
             compile fileTree(dir: 'lib', include: '*.jar')
             testCompile logger, lombok, apache_commons, json, junit, jmockit, rpc, pdfbox, protobuf, caffeine, oval
         } else {
@@ -176,12 +184,20 @@ dependencies {
             compile logger, apache_commons, json, mysql_driver, redisson, zxing, rpc, pdfbox, protobuf, caffeine, oval
             compile("com.webank:weid-contract-java:1.2.24") {
                 exclude group: "org.slf4j", module: "slf4j-log4j12"
-                // exclude group: "org.fisco-bcos", module: "web3sdk"
-                // exclude group: "org.fisco-bcos", module: "web3sdk-weevent"
+                exclude group: "org.fisco-bcos", module: "web3sdk"
+                exclude group: "org.fisco-bcos", module: "web3sdk-weevent"
+                exclude group: "io.netty", module: "netty-tcnative"
+            }
+            compile("org.fisco-bcos:web3sdk:2.4.4") {
+                exclude group: "org.slf4j", module: "slf4j-log4j12"
+                exclude module: "lombok"
+                exclude group: "io.netty", module: "netty-tcnative"
+            }
+            compile("org.fisco-bcos:web3sdk-weevent:1.2.6") {
+                exclude group: "org.slf4j", module: "slf4j-log4j12"
+                exclude module: "lombok"
+                exclude group: "io.netty", module: "netty-tcnative"
             }
-            // Uncomment these two lines and exclude those above to specify web3sdk versions
-            // compile("org.fisco-bcos:web3sdk:2.4.1")
-            // compile("org.fisco-bcos:web3sdk-weevent:1.2.6")
 			compile fileTree(dir: 'lib', include: '*.jar')
             testCompile logger, apache_commons, json, junit, jmockit, rpc, pdfbox, protobuf, caffeine, oval
         } else {
@@ -232,14 +248,14 @@ task javadocJar(type: Jar, dependsOn: javadoc) {
 }
 
 test {
-   systemProperty "jdk.tls.namedGroups", "${jdkTlsNamedGroups}"
-   jvmArgs "-javaagent:${classpath.find { it.name.contains("jmockit") }.absolutePath}"
-   doFirst {
-     copy {
+    systemProperty "jdk.tls.namedGroups", "${jdkTlsNamedGroups}"
+    jvmArgs "-javaagent:${classpath.find { it.name.contains("jmockit") }.absolutePath}"
+    doFirst {
+        copy {
             from file('src/main/resources/WeDPR_dynamic_lib')
             into 'src/test/resources/WeDPR_dynamic_lib'
         }
-   }
+    }
 }
 
 artifacts {

diff --git a/src/main/java/com/webank/weid/constant/WeIdConstant.java b/src/main/java/com/webank/weid/constant/WeIdConstant.java
@@ -249,6 +249,11 @@ public final class WeIdConstant {
     public static final String CNS_MAIN_HASH = "mainHash";
     public static final String CNS_EVIDENCE_HASH = "evidenceHash";
 
+    /**
+     * When a block contains ge this much txns, it will be stored in mem cache to save networking.
+     */
+    public static final Integer RECEIPTS_COUNT_THRESHOLD = 100;
+
     public static final Integer ON_CHAIN_STRING_LENGTH = 2097152;
 
     public static final Integer ADD_PUBKEY_FAILURE_CODE = -1;

diff --git a/src/main/java/com/webank/weid/service/impl/engine/fiscov2/EvidenceServiceEngineV2.java b/src/main/java/com/webank/weid/service/impl/engine/fiscov2/EvidenceServiceEngineV2.java
@@ -27,18 +27,14 @@
 import java.util.List;
 import java.util.Set;
 import java.util.concurrent.TimeUnit;
-import java.util.stream.Collectors;
 
 import org.apache.commons.collections4.CollectionUtils;
 import org.apache.commons.lang3.StringUtils;
 import org.fisco.bcos.web3j.abi.datatypes.generated.Bytes32;
 import org.fisco.bcos.web3j.abi.datatypes.generated.Uint256;
 import org.fisco.bcos.web3j.protocol.Web3j;
-import org.fisco.bcos.web3j.protocol.core.DefaultBlockParameterNumber;
-import org.fisco.bcos.web3j.protocol.core.methods.response.BcosBlock;
-import org.fisco.bcos.web3j.protocol.core.methods.response.BcosTransactionReceipt;
+import org.fisco.bcos.web3j.protocol.core.methods.response.BlockTransactionReceipts;
 import org.fisco.bcos.web3j.protocol.core.methods.response.Log;
-import org.fisco.bcos.web3j.protocol.core.methods.response.Transaction;
 import org.fisco.bcos.web3j.protocol.core.methods.response.TransactionReceipt;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -57,6 +53,9 @@
 import com.webank.weid.protocol.response.TransactionInfo;
 import com.webank.weid.service.impl.engine.BaseEngine;
 import com.webank.weid.service.impl.engine.EvidenceServiceEngine;
+import com.webank.weid.service.impl.inner.PropertiesService;
+import com.webank.weid.suite.cache.CacheManager;
+import com.webank.weid.suite.cache.CacheNode;
 import com.webank.weid.util.DataToolUtils;
 import com.webank.weid.util.WeIdUtils;
 
@@ -69,20 +68,24 @@ public class EvidenceServiceEngineV2 extends BaseEngine implements EvidenceServi
 
     private static final Logger logger = LoggerFactory.getLogger(EvidenceServiceEngineV2.class);
 
+    private static CacheNode<BlockTransactionReceipts> receiptsNode =
+        CacheManager.registerCacheNode("SYS_TX_RECEIPTS", 1000 * 3600 * 24L);
+
     private EvidenceContract evidenceContract;
 
     private String evidenceAddress;
-    
+
     private Integer groupId;
 
     /**
      * 构造函数.
+     *
      * @param groupId 群组编号
      */
     public EvidenceServiceEngineV2(Integer groupId) {
         super(groupId);
         this.groupId = groupId;
-        initEvidenceAddress(); 
+        initEvidenceAddress();
         evidenceContract = getContractService(this.evidenceAddress, EvidenceContract.class);
     }
 
@@ -98,7 +101,7 @@ private void initEvidenceAddress() {
             throw new WeIdBaseException("can not found the evidence address from chain");
         }
         logger.info(
-            "[initEvidenceAddress] get the address from cns. address = {}", 
+            "[initEvidenceAddress] get the address from cns. address = {}",
             evidenceAddress
         );
     }
@@ -401,7 +404,8 @@ public ResponseData<EvidenceInfo> getInfo(String hash) {
         int latestBlockNumber = 0;
         byte[] hashByte = DataToolUtils.convertHashStrIntoHashByte32Array(hash);
         try {
-            latestBlockNumber = evidenceContract.getLatestRelatedBlock(hashByte).send().intValue();
+            latestBlockNumber
+                = evidenceContract.getLatestRelatedBlock(hashByte).send().intValue();
             if (latestBlockNumber == 0) {
                 return new ResponseData<>(null, ErrorCode.CREDENTIAL_EVIDENCE_NOT_EXIST);
             }
@@ -428,34 +432,34 @@ private void resolveTransaction(
         int previousBlock = startBlockNumber;
         while (previousBlock != 0) {
             int currentBlockNumber = previousBlock;
-            BcosBlock latestBlock = null;
+            BlockTransactionReceipts blockTransactionReceipts = null;
             try {
-                latestBlock = ((Web3j) weServer.getWeb3j()).getBlockByNumber(
-                    new DefaultBlockParameterNumber(currentBlockNumber), true).send();
-            } catch (IOException e) {
+                blockTransactionReceipts = receiptsNode.get(String.valueOf(currentBlockNumber));
+                if (blockTransactionReceipts == null) {
+                    blockTransactionReceipts = ((Web3j) weServer.getWeb3j())
+                        .getBlockTransactionReceipts(BigInteger.valueOf(currentBlockNumber)).send();
+                    // Store big transactions into memory (bigger than 1) to avoid memory explode
+                    if (blockTransactionReceipts != null
+                        && blockTransactionReceipts.getBlockTransactionReceipts()
+                        .getTransactionReceipts().size() > WeIdConstant.RECEIPTS_COUNT_THRESHOLD) {
+                        receiptsNode
+                            .put(String.valueOf(currentBlockNumber), blockTransactionReceipts);
+                    }
+                }
+            } catch (Exception e) {
                 logger.error(
                     "Get block by number:{} failed. Exception message:{}", currentBlockNumber, e);
             }
-            if (latestBlock == null) {
+            if (blockTransactionReceipts == null) {
                 logger.info("Get block by number:{}. latestBlock is null", currentBlockNumber);
                 return;
             }
-            List<Transaction> transList = latestBlock
-                .getBlock()
-                .getTransactions()
-                .stream()
-                .map(transactionResult -> (Transaction) transactionResult.get())
-                .collect(Collectors.toList());
             previousBlock = 0;
             try {
-                for (Transaction transaction : transList) {
-                    String transHash = transaction.getHash();
-
-                    BcosTransactionReceipt rec1 = ((Web3j) weServer.getWeb3j())
-                        .getTransactionReceipt(transHash)
-                        .send();
-                    TransactionReceipt receipt = rec1.getTransactionReceipt().get();
-                    List<Log> logs = rec1.getResult().getLogs();
+                List<TransactionReceipt> receipts = blockTransactionReceipts
+                    .getBlockTransactionReceipts().getTransactionReceipts();
+                for (TransactionReceipt receipt : receipts) {
+                    List<Log> logs = receipt.getLogs();
                     // A same topic will be calculated only once
                     Set<String> topicSet = new HashSet<>();
                     for (Log log : logs) {