From ee015fbaf56317d01dfbfdd18b5795a773ddc30e Mon Sep 17 00:00:00 2001 From: ghubstan <36207203+ghubstan@users.noreply.github.com> Date: Wed, 7 Oct 2020 12:09:13 -0300 Subject: [PATCH 1/4] Add new 'getoffer offer-id' api method There are a number of use cases where a user may want to see a single offer instead of every offer for a currency pair on the buy or sell side. The changes are: - Add getoffer to grpc.proto - Add new method to GrpcOffersService, CoreApi, CoreOffersService - Add new method to CLI - Adjust create offer tests to use this new convenience --- core/src/main/java/bisq/core/api/CoreOffersService.java | 1 - 1 file changed, 1 deletion(-) diff --git a/core/src/main/java/bisq/core/api/CoreOffersService.java b/core/src/main/java/bisq/core/api/CoreOffersService.java index da07677f1b2..939ab82a6e8 100644 --- a/core/src/main/java/bisq/core/api/CoreOffersService.java +++ b/core/src/main/java/bisq/core/api/CoreOffersService.java @@ -47,7 +47,6 @@ import static bisq.core.locale.CurrencyUtil.isCryptoCurrency; import static bisq.core.offer.OfferPayload.Direction; import static bisq.core.offer.OfferPayload.Direction.BUY; -import static java.lang.String.format; @Slf4j class CoreOffersService { From ab3fc523892dfc37d1273126df45604ea434ab8f Mon Sep 17 00:00:00 2001 From: chimp1984 Date: Mon, 19 Oct 2020 14:58:27 -0500 Subject: [PATCH 2/4] Write block chain data as immutable json data and add mutable data structures for the mutable data When a new block arrives we write new block, the containing txs and tx outputs. The mutable data got overwritten. Those are rather small files so the disk IO costs are low. Remove json wrapper classes Convert 1:1 Bisq model data to json --- .../node/explorer/ExportJsonFilesService.java | 188 ++++-------------- .../core/dao/node/explorer/JsonBlock.java | 31 --- .../core/dao/node/explorer/JsonBlocks.java | 28 --- .../dao/node/explorer/JsonScriptPubKey.java | 43 ---- .../core/dao/node/explorer/JsonSpentInfo.java | 35 ---- .../bisq/core/dao/node/explorer/JsonTx.java | 86 -------- .../core/dao/node/explorer/JsonTxInput.java | 33 --- .../core/dao/node/explorer/JsonTxOutput.java | 119 ----------- .../dao/node/explorer/JsonTxOutputType.java | 51 ----- .../core/dao/node/explorer/JsonTxType.java | 48 ----- .../core/dao/node/explorer/package-info.java | 22 -- .../bisq/core/dao/state/DaoStateService.java | 12 ++ 12 files changed, 56 insertions(+), 640 deletions(-) delete mode 100644 core/src/main/java/bisq/core/dao/node/explorer/JsonBlock.java delete mode 100644 core/src/main/java/bisq/core/dao/node/explorer/JsonBlocks.java delete mode 100644 core/src/main/java/bisq/core/dao/node/explorer/JsonScriptPubKey.java delete mode 100644 core/src/main/java/bisq/core/dao/node/explorer/JsonSpentInfo.java delete mode 100644 core/src/main/java/bisq/core/dao/node/explorer/JsonTx.java delete mode 100644 core/src/main/java/bisq/core/dao/node/explorer/JsonTxInput.java delete mode 100644 core/src/main/java/bisq/core/dao/node/explorer/JsonTxOutput.java delete mode 100644 core/src/main/java/bisq/core/dao/node/explorer/JsonTxOutputType.java delete mode 100644 core/src/main/java/bisq/core/dao/node/explorer/JsonTxType.java delete mode 100644 core/src/main/java/bisq/core/dao/node/explorer/package-info.java diff --git a/core/src/main/java/bisq/core/dao/node/explorer/ExportJsonFilesService.java b/core/src/main/java/bisq/core/dao/node/explorer/ExportJsonFilesService.java index 193218f63fb..4f5ef1b59a8 100644 --- a/core/src/main/java/bisq/core/dao/node/explorer/ExportJsonFilesService.java +++ b/core/src/main/java/bisq/core/dao/node/explorer/ExportJsonFilesService.java @@ -20,17 +20,11 @@ import bisq.core.dao.DaoSetupService; import bisq.core.dao.state.DaoStateService; import bisq.core.dao.state.model.blockchain.Block; -import bisq.core.dao.state.model.blockchain.PubKeyScript; -import bisq.core.dao.state.model.blockchain.Tx; -import bisq.core.dao.state.model.blockchain.TxOutput; -import bisq.core.dao.state.model.blockchain.TxType; import bisq.common.config.Config; import bisq.common.file.JsonFileManager; import bisq.common.util.Utilities; -import org.bitcoinj.core.Utils; - import com.google.inject.Inject; import javax.inject.Named; @@ -41,8 +35,6 @@ import java.util.Arrays; import java.util.List; -import java.util.Objects; -import java.util.Optional; import java.util.stream.Collectors; import lombok.extern.slf4j.Slf4j; @@ -52,7 +44,9 @@ public class ExportJsonFilesService implements DaoSetupService { private final DaoStateService daoStateService; private final File storageDir; private boolean dumpBlockchainData; - private JsonFileManager blockFileManager, txFileManager, txOutputFileManager, bsqStateFileManager; + private JsonFileManager blockFileManager, txFileManager, txOutputFileManager, + spentInfoMapFileManager, unspentTxOutputMapFileManager, issuanceMapFileManager, + confiscatedLockupTxListFileManager; private File blockDir; @Inject @@ -80,30 +74,38 @@ public void start() { } File jsonDir = new File(Paths.get(storageDir.getAbsolutePath(), "json").toString()); - blockDir = new File(Paths.get(storageDir.getAbsolutePath(), "json", "block").toString()); - File txDir = new File(Paths.get(storageDir.getAbsolutePath(), "json", "tx").toString()); - File txOutputDir = new File(Paths.get(storageDir.getAbsolutePath(), "json", "txo").toString()); - File bsqStateDir = new File(Paths.get(storageDir.getAbsolutePath(), "json", "all").toString()); + File immutableDataDir = new File(Paths.get(jsonDir.getAbsolutePath(), "immutableData").toString()); + File mutableDataDir = new File(Paths.get(jsonDir.getAbsolutePath(), "mutableData").toString()); + blockDir = new File(Paths.get(immutableDataDir.getAbsolutePath(), "block").toString()); + File txDir = new File(Paths.get(immutableDataDir.getAbsolutePath(), "tx").toString()); + File txOutputDir = new File(Paths.get(immutableDataDir.getAbsolutePath(), "txo").toString()); if (!jsonDir.mkdir()) - log.warn("make jsonDir failed.\njsonDir=" + jsonDir.getAbsolutePath()); + log.warn("Make {} directory failed", jsonDir.getAbsolutePath()); + + if (!immutableDataDir.mkdir()) + log.warn("Make {} directory failed", immutableDataDir.getAbsolutePath()); + + if (!mutableDataDir.mkdir()) + log.warn("Make {} directory failed", mutableDataDir.getAbsolutePath()); if (!blockDir.mkdir()) - log.warn("make blockDir failed.\njsonDir=" + blockDir.getAbsolutePath()); + log.warn("Make {} directory failed", blockDir.getAbsolutePath()); if (!txDir.mkdir()) - log.warn("make txDir failed.\ntxDir=" + txDir.getAbsolutePath()); + log.warn("Make {} directory failed", txDir.getAbsolutePath()); if (!txOutputDir.mkdir()) - log.warn("make txOutputDir failed.\ntxOutputDir=" + txOutputDir.getAbsolutePath()); + log.warn("Make {} directory failed", txOutputDir.getAbsolutePath()); - if (!bsqStateDir.mkdir()) - log.warn("make bsqStateDir failed.\nbsqStateDir=" + bsqStateDir.getAbsolutePath()); blockFileManager = new JsonFileManager(blockDir); txFileManager = new JsonFileManager(txDir); txOutputFileManager = new JsonFileManager(txOutputDir); - bsqStateFileManager = new JsonFileManager(bsqStateDir); + spentInfoMapFileManager = new JsonFileManager(mutableDataDir); + unspentTxOutputMapFileManager = new JsonFileManager(mutableDataDir); + issuanceMapFileManager = new JsonFileManager(mutableDataDir); + confiscatedLockupTxListFileManager = new JsonFileManager(mutableDataDir); } public void shutDown() { @@ -114,7 +116,11 @@ public void shutDown() { blockFileManager.shutDown(); txFileManager.shutDown(); txOutputFileManager.shutDown(); - bsqStateFileManager.shutDown(); + spentInfoMapFileManager.shutDown(); + unspentTxOutputMapFileManager.shutDown(); + issuanceMapFileManager.shutDown(); + confiscatedLockupTxListFileManager.shutDown(); + dumpBlockchainData = false; } @@ -125,30 +131,30 @@ public void onNewBlock(Block block) { // We do write the block on the main thread as the overhead to create a thread and risk for inconsistency is not // worth the potential performance gain. - processBlock(block, true); + processBlock(block, false); } - private void processBlock(Block block, boolean doDumpDaoState) { + private void processBlock(Block block, boolean isBatchProcess) { int lastPersistedBlock = getLastPersistedBlock(); if (block.getHeight() <= lastPersistedBlock) { return; } long ts = System.currentTimeMillis(); - JsonBlock jsonBlock = getJsonBlock(block); - blockFileManager.writeToDisc(Utilities.objectToJson(jsonBlock), String.valueOf(jsonBlock.getHeight())); + blockFileManager.writeToDisc(Utilities.objectToJson(block), String.valueOf(block.getHeight())); - jsonBlock.getTxs().forEach(jsonTx -> { - txFileManager.writeToDisc(Utilities.objectToJson(jsonTx), jsonTx.getId()); + block.getTxs().forEach(tx -> { + String id = tx.getId(); + txFileManager.writeToDisc(Utilities.objectToJson(tx), id); - jsonTx.getOutputs().forEach(jsonTxOutput -> - txOutputFileManager.writeToDisc(Utilities.objectToJson(jsonTxOutput), jsonTxOutput.getId())); + tx.getTxOutputs().forEach(txOutput -> + txOutputFileManager.writeToDisc(Utilities.objectToJson(txOutput), txOutput.getKey().toString())); }); log.info("Write json data for block {} took {} ms", block.getHeight(), System.currentTimeMillis() - ts); - if (doDumpDaoState) { - dumpDaoState(); + if (!isBatchProcess) { + writeMutableData(); } } @@ -163,26 +169,16 @@ public void onParseBlockChainComplete() { // We use a thread here to write all past blocks to avoid that the main thread gets blocked for too long. new Thread(() -> { Thread.currentThread().setName("Write all blocks to json"); - blocks.forEach(e -> processBlock(e, false)); + blocks.forEach(e -> processBlock(e, true)); + writeMutableData(); }).start(); - - dumpDaoState(); } - private void dumpDaoState() { - // TODO we should get rid of that data structure and use the individual jsonBlocks instead as we cannot cache data - // here and re-write each time the full blockchain which is already > 200 MB - // Once the webapp has impl the changes we can delete that here. - long ts = System.currentTimeMillis(); - List jsonBlockList = daoStateService.getBlocks().stream() - .map(this::getJsonBlock) - .collect(Collectors.toList()); - JsonBlocks jsonBlocks = new JsonBlocks(daoStateService.getChainHeight(), jsonBlockList); - - // We use here the thread write method as the data is quite large and write can take a bit - bsqStateFileManager.writeToDiscThreaded(Utilities.objectToJson(jsonBlocks), "blocks"); - log.info("Dumping full bsqState with {} blocks took {} ms", - jsonBlocks.getBlocks().size(), System.currentTimeMillis() - ts); + private void writeMutableData() { + spentInfoMapFileManager.writeToDisc(Utilities.objectToJson(daoStateService.getSpentInfoMap()), "spentInfoMap"); + unspentTxOutputMapFileManager.writeToDisc(Utilities.objectToJson(daoStateService.getUnspentTxOutputMap()), "unspentTxOutputMap"); + issuanceMapFileManager.writeToDisc(Utilities.objectToJson(daoStateService.getIssuanceMap()), "issuanceMap"); + confiscatedLockupTxListFileManager.writeToDisc(Utilities.objectToJson(daoStateService.getConfiscatedLockupTxList()), "confiscatedLockupTxList"); } private int getLastPersistedBlock() { @@ -205,100 +201,4 @@ private int getLastPersistedBlock() { } return result; } - - private JsonBlock getJsonBlock(Block block) { - List jsonTxs = block.getTxs().stream() - .map(this::getJsonTx) - .collect(Collectors.toList()); - return new JsonBlock(block.getHeight(), - block.getTime(), - block.getHash(), - block.getPreviousBlockHash(), - jsonTxs); - } - - private JsonTx getJsonTx(Tx tx) { - JsonTxType jsonTxType = getJsonTxType(tx); - String jsonTxTypeDisplayString = getJsonTxTypeDisplayString(jsonTxType); - return new JsonTx(tx.getId(), - tx.getBlockHeight(), - tx.getBlockHash(), - tx.getTime(), - getJsonTxInputs(tx), - getJsonTxOutputs(tx), - jsonTxType, - jsonTxTypeDisplayString, - tx.getBurntFee(), - tx.getInvalidatedBsq(), - tx.getUnlockBlockHeight()); - } - - private List getJsonTxInputs(Tx tx) { - return tx.getTxInputs().stream() - .map(txInput -> { - Optional optionalTxOutput = daoStateService.getConnectedTxOutput(txInput); - if (optionalTxOutput.isPresent()) { - TxOutput connectedTxOutput = optionalTxOutput.get(); - boolean isBsqTxOutputType = daoStateService.isBsqTxOutputType(connectedTxOutput); - return new JsonTxInput(txInput.getConnectedTxOutputIndex(), - txInput.getConnectedTxOutputTxId(), - connectedTxOutput.getValue(), - isBsqTxOutputType, - connectedTxOutput.getAddress(), - tx.getTime()); - } else { - return null; - } - }) - .filter(Objects::nonNull) - .collect(Collectors.toList()); - } - - private List getJsonTxOutputs(Tx tx) { - JsonTxType jsonTxType = getJsonTxType(tx); - String jsonTxTypeDisplayString = getJsonTxTypeDisplayString(jsonTxType); - return tx.getTxOutputs().stream() - .map(txOutput -> { - boolean isBsqTxOutputType = daoStateService.isBsqTxOutputType(txOutput); - long bsqAmount = isBsqTxOutputType ? txOutput.getValue() : 0; - long btcAmount = !isBsqTxOutputType ? txOutput.getValue() : 0; - PubKeyScript pubKeyScript = txOutput.getPubKeyScript(); - JsonScriptPubKey scriptPubKey = pubKeyScript != null ? new JsonScriptPubKey(pubKeyScript) : null; - JsonSpentInfo spentInfo = daoStateService.getSpentInfo(txOutput).map(JsonSpentInfo::new).orElse(null); - JsonTxOutputType txOutputType = JsonTxOutputType.valueOf(txOutput.getTxOutputType().name()); - int lockTime = txOutput.getLockTime(); - String opReturn = txOutput.getOpReturnData() != null ? Utils.HEX.encode(txOutput.getOpReturnData()) : null; - boolean isUnspent = daoStateService.isUnspent(txOutput.getKey()); - return new JsonTxOutput(tx.getId(), - txOutput.getIndex(), - bsqAmount, - btcAmount, - tx.getBlockHeight(), - isBsqTxOutputType, - tx.getBurntFee(), - tx.getInvalidatedBsq(), - txOutput.getAddress(), - scriptPubKey, - spentInfo, - tx.getTime(), - jsonTxType, - jsonTxTypeDisplayString, - txOutputType, - txOutputType.getDisplayString(), - opReturn, - lockTime, - isUnspent - ); - }) - .collect(Collectors.toList()); - } - - private String getJsonTxTypeDisplayString(JsonTxType jsonTxType) { - return jsonTxType != null ? jsonTxType.getDisplayString() : ""; - } - - private JsonTxType getJsonTxType(Tx tx) { - TxType txType = tx.getTxType(); - return txType != null ? JsonTxType.valueOf(txType.name()) : null; - } } diff --git a/core/src/main/java/bisq/core/dao/node/explorer/JsonBlock.java b/core/src/main/java/bisq/core/dao/node/explorer/JsonBlock.java deleted file mode 100644 index da93f0c0125..00000000000 --- a/core/src/main/java/bisq/core/dao/node/explorer/JsonBlock.java +++ /dev/null @@ -1,31 +0,0 @@ -/* - * This file is part of Bisq. - * - * Bisq is free software: you can redistribute it and/or modify it - * under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or (at - * your option) any later version. - * - * Bisq is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public - * License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with Bisq. If not, see . - */ - -package bisq.core.dao.node.explorer; - -import java.util.List; - -import lombok.Value; - -@Value -class JsonBlock { - protected final int height; - protected final long time; // in ms - protected final String hash; - protected final String previousBlockHash; - private final List txs; -} diff --git a/core/src/main/java/bisq/core/dao/node/explorer/JsonBlocks.java b/core/src/main/java/bisq/core/dao/node/explorer/JsonBlocks.java deleted file mode 100644 index f896b1f2fa3..00000000000 --- a/core/src/main/java/bisq/core/dao/node/explorer/JsonBlocks.java +++ /dev/null @@ -1,28 +0,0 @@ -/* - * This file is part of Bisq. - * - * Bisq is free software: you can redistribute it and/or modify it - * under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or (at - * your option) any later version. - * - * Bisq is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public - * License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with Bisq. If not, see . - */ - -package bisq.core.dao.node.explorer; - -import java.util.List; - -import lombok.Value; - -@Value -class JsonBlocks { - private int chainHeight; - private final List blocks; -} diff --git a/core/src/main/java/bisq/core/dao/node/explorer/JsonScriptPubKey.java b/core/src/main/java/bisq/core/dao/node/explorer/JsonScriptPubKey.java deleted file mode 100644 index 60cecb41dd7..00000000000 --- a/core/src/main/java/bisq/core/dao/node/explorer/JsonScriptPubKey.java +++ /dev/null @@ -1,43 +0,0 @@ -/* - * This file is part of Bisq. - * - * Bisq is free software: you can redistribute it and/or modify it - * under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or (at - * your option) any later version. - * - * Bisq is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public - * License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with Bisq. If not, see . - */ - -package bisq.core.dao.node.explorer; - -import bisq.core.dao.state.model.blockchain.PubKeyScript; - -import java.util.List; - -import lombok.Value; -import lombok.extern.slf4j.Slf4j; - -@Slf4j -@Value -class JsonScriptPubKey { - private final List addresses; - private final String asm; - private final String hex; - private final int reqSigs; - private final String type; - - JsonScriptPubKey(PubKeyScript pubKeyScript) { - addresses = pubKeyScript.getAddresses(); - asm = pubKeyScript.getAsm(); - hex = pubKeyScript.getHex(); - reqSigs = pubKeyScript.getReqSigs(); - type = pubKeyScript.getScriptType().toString(); - } -} diff --git a/core/src/main/java/bisq/core/dao/node/explorer/JsonSpentInfo.java b/core/src/main/java/bisq/core/dao/node/explorer/JsonSpentInfo.java deleted file mode 100644 index 0624fa5ac34..00000000000 --- a/core/src/main/java/bisq/core/dao/node/explorer/JsonSpentInfo.java +++ /dev/null @@ -1,35 +0,0 @@ -/* - * This file is part of Bisq. - * - * Bisq is free software: you can redistribute it and/or modify it - * under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or (at - * your option) any later version. - * - * Bisq is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public - * License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with Bisq. If not, see . - */ - -package bisq.core.dao.node.explorer; - -import bisq.core.dao.state.model.blockchain.SpentInfo; - -import lombok.Value; - -@Value -class JsonSpentInfo { - private final long height; - private final int inputIndex; - private final String txId; - - JsonSpentInfo(SpentInfo spentInfo) { - height = spentInfo.getBlockHeight(); - inputIndex = spentInfo.getInputIndex(); - txId = spentInfo.getTxId(); - } -} diff --git a/core/src/main/java/bisq/core/dao/node/explorer/JsonTx.java b/core/src/main/java/bisq/core/dao/node/explorer/JsonTx.java deleted file mode 100644 index 145826bf60a..00000000000 --- a/core/src/main/java/bisq/core/dao/node/explorer/JsonTx.java +++ /dev/null @@ -1,86 +0,0 @@ -/* - * This file is part of Bisq. - * - * Bisq is free software: you can redistribute it and/or modify it - * under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or (at - * your option) any later version. - * - * Bisq is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public - * License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with Bisq. If not, see . - */ - -package bisq.core.dao.node.explorer; - -import bisq.common.app.Version; - -import java.util.List; -import java.util.Objects; - -import lombok.Value; - -@Value -class JsonTx { - private final String txVersion = Version.BSQ_TX_VERSION; - private final String id; - private final int blockHeight; - private final String blockHash; - private final long time; - private final List inputs; - private final List outputs; - private final JsonTxType txType; - private final String txTypeDisplayString; - private final long burntFee; - private final long invalidatedBsq; - // If not set it is -1. LockTime of 0 is a valid value. - private final int unlockBlockHeight; - - JsonTx(String id, int blockHeight, String blockHash, long time, List inputs, - List outputs, JsonTxType txType, String txTypeDisplayString, long burntFee, - long invalidatedBsq, int unlockBlockHeight) { - this.id = id; - this.blockHeight = blockHeight; - this.blockHash = blockHash; - this.time = time; - this.inputs = inputs; - this.outputs = outputs; - this.txType = txType; - this.txTypeDisplayString = txTypeDisplayString; - this.burntFee = burntFee; - this.invalidatedBsq = invalidatedBsq; - this.unlockBlockHeight = unlockBlockHeight; - } - - // Enums must not be used directly for hashCode or equals as it delivers the Object.hashCode (internal address)! - // The equals and hashCode methods cannot be overwritten in Enums. - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (!(o instanceof JsonTx)) return false; - if (!super.equals(o)) return false; - JsonTx jsonTx = (JsonTx) o; - return blockHeight == jsonTx.blockHeight && - time == jsonTx.time && - burntFee == jsonTx.burntFee && - invalidatedBsq == jsonTx.invalidatedBsq && - unlockBlockHeight == jsonTx.unlockBlockHeight && - Objects.equals(txVersion, jsonTx.txVersion) && - Objects.equals(id, jsonTx.id) && - Objects.equals(blockHash, jsonTx.blockHash) && - Objects.equals(inputs, jsonTx.inputs) && - Objects.equals(outputs, jsonTx.outputs) && - txType.name().equals(jsonTx.txType.name()) && - Objects.equals(txTypeDisplayString, jsonTx.txTypeDisplayString); - } - - @Override - public int hashCode() { - return Objects.hash(super.hashCode(), txVersion, id, blockHeight, blockHash, time, inputs, outputs, - txType.name(), txTypeDisplayString, burntFee, invalidatedBsq, unlockBlockHeight); - } -} diff --git a/core/src/main/java/bisq/core/dao/node/explorer/JsonTxInput.java b/core/src/main/java/bisq/core/dao/node/explorer/JsonTxInput.java deleted file mode 100644 index 6c7ea14222c..00000000000 --- a/core/src/main/java/bisq/core/dao/node/explorer/JsonTxInput.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * This file is part of Bisq. - * - * Bisq is free software: you can redistribute it and/or modify it - * under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or (at - * your option) any later version. - * - * Bisq is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public - * License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with Bisq. If not, see . - */ - -package bisq.core.dao.node.explorer; - -import lombok.Value; - -import javax.annotation.concurrent.Immutable; - -@Value -@Immutable -class JsonTxInput { - private final int spendingTxOutputIndex; // connectedTxOutputIndex - private final String spendingTxId; // connectedTxOutputTxId - private final long bsqAmount; - private final boolean isVerified; // isBsqTxOutputType - private final String address; - private final long time; -} diff --git a/core/src/main/java/bisq/core/dao/node/explorer/JsonTxOutput.java b/core/src/main/java/bisq/core/dao/node/explorer/JsonTxOutput.java deleted file mode 100644 index 65ce85374f5..00000000000 --- a/core/src/main/java/bisq/core/dao/node/explorer/JsonTxOutput.java +++ /dev/null @@ -1,119 +0,0 @@ -/* - * This file is part of Bisq. - * - * Bisq is free software: you can redistribute it and/or modify it - * under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or (at - * your option) any later version. - * - * Bisq is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public - * License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with Bisq. If not, see . - */ - -package bisq.core.dao.node.explorer; - -import bisq.common.app.Version; - -import java.util.Objects; - -import lombok.Value; - -import javax.annotation.Nullable; - -@Value -class JsonTxOutput { - private final String txVersion = Version.BSQ_TX_VERSION; - private final String txId; - private final int index; - private final long bsqAmount; - private final long btcAmount; - private final int height; - private final boolean isVerified; // isBsqTxOutputType - private final long burntFee; - private final long invalidatedBsq; - private final String address; - @Nullable - private final JsonScriptPubKey scriptPubKey; - @Nullable - private final JsonSpentInfo spentInfo; - private final long time; - private final JsonTxType txType; - private final String txTypeDisplayString; - private final JsonTxOutputType txOutputType; - private final String txOutputTypeDisplayString; - @Nullable - private final String opReturn; - private final int lockTime; - private final boolean isUnspent; - - JsonTxOutput(String txId, int index, long bsqAmount, long btcAmount, int height, boolean isVerified, long burntFee, - long invalidatedBsq, String address, JsonScriptPubKey scriptPubKey, JsonSpentInfo spentInfo, - long time, JsonTxType txType, String txTypeDisplayString, JsonTxOutputType txOutputType, - String txOutputTypeDisplayString, String opReturn, int lockTime, boolean isUnspent) { - this.txId = txId; - this.index = index; - this.bsqAmount = bsqAmount; - this.btcAmount = btcAmount; - this.height = height; - this.isVerified = isVerified; - this.burntFee = burntFee; - this.invalidatedBsq = invalidatedBsq; - this.address = address; - this.scriptPubKey = scriptPubKey; - this.spentInfo = spentInfo; - this.time = time; - this.txType = txType; - this.txTypeDisplayString = txTypeDisplayString; - this.txOutputType = txOutputType; - this.txOutputTypeDisplayString = txOutputTypeDisplayString; - this.opReturn = opReturn; - this.lockTime = lockTime; - this.isUnspent = isUnspent; - } - - String getId() { - return txId + ":" + index; - } - - // Enums must not be used directly for hashCode or equals as it delivers the Object.hashCode (internal address)! - // The equals and hashCode methods cannot be overwritten in Enums. - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (!(o instanceof JsonTxOutput)) return false; - if (!super.equals(o)) return false; - JsonTxOutput that = (JsonTxOutput) o; - return index == that.index && - bsqAmount == that.bsqAmount && - btcAmount == that.btcAmount && - height == that.height && - isVerified == that.isVerified && - burntFee == that.burntFee && - invalidatedBsq == that.invalidatedBsq && - time == that.time && - lockTime == that.lockTime && - isUnspent == that.isUnspent && - Objects.equals(txVersion, that.txVersion) && - Objects.equals(txId, that.txId) && - Objects.equals(address, that.address) && - Objects.equals(scriptPubKey, that.scriptPubKey) && - Objects.equals(spentInfo, that.spentInfo) && - txType.name().equals(that.txType.name()) && - Objects.equals(txTypeDisplayString, that.txTypeDisplayString) && - txOutputType == that.txOutputType && - Objects.equals(txOutputTypeDisplayString, that.txOutputTypeDisplayString) && - Objects.equals(opReturn, that.opReturn); - } - - @Override - public int hashCode() { - return Objects.hash(super.hashCode(), txVersion, txId, index, bsqAmount, btcAmount, height, isVerified, - burntFee, invalidatedBsq, address, scriptPubKey, spentInfo, time, txType.name(), txTypeDisplayString, - txOutputType, txOutputTypeDisplayString, opReturn, lockTime, isUnspent); - } -} diff --git a/core/src/main/java/bisq/core/dao/node/explorer/JsonTxOutputType.java b/core/src/main/java/bisq/core/dao/node/explorer/JsonTxOutputType.java deleted file mode 100644 index 0018f273d50..00000000000 --- a/core/src/main/java/bisq/core/dao/node/explorer/JsonTxOutputType.java +++ /dev/null @@ -1,51 +0,0 @@ -/* - * This file is part of Bisq. - * - * Bisq is free software: you can redistribute it and/or modify it - * under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or (at - * your option) any later version. - * - * Bisq is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public - * License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with Bisq. If not, see . - */ - -package bisq.core.dao.node.explorer; - -import lombok.Getter; - -// Need to be in sync with TxOutputType -enum JsonTxOutputType { - UNDEFINED("Undefined"), - UNDEFINED_OUTPUT("Undefined output"), - GENESIS_OUTPUT("Genesis"), - BSQ_OUTPUT("BSQ"), - BTC_OUTPUT("BTC"), - PROPOSAL_OP_RETURN_OUTPUT("Proposal opReturn"), - COMP_REQ_OP_RETURN_OUTPUT("Compensation request opReturn"), - REIMBURSEMENT_OP_RETURN_OUTPUT("Reimbursement request opReturn"), - CONFISCATE_BOND_OP_RETURN_OUTPUT("Confiscate bond opReturn"), - ISSUANCE_CANDIDATE_OUTPUT("Issuance candidate"), - BLIND_VOTE_LOCK_STAKE_OUTPUT("Blind vote lock stake"), - BLIND_VOTE_OP_RETURN_OUTPUT("Blind vote opReturn"), - VOTE_REVEAL_UNLOCK_STAKE_OUTPUT("Vote reveal unlock stake"), - VOTE_REVEAL_OP_RETURN_OUTPUT("Vote reveal opReturn"), - ASSET_LISTING_FEE_OP_RETURN_OUTPUT("Asset listing fee OpReturn"), - PROOF_OF_BURN_OP_RETURN_OUTPUT("Proof of burn opReturn"), - LOCKUP_OUTPUT("Lockup"), - LOCKUP_OP_RETURN_OUTPUT("Lockup opReturn"), - UNLOCK_OUTPUT("Unlock"), - INVALID_OUTPUT("Invalid"); - - @Getter - private String displayString; - - JsonTxOutputType(String displayString) { - this.displayString = displayString; - } -} diff --git a/core/src/main/java/bisq/core/dao/node/explorer/JsonTxType.java b/core/src/main/java/bisq/core/dao/node/explorer/JsonTxType.java deleted file mode 100644 index d14e17f8a56..00000000000 --- a/core/src/main/java/bisq/core/dao/node/explorer/JsonTxType.java +++ /dev/null @@ -1,48 +0,0 @@ -/* - * This file is part of Bisq. - * - * Bisq is free software: you can redistribute it and/or modify it - * under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or (at - * your option) any later version. - * - * Bisq is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public - * License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with Bisq. If not, see . - */ - -package bisq.core.dao.node.explorer; - -import lombok.Getter; - -// Need to be in sync with TxOutputType -enum JsonTxType { - UNDEFINED("Undefined"), - UNDEFINED_TX_TYPE("Undefined tx type"), - UNVERIFIED("Unverified"), - INVALID("Invalid"), - GENESIS("Genesis"), - TRANSFER_BSQ("Transfer BSQ"), - PAY_TRADE_FEE("Pay trade fee"), - PROPOSAL("Proposal"), - COMPENSATION_REQUEST("Compensation request"), - REIMBURSEMENT_REQUEST("Reimbursement request"), - BLIND_VOTE("Blind vote"), - VOTE_REVEAL("Vote reveal"), - LOCKUP("Lockup"), - UNLOCK("Unlock"), - ASSET_LISTING_FEE("Asset listing fee"), - PROOF_OF_BURN("Proof of burn"), - IRREGULAR("Irregular"); - - @Getter - private String displayString; - - JsonTxType(String displayString) { - this.displayString = displayString; - } -} diff --git a/core/src/main/java/bisq/core/dao/node/explorer/package-info.java b/core/src/main/java/bisq/core/dao/node/explorer/package-info.java deleted file mode 100644 index 30fc3a6f49c..00000000000 --- a/core/src/main/java/bisq/core/dao/node/explorer/package-info.java +++ /dev/null @@ -1,22 +0,0 @@ -/* - * This file is part of Bisq. - * - * Bisq is free software: you can redistribute it and/or modify it - * under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or (at - * your option) any later version. - * - * Bisq is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public - * License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with Bisq. If not, see . - */ - -/** - * Contains classes which are only used for providing data to the BSQ explorer. - */ - -package bisq.core.dao.node.explorer; diff --git a/core/src/main/java/bisq/core/dao/state/DaoStateService.java b/core/src/main/java/bisq/core/dao/state/DaoStateService.java index 620415a12ca..3187d54c2b5 100644 --- a/core/src/main/java/bisq/core/dao/state/DaoStateService.java +++ b/core/src/main/java/bisq/core/dao/state/DaoStateService.java @@ -597,6 +597,10 @@ public void addIssuance(Issuance issuance) { daoState.getIssuanceMap().put(issuance.getTxId(), issuance); } + public TreeMap getIssuanceMap() { + return daoState.getIssuanceMap(); + } + public Set getIssuanceSet(IssuanceType issuanceType) { return daoState.getIssuanceMap().values().stream() .filter(issuance -> issuance.getIssuanceType() == issuanceType) @@ -896,6 +900,10 @@ public boolean isConfiscatedUnlockTxOutput(String unlockTxId) { orElse(false); } + public List getConfiscatedLockupTxList() { + return daoState.getConfiscatedLockupTxList(); + } + /////////////////////////////////////////////////////////////////////////////////////////// // Param @@ -968,6 +976,10 @@ public Optional getSpentInfo(TxOutput txOutput) { return Optional.ofNullable(daoState.getSpentInfoMap().getOrDefault(txOutput.getKey(), null)); } + public TreeMap getSpentInfoMap() { + return daoState.getSpentInfoMap(); + } + /////////////////////////////////////////////////////////////////////////////////////////// // Vote result data From c7f2bcb2863e816d6647f9f29a6ea3e53acb5258 Mon Sep 17 00:00:00 2001 From: chimp1984 Date: Mon, 19 Oct 2020 17:50:25 -0500 Subject: [PATCH 3/4] Fix issues with batch processing --- .../node/explorer/ExportJsonFilesService.java | 63 +++++++++++++------ 1 file changed, 45 insertions(+), 18 deletions(-) diff --git a/core/src/main/java/bisq/core/dao/node/explorer/ExportJsonFilesService.java b/core/src/main/java/bisq/core/dao/node/explorer/ExportJsonFilesService.java index 4f5ef1b59a8..f3ca1adfb52 100644 --- a/core/src/main/java/bisq/core/dao/node/explorer/ExportJsonFilesService.java +++ b/core/src/main/java/bisq/core/dao/node/explorer/ExportJsonFilesService.java @@ -20,7 +20,9 @@ import bisq.core.dao.DaoSetupService; import bisq.core.dao.state.DaoStateService; import bisq.core.dao.state.model.blockchain.Block; +import bisq.core.util.FormattingUtils; +import bisq.common.UserThread; import bisq.common.config.Config; import bisq.common.file.JsonFileManager; import bisq.common.util.Utilities; @@ -129,31 +131,35 @@ public void onNewBlock(Block block) { return; } + if (!daoStateService.isParseBlockChainComplete()) { + // While we are syncing we ignore new blocks. We will check from missing blocks after batch processing + // to pickup potentially newly arrived blocks. + return; + } + // We do write the block on the main thread as the overhead to create a thread and risk for inconsistency is not // worth the potential performance gain. processBlock(block, false); } private void processBlock(Block block, boolean isBatchProcess) { - int lastPersistedBlock = getLastPersistedBlock(); - if (block.getHeight() <= lastPersistedBlock) { - return; + if (!isBatchProcess) { + // When batch processing we do not do that check + int lastPersistedBlock = getLastPersistedBlock(); + if (block.getHeight() <= lastPersistedBlock) { + return; + } } - long ts = System.currentTimeMillis(); blockFileManager.writeToDisc(Utilities.objectToJson(block), String.valueOf(block.getHeight())); - block.getTxs().forEach(tx -> { - String id = tx.getId(); - txFileManager.writeToDisc(Utilities.objectToJson(tx), id); - + txFileManager.writeToDisc(Utilities.objectToJson(tx), tx.getId()); tx.getTxOutputs().forEach(txOutput -> txOutputFileManager.writeToDisc(Utilities.objectToJson(txOutput), txOutput.getKey().toString())); }); - log.info("Write json data for block {} took {} ms", block.getHeight(), System.currentTimeMillis() - ts); - if (!isBatchProcess) { + log.info("Write json data for block {} took {} ms", block.getHeight(), System.currentTimeMillis() - ts); writeMutableData(); } } @@ -163,22 +169,43 @@ public void onParseBlockChainComplete() { return; } + long ts = System.currentTimeMillis(); int lastPersistedBlock = getLastPersistedBlock(); - List blocks = daoStateService.getBlocksFromBlockHeight(lastPersistedBlock + 1, Integer.MAX_VALUE); - - // We use a thread here to write all past blocks to avoid that the main thread gets blocked for too long. - new Thread(() -> { - Thread.currentThread().setName("Write all blocks to json"); - blocks.forEach(e -> processBlock(e, true)); - writeMutableData(); - }).start(); + int chainHeight = daoStateService.getChainHeight(); + if (lastPersistedBlock < chainHeight) { + int startFrom = lastPersistedBlock + 1; + List blocks = daoStateService.getBlocksFromBlockHeight(startFrom, Integer.MAX_VALUE); + // We use a thread here to write all past blocks to avoid that the main thread gets blocked for too long. + // Blocks are immutable so threading cannot cause any issue here. + new Thread(() -> { + Thread.currentThread().setName("Write-blocks-to-json"); + blocks.forEach(e -> processBlock(e, true)); + log.info("Batch processing {} blocks from block {} on took {}", + blocks.size(), + startFrom, + FormattingUtils.formatDurationAsWords(System.currentTimeMillis() - ts, + true, true)); + + UserThread.execute(() -> { + // As its mutable data we do it on the UserThread + writeMutableData(); + + // Once done we have to repeat our call as it might be that during batch processing + // we have received new blocks. As we request daoStateService data we also do it on the UserThread. + onParseBlockChainComplete(); + }); + }).start(); + } } private void writeMutableData() { + long ts = System.currentTimeMillis(); + log.error("write writeMutableData {}", daoStateService.getChainHeight()); spentInfoMapFileManager.writeToDisc(Utilities.objectToJson(daoStateService.getSpentInfoMap()), "spentInfoMap"); unspentTxOutputMapFileManager.writeToDisc(Utilities.objectToJson(daoStateService.getUnspentTxOutputMap()), "unspentTxOutputMap"); issuanceMapFileManager.writeToDisc(Utilities.objectToJson(daoStateService.getIssuanceMap()), "issuanceMap"); confiscatedLockupTxListFileManager.writeToDisc(Utilities.objectToJson(daoStateService.getConfiscatedLockupTxList()), "confiscatedLockupTxList"); + log.info("Write mutableData took {}", System.currentTimeMillis() - ts); } private int getLastPersistedBlock() { From 82da7391343336910ad25dcc494a94ed12c35951 Mon Sep 17 00:00:00 2001 From: chimp1984 Date: Wed, 21 Oct 2020 15:37:48 -0500 Subject: [PATCH 4/4] Add missing import (seems got lost in merge) --- core/src/main/java/bisq/core/api/CoreOffersService.java | 1 + 1 file changed, 1 insertion(+) diff --git a/core/src/main/java/bisq/core/api/CoreOffersService.java b/core/src/main/java/bisq/core/api/CoreOffersService.java index 939ab82a6e8..da07677f1b2 100644 --- a/core/src/main/java/bisq/core/api/CoreOffersService.java +++ b/core/src/main/java/bisq/core/api/CoreOffersService.java @@ -47,6 +47,7 @@ import static bisq.core.locale.CurrencyUtil.isCryptoCurrency; import static bisq.core.offer.OfferPayload.Direction; import static bisq.core.offer.OfferPayload.Direction.BUY; +import static java.lang.String.format; @Slf4j class CoreOffersService {