From 53220499e7dd20a018c3657dba3ded1063c75d9d Mon Sep 17 00:00:00 2001 From: Steven Barclay Date: Sun, 21 Nov 2021 17:10:40 +0000 Subject: [PATCH 01/15] Code cleanup: remove unused PoW class & test Remove (possible draft) 'ProofOfWorkService(Test)', which is a near duplicate of the class 'HashCashService' but is currently unused. --- .../common/crypto/ProofOfWorkService.java | 156 ------------------ .../common/crypto/ProofOfWorkServiceTest.java | 81 --------- 2 files changed, 237 deletions(-) delete mode 100644 common/src/main/java/bisq/common/crypto/ProofOfWorkService.java delete mode 100644 common/src/test/java/bisq/common/crypto/ProofOfWorkServiceTest.java diff --git a/common/src/main/java/bisq/common/crypto/ProofOfWorkService.java b/common/src/main/java/bisq/common/crypto/ProofOfWorkService.java deleted file mode 100644 index c94e4e5c1ff..00000000000 --- a/common/src/main/java/bisq/common/crypto/ProofOfWorkService.java +++ /dev/null @@ -1,156 +0,0 @@ -/* - * This file is part of Bisq. - * - * Bisq is free software: you can redistribute it and/or modify it - * under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or (at - * your option) any later version. - * - * Bisq is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public - * License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with Bisq. If not, see . - */ - -package bisq.common.crypto; - -import com.google.common.primitives.Longs; - -import java.math.BigInteger; - -import java.util.Arrays; -import java.util.concurrent.CompletableFuture; -import java.util.function.BiFunction; - -import lombok.extern.slf4j.Slf4j; - -/** - * Bitcoin-like proof of work implementation. Differs from original hashcash by using BigInteger for comparing - * the hash result with the target difficulty to gain more fine grained control for difficulty adjustment. - * This class provides a convenience method getDifficultyAsBigInteger(numLeadingZeros) to get values which are - * equivalent to the hashcash difficulty values. - * - * See https://en.wikipedia.org/wiki/Hashcash" - * "Unlike hashcash, Bitcoin's difficulty target does not specify a minimum number of leading zeros in the hash. - * Instead, the hash is interpreted as a (very large) integer, and this integer must be less than the target integer." - */ -@Slf4j -public class ProofOfWorkService { - // Default validations. Custom implementations might use tolerance. - private static final BiFunction isChallengeValid = Arrays::equals; - private static final BiFunction isTargetValid = BigInteger::equals; - - public static CompletableFuture mint(byte[] payload, - byte[] challenge, - BigInteger target) { - return mint(payload, - challenge, - target, - ProofOfWorkService::testTarget); - } - - public static boolean verify(ProofOfWork proofOfWork) { - return verify(proofOfWork, - proofOfWork.getChallenge(), - proofOfWork.getTarget()); - } - - public static boolean verify(ProofOfWork proofOfWork, - byte[] controlChallenge, - BigInteger controlTarget) { - return verify(proofOfWork, - controlChallenge, - controlTarget, - ProofOfWorkService::testTarget); - } - - public static boolean verify(ProofOfWork proofOfWork, - byte[] controlChallenge, - BigInteger controlTarget, - BiFunction challengeValidation, - BiFunction targetValidation) { - return verify(proofOfWork, - controlChallenge, - controlTarget, - challengeValidation, - targetValidation, - ProofOfWorkService::testTarget); - - } - - public static BigInteger getTarget(int numLeadingZeros) { - return BigInteger.TWO.pow(255 - numLeadingZeros).subtract(BigInteger.ONE); - } - - private static boolean testTarget(byte[] result, BigInteger target) { - return getUnsignedBigInteger(result).compareTo(target) < 0; - } - - - /////////////////////////////////////////////////////////////////////////////////////////// - // Generic - /////////////////////////////////////////////////////////////////////////////////////////// - - static CompletableFuture mint(byte[] payload, - byte[] challenge, - BigInteger target, - BiFunction testTarget) { - return CompletableFuture.supplyAsync(() -> { - long ts = System.currentTimeMillis(); - byte[] result; - long counter = 0; - do { - result = toSha256Hash(payload, challenge, ++counter); - } - while (!testTarget.apply(result, target)); - return new ProofOfWork(payload, counter, challenge, target, System.currentTimeMillis() - ts); - }); - } - - static boolean verify(ProofOfWork proofOfWork, - byte[] controlChallenge, - BigInteger controlTarget, - BiFunction testTarget) { - return verify(proofOfWork, - controlChallenge, - controlTarget, - ProofOfWorkService.isChallengeValid, - ProofOfWorkService.isTargetValid, - testTarget); - } - - static boolean verify(ProofOfWork proofOfWork, - byte[] controlChallenge, - BigInteger controlTarget, - BiFunction challengeValidation, - BiFunction targetValidation, - BiFunction testTarget) { - return challengeValidation.apply(proofOfWork.getChallenge(), controlChallenge) && - targetValidation.apply(proofOfWork.getTarget(), controlTarget) && - verify(proofOfWork, testTarget); - } - - private static boolean verify(ProofOfWork proofOfWork, BiFunction testTarget) { - byte[] hash = toSha256Hash(proofOfWork.getPayload(), proofOfWork.getChallenge(), proofOfWork.getCounter()); - return testTarget.apply(hash, proofOfWork.getTarget()); - } - - - /////////////////////////////////////////////////////////////////////////////////////////// - // Utils - /////////////////////////////////////////////////////////////////////////////////////////// - - private static BigInteger getUnsignedBigInteger(byte[] result) { - return new BigInteger(1, result); - } - - private static byte[] toSha256Hash(byte[] payload, byte[] challenge, long counter) { - byte[] preImage = org.bouncycastle.util.Arrays.concatenate(payload, - challenge, - Longs.toByteArray(counter)); - return Hash.getSha256Hash(preImage); - } -} diff --git a/common/src/test/java/bisq/common/crypto/ProofOfWorkServiceTest.java b/common/src/test/java/bisq/common/crypto/ProofOfWorkServiceTest.java deleted file mode 100644 index a4bab784b99..00000000000 --- a/common/src/test/java/bisq/common/crypto/ProofOfWorkServiceTest.java +++ /dev/null @@ -1,81 +0,0 @@ -package bisq.common.crypto; - -import org.apache.commons.lang3.RandomStringUtils; - -import java.nio.charset.StandardCharsets; - -import java.math.BigInteger; - -import java.util.ArrayList; -import java.util.List; -import java.util.UUID; -import java.util.concurrent.ExecutionException; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import org.junit.Test; - -import static org.junit.Assert.assertTrue; - -public class ProofOfWorkServiceTest { - private final static Logger log = LoggerFactory.getLogger(ProofOfWorkServiceTest.class); - - // @Ignore - @Test - public void testDiffIncrease() throws ExecutionException, InterruptedException { - StringBuilder stringBuilder = new StringBuilder(); - for (int i = 0; i < 12; i++) { - run(i, stringBuilder); - } - log.info(stringBuilder.toString()); - - //Test result on a 4 GHz Intel Core i7: - //Minting 1000 tokens with 0 leading zeros took 0.279 ms per token and 2 iterations in average. Verification took 0.025 ms per token. - //Minting 1000 tokens with 1 leading zeros took 0.063 ms per token and 4 iterations in average. Verification took 0.007 ms per token. - //Minting 1000 tokens with 2 leading zeros took 0.074 ms per token and 8 iterations in average. Verification took 0.004 ms per token. - //Minting 1000 tokens with 3 leading zeros took 0.117 ms per token and 16 iterations in average. Verification took 0.003 ms per token. - //Minting 1000 tokens with 4 leading zeros took 0.116 ms per token and 33 iterations in average. Verification took 0.003 ms per token. - //Minting 1000 tokens with 5 leading zeros took 0.204 ms per token and 65 iterations in average. Verification took 0.003 ms per token. - //Minting 1000 tokens with 6 leading zeros took 0.23 ms per token and 131 iterations in average. Verification took 0.002 ms per token. - //Minting 1000 tokens with 7 leading zeros took 0.445 ms per token and 270 iterations in average. Verification took 0.002 ms per token. - //Minting 1000 tokens with 8 leading zeros took 0.856 ms per token and 530 iterations in average. Verification took 0.002 ms per token. - //Minting 1000 tokens with 9 leading zeros took 1.629 ms per token and 988 iterations in average. Verification took 0.002 ms per token. - //Minting 1000 tokens with 10 leading zeros took 3.291 ms per token and 2103 iterations in average. Verification took 0.002 ms per token. - //Minting 1000 tokens with 11 leading zeros took 6.259 ms per token and 4009 iterations in average. Verification took 0.001 ms per token. - //Minting 1000 tokens with 12 leading zeros took 13.845 ms per token and 8254 iterations in average. Verification took 0.002 ms per token. - //Minting 1000 tokens with 13 leading zeros took 26.052 ms per token and 16645 iterations in average. Verification took 0.002 ms per token. - - //Minting 100 tokens with 14 leading zeros took 69.14 ms per token and 40917 iterations in average. Verification took 0.06 ms per token. - //Minting 100 tokens with 15 leading zeros took 102.14 ms per token and 65735 iterations in average. Verification took 0.01 ms per token. - //Minting 100 tokens with 16 leading zeros took 209.44 ms per token and 135137 iterations in average. Verification took 0.01 ms per token. - //Minting 100 tokens with 17 leading zeros took 409.46 ms per token and 263751 iterations in average. Verification took 0.01 ms per token. - //Minting 100 tokens with 18 leading zeros took 864.21 ms per token and 555671 iterations in average. Verification took 0.0 ms per token. - //Minting 100 tokens with 19 leading zeros took 1851.33 ms per token and 1097760 iterations in average. Verification took 0.0 ms per token. - } - - private void run(int numLeadingZeros, StringBuilder stringBuilder) throws ExecutionException, InterruptedException { - int numTokens = 1000; - BigInteger target = ProofOfWorkService.getTarget(numLeadingZeros); - byte[] payload = RandomStringUtils.random(50, true, true).getBytes(StandardCharsets.UTF_8); - long ts = System.currentTimeMillis(); - List tokens = new ArrayList<>(); - for (int i = 0; i < numTokens; i++) { - byte[] challenge = UUID.randomUUID().toString().getBytes(StandardCharsets.UTF_8); - tokens.add(ProofOfWorkService.mint(payload, challenge, target).get()); - } - double size = tokens.size(); - long ts2 = System.currentTimeMillis(); - long averageCounter = Math.round(tokens.stream().mapToLong(ProofOfWork::getCounter).average().orElse(0)); - boolean allValid = tokens.stream().allMatch(ProofOfWorkService::verify); - assertTrue(allValid); - double time1 = (System.currentTimeMillis() - ts) / size; - double time2 = (System.currentTimeMillis() - ts2) / size; - stringBuilder.append("\nMinting ").append(numTokens) - .append(" tokens with ").append(numLeadingZeros) - .append(" leading zeros took ").append(time1) - .append(" ms per token and ").append(averageCounter) - .append(" iterations in average. Verification took ").append(time2) - .append(" ms per token."); - } -} From 5da8df266ad170593f7ccd0d8bd22f0c8c1079a3 Mon Sep 17 00:00:00 2001 From: Steven Barclay Date: Sun, 21 Nov 2021 17:46:15 +0000 Subject: [PATCH 02/15] Code cleanup: replace (Bi)Function<..,Boolean> with (Bi)Predicate<..> Replace 'BiFunction' with the primitive specialisation 'BiPredicate' in HashCashService & FilterManager. As part of this, replace similar predicate constructs found elsewhere. NOTE: This touches the DAO packages (trivially @ VoteResultService). --- .../bisq/common/crypto/HashCashService.java | 30 +++++++++---------- .../bisq/common/util/PermutationUtil.java | 12 ++++---- .../bisq/common/util/PermutationTest.java | 9 +++--- .../voteresult/VoteResultService.java | 4 +-- .../java/bisq/core/filter/FilterManager.java | 8 ++--- .../bisq/core/network/CoreNetworkFilter.java | 10 +++---- .../bisq/core/payment/PaymentAccounts.java | 8 ++--- .../trade/bisq_v1/FailedTradesManager.java | 6 ++-- .../network/p2p/network/NetworkFilter.java | 4 +-- 9 files changed, 45 insertions(+), 46 deletions(-) diff --git a/common/src/main/java/bisq/common/crypto/HashCashService.java b/common/src/main/java/bisq/common/crypto/HashCashService.java index 80e6f929594..eabba30f8d6 100644 --- a/common/src/main/java/bisq/common/crypto/HashCashService.java +++ b/common/src/main/java/bisq/common/crypto/HashCashService.java @@ -24,7 +24,7 @@ import java.util.Arrays; import java.util.concurrent.CompletableFuture; -import java.util.function.BiFunction; +import java.util.function.BiPredicate; import lombok.extern.slf4j.Slf4j; @@ -37,8 +37,8 @@ @Slf4j public class HashCashService { // Default validations. Custom implementations might use tolerance. - private static final BiFunction isChallengeValid = Arrays::equals; - private static final BiFunction isDifficultyValid = Integer::equals; + private static final BiPredicate isChallengeValid = Arrays::equals; + private static final BiPredicate isDifficultyValid = Integer::equals; public static CompletableFuture mint(byte[] payload, byte[] challenge, @@ -67,8 +67,8 @@ public static boolean verify(ProofOfWork proofOfWork, public static boolean verify(ProofOfWork proofOfWork, byte[] controlChallenge, int controlDifficulty, - BiFunction challengeValidation, - BiFunction difficultyValidation) { + BiPredicate challengeValidation, + BiPredicate difficultyValidation) { return HashCashService.verify(proofOfWork, controlChallenge, controlDifficulty, @@ -89,7 +89,7 @@ private static boolean testDifficulty(byte[] result, long difficulty) { static CompletableFuture mint(byte[] payload, byte[] challenge, int difficulty, - BiFunction testDifficulty) { + BiPredicate testDifficulty) { return CompletableFuture.supplyAsync(() -> { long ts = System.currentTimeMillis(); byte[] result; @@ -97,7 +97,7 @@ static CompletableFuture mint(byte[] payload, do { result = toSha256Hash(payload, challenge, ++counter); } - while (!testDifficulty.apply(result, difficulty)); + while (!testDifficulty.test(result, difficulty)); ProofOfWork proofOfWork = new ProofOfWork(payload, counter, challenge, difficulty, System.currentTimeMillis() - ts); log.info("Completed minting proofOfWork: {}", proofOfWork); return proofOfWork; @@ -107,7 +107,7 @@ static CompletableFuture mint(byte[] payload, static boolean verify(ProofOfWork proofOfWork, byte[] controlChallenge, int controlDifficulty, - BiFunction testDifficulty) { + BiPredicate testDifficulty) { return verify(proofOfWork, controlChallenge, controlDifficulty, @@ -119,19 +119,19 @@ static boolean verify(ProofOfWork proofOfWork, static boolean verify(ProofOfWork proofOfWork, byte[] controlChallenge, int controlDifficulty, - BiFunction challengeValidation, - BiFunction difficultyValidation, - BiFunction testDifficulty) { - return challengeValidation.apply(proofOfWork.getChallenge(), controlChallenge) && - difficultyValidation.apply(proofOfWork.getNumLeadingZeros(), controlDifficulty) && + BiPredicate challengeValidation, + BiPredicate difficultyValidation, + BiPredicate testDifficulty) { + return challengeValidation.test(proofOfWork.getChallenge(), controlChallenge) && + difficultyValidation.test(proofOfWork.getNumLeadingZeros(), controlDifficulty) && verify(proofOfWork, testDifficulty); } - private static boolean verify(ProofOfWork proofOfWork, BiFunction testDifficulty) { + private static boolean verify(ProofOfWork proofOfWork, BiPredicate testDifficulty) { byte[] hash = HashCashService.toSha256Hash(proofOfWork.getPayload(), proofOfWork.getChallenge(), proofOfWork.getCounter()); - return testDifficulty.apply(hash, proofOfWork.getNumLeadingZeros()); + return testDifficulty.test(hash, proofOfWork.getNumLeadingZeros()); } diff --git a/common/src/main/java/bisq/common/util/PermutationUtil.java b/common/src/main/java/bisq/common/util/PermutationUtil.java index a7c3e980231..0fb2208b5bd 100644 --- a/common/src/main/java/bisq/common/util/PermutationUtil.java +++ b/common/src/main/java/bisq/common/util/PermutationUtil.java @@ -22,7 +22,7 @@ import java.util.HashSet; import java.util.List; import java.util.concurrent.atomic.AtomicInteger; -import java.util.function.BiFunction; +import java.util.function.BiPredicate; import lombok.NonNull; import lombok.extern.slf4j.Slf4j; @@ -58,9 +58,9 @@ public static List getPartialList(List list, List indicesToRe public static List findMatchingPermutation(R targetValue, List list, - BiFunction, Boolean> predicate, + BiPredicate> predicate, int maxIterations) { - if (predicate.apply(targetValue, list)) { + if (predicate.test(targetValue, list)) { return list; } else { return findMatchingPermutation(targetValue, @@ -74,7 +74,7 @@ public static List findMatchingPermutation(R targetValue, private static List findMatchingPermutation(R targetValue, List list, List> lists, - BiFunction, Boolean> predicate, + BiPredicate> predicate, AtomicInteger maxIterations) { for (int level = 0; level < list.size(); level++) { // Test one level at a time @@ -90,7 +90,7 @@ private static List findMatchingPermutation(R targetValue, @NonNull private static List checkLevel(R targetValue, List previousLevel, - BiFunction, Boolean> predicate, + BiPredicate> predicate, int level, int permutationIndex, AtomicInteger maxIterations) { @@ -106,7 +106,7 @@ private static List checkLevel(R targetValue, if (level == 0) { maxIterations.decrementAndGet(); // Check all permutations on this level - if (predicate.apply(targetValue, newList)) { + if (predicate.test(targetValue, newList)) { return newList; } } else { diff --git a/common/src/test/java/bisq/common/util/PermutationTest.java b/common/src/test/java/bisq/common/util/PermutationTest.java index dc3c65c5130..c539992d2a0 100644 --- a/common/src/test/java/bisq/common/util/PermutationTest.java +++ b/common/src/test/java/bisq/common/util/PermutationTest.java @@ -21,8 +21,7 @@ import java.util.Arrays; import java.util.Collections; import java.util.List; -import java.util.function.BiFunction; - +import java.util.function.BiPredicate; import org.junit.Test; @@ -108,7 +107,7 @@ public void testFindMatchingPermutation() { List result; List list; List expected; - BiFunction, Boolean> predicate = (target, variationList) -> variationList.toString().equals(target); + BiPredicate> predicate = (target, variationList) -> variationList.toString().equals(target); list = Arrays.asList(a, b, c, d, e); @@ -124,11 +123,11 @@ public void testFindMatchingPermutation() { @Test public void testBreakAtLimit() { - BiFunction, Boolean> predicate = + BiPredicate> predicate = (target, variationList) -> variationList.toString().equals(target); var list = Arrays.asList("a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k", "l", "m", "n", "o"); var expected = Arrays.asList("b", "g", "m"); - + // Takes around 32508 tries starting from longer strings var limit = 100000; var result = PermutationUtil.findMatchingPermutation(expected.toString(), list, predicate, limit); diff --git a/core/src/main/java/bisq/core/dao/governance/voteresult/VoteResultService.java b/core/src/main/java/bisq/core/dao/governance/voteresult/VoteResultService.java index ddf402d323d..0f85803712d 100644 --- a/core/src/main/java/bisq/core/dao/governance/voteresult/VoteResultService.java +++ b/core/src/main/java/bisq/core/dao/governance/voteresult/VoteResultService.java @@ -77,7 +77,7 @@ import java.util.Objects; import java.util.Optional; import java.util.Set; -import java.util.function.BiFunction; +import java.util.function.BiPredicate; import java.util.function.Function; import java.util.stream.Collectors; @@ -488,7 +488,7 @@ private Optional> findPermutatedListMatchingMajority(byte[] majo List list = BlindVoteConsensus.getSortedBlindVoteListOfCycle(blindVoteListService); long ts = System.currentTimeMillis(); - BiFunction, Boolean> predicate = (hash, variation) -> + BiPredicate> predicate = (hash, variation) -> isListMatchingMajority(hash, variation, false); List result = PermutationUtil.findMatchingPermutation(majorityVoteListHash, list, predicate, 1000000); diff --git a/core/src/main/java/bisq/core/filter/FilterManager.java b/core/src/main/java/bisq/core/filter/FilterManager.java index dfae9cf6858..3ecc11a7465 100644 --- a/core/src/main/java/bisq/core/filter/FilterManager.java +++ b/core/src/main/java/bisq/core/filter/FilterManager.java @@ -65,7 +65,7 @@ import java.util.List; import java.util.Set; import java.util.concurrent.CopyOnWriteArrayList; -import java.util.function.BiFunction; +import java.util.function.BiPredicate; import java.util.function.Consumer; import java.lang.reflect.Method; @@ -87,9 +87,9 @@ public class FilterManager { private static final String BANNED_SEED_NODES = "bannedSeedNodes"; private static final String BANNED_BTC_NODES = "bannedBtcNodes"; - private final BiFunction challengeValidation = Arrays::equals; + private final BiPredicate challengeValidation = Arrays::equals; // We only require a new pow if difficulty has increased - private final BiFunction difficultyValidation = + private final BiPredicate difficultyValidation = (value, controlValue) -> value - controlValue >= 0; @@ -144,7 +144,7 @@ public FilterManager(P2PService p2PService, "029340c3e7d4bb0f9e651b5f590b434fecb6175aeaa57145c7804ff05d210e534f", "034dc7530bf66ffd9580aa98031ea9a18ac2d269f7c56c0e71eca06105b9ed69f9"); - networkFilter.setBannedNodeFunction(this::isNodeAddressBannedFromNetwork); + networkFilter.setBannedNodePredicate(this::isNodeAddressBannedFromNetwork); } diff --git a/core/src/main/java/bisq/core/network/CoreNetworkFilter.java b/core/src/main/java/bisq/core/network/CoreNetworkFilter.java index b261d421537..536ea82287a 100644 --- a/core/src/main/java/bisq/core/network/CoreNetworkFilter.java +++ b/core/src/main/java/bisq/core/network/CoreNetworkFilter.java @@ -28,14 +28,14 @@ import java.util.HashSet; import java.util.List; import java.util.Set; -import java.util.function.Function; +import java.util.function.Predicate; import lombok.extern.slf4j.Slf4j; @Slf4j public class CoreNetworkFilter implements NetworkFilter { private final Set bannedPeersFromOptions = new HashSet<>(); - private Function bannedNodeFunction; + private Predicate bannedNodePredicate; /** * @param banList List of banned peers from program argument @@ -46,13 +46,13 @@ public CoreNetworkFilter(@Named(Config.BAN_LIST) List banList) { } @Override - public void setBannedNodeFunction(Function bannedNodeFunction) { - this.bannedNodeFunction = bannedNodeFunction; + public void setBannedNodePredicate(Predicate bannedNodePredicate) { + this.bannedNodePredicate = bannedNodePredicate; } @Override public boolean isPeerBanned(NodeAddress nodeAddress) { return bannedPeersFromOptions.contains(nodeAddress) || - bannedNodeFunction != null && bannedNodeFunction.apply(nodeAddress); + bannedNodePredicate != null && bannedNodePredicate.test(nodeAddress); } } diff --git a/core/src/main/java/bisq/core/payment/PaymentAccounts.java b/core/src/main/java/bisq/core/payment/PaymentAccounts.java index 6f871ce20f9..578dfee7b4b 100644 --- a/core/src/main/java/bisq/core/payment/PaymentAccounts.java +++ b/core/src/main/java/bisq/core/payment/PaymentAccounts.java @@ -25,7 +25,7 @@ import java.util.Date; import java.util.List; import java.util.Set; -import java.util.function.BiFunction; +import java.util.function.BiPredicate; import java.util.stream.Collectors; import org.slf4j.Logger; @@ -38,14 +38,14 @@ class PaymentAccounts { private final Set accounts; private final AccountAgeWitnessService accountAgeWitnessService; - private final BiFunction validator; + private final BiPredicate validator; PaymentAccounts(Set accounts, AccountAgeWitnessService accountAgeWitnessService) { this(accounts, accountAgeWitnessService, PaymentAccountUtil::isPaymentAccountValidForOffer); } PaymentAccounts(Set accounts, AccountAgeWitnessService accountAgeWitnessService, - BiFunction validator) { + BiPredicate validator) { this.accounts = accounts; this.accountAgeWitnessService = accountAgeWitnessService; this.validator = validator; @@ -63,7 +63,7 @@ PaymentAccount getOldestPaymentAccountForOffer(Offer offer) { private List sortValidAccounts(Offer offer) { Comparator comparator = this::compareByTradeLimit; return accounts.stream() - .filter(account -> validator.apply(offer, account)) + .filter(account -> validator.test(offer, account)) .sorted(comparator.reversed()) .collect(Collectors.toList()); } diff --git a/core/src/main/java/bisq/core/trade/bisq_v1/FailedTradesManager.java b/core/src/main/java/bisq/core/trade/bisq_v1/FailedTradesManager.java index 295342511a6..412bacb03df 100644 --- a/core/src/main/java/bisq/core/trade/bisq_v1/FailedTradesManager.java +++ b/core/src/main/java/bisq/core/trade/bisq_v1/FailedTradesManager.java @@ -33,7 +33,7 @@ import javafx.collections.ObservableList; import java.util.Optional; -import java.util.function.Function; +import java.util.function.Predicate; import java.util.stream.Stream; import org.slf4j.Logger; @@ -52,7 +52,7 @@ public class FailedTradesManager implements PersistedDataHost { private final TradeUtil tradeUtil; private final DumpDelayedPayoutTx dumpDelayedPayoutTx; @Setter - private Function unFailTradeCallback; + private Predicate unFailTradeCallback; @Inject public FailedTradesManager(KeyRing keyRing, @@ -123,7 +123,7 @@ public void unFailTrade(Trade trade) { if (unFailTradeCallback == null) return; - if (unFailTradeCallback.apply(trade)) { + if (unFailTradeCallback.test(trade)) { log.info("Unfailing trade {}", trade.getId()); if (failedTrades.remove(trade)) { requestPersistence(); diff --git a/p2p/src/main/java/bisq/network/p2p/network/NetworkFilter.java b/p2p/src/main/java/bisq/network/p2p/network/NetworkFilter.java index 3dcf040e21f..15940c35326 100644 --- a/p2p/src/main/java/bisq/network/p2p/network/NetworkFilter.java +++ b/p2p/src/main/java/bisq/network/p2p/network/NetworkFilter.java @@ -19,10 +19,10 @@ import bisq.network.p2p.NodeAddress; -import java.util.function.Function; +import java.util.function.Predicate; public interface NetworkFilter { boolean isPeerBanned(NodeAddress nodeAddress); - void setBannedNodeFunction(Function isNodeAddressBanned); + void setBannedNodePredicate(Predicate isNodeAddressBanned); } From c2b3a078ff0796a9ef1ad4ae4f1d5e54f5e3f586 Mon Sep 17 00:00:00 2001 From: Steven Barclay Date: Tue, 23 Nov 2021 06:27:33 +0000 Subject: [PATCH 03/15] Add Equihash implementation for use in ASIC-resistant PoW Implement the Equihash (https://eprint.iacr.org/2015/946.pdf) algorithm for solving/verifying memory-hard client-puzzles/proof-of-work problems for ASIC-resistant DoS attack protection. The scheme is asymmetric, so that even though solving a puzzle is slow and memory-intensive, needing 100's of kB to MB's of memory, the solution verification is instant. Instead of a single 64-bit counter/nonce, as in the case of Hashcash, Equihash solutions are larger objects ranging from 10's of bytes to a few kB, depending on the puzzle parameters used. These need to be stored in entirety, in the proof-of-work field of each offer payload. Include logic for fine-grained difficulty control in Equihash with a double-precision floating point number. This is based on lexicographic comparison with a target hash, like in Bitcoin, instead of just counting the number of leading zeros of a hash. The code is unused at present. Also add some simple unit tests. --- .../java/bisq/common/crypto/Equihash.java | 349 ++++++++++++++++++ .../main/java/bisq/common/util/Utilities.java | 21 ++ .../java/bisq/common/crypto/EquihashTest.java | 71 ++++ 3 files changed, 441 insertions(+) create mode 100644 common/src/main/java/bisq/common/crypto/Equihash.java create mode 100644 common/src/test/java/bisq/common/crypto/EquihashTest.java diff --git a/common/src/main/java/bisq/common/crypto/Equihash.java b/common/src/main/java/bisq/common/crypto/Equihash.java new file mode 100644 index 00000000000..87f0b736341 --- /dev/null +++ b/common/src/main/java/bisq/common/crypto/Equihash.java @@ -0,0 +1,349 @@ +/* + * This file is part of Bisq. + * + * Bisq is free software: you can redistribute it and/or modify it + * under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or (at + * your option) any later version. + * + * Bisq is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public + * License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with Bisq. If not, see . + */ + +package bisq.common.crypto; + +import bisq.common.util.Utilities; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.collect.ListMultimap; +import com.google.common.collect.MultimapBuilder; +import com.google.common.primitives.Bytes; +import com.google.common.primitives.ImmutableIntArray; +import com.google.common.primitives.Ints; +import com.google.common.primitives.Longs; +import com.google.common.primitives.UnsignedInts; + +import org.bouncycastle.crypto.digests.Blake2bDigest; + +import java.nio.ByteBuffer; +import java.nio.IntBuffer; + +import java.math.BigInteger; + +import java.util.ArrayDeque; +import java.util.Arrays; +import java.util.Deque; +import java.util.Optional; + +import lombok.ToString; + +import static com.google.common.base.Preconditions.checkArgument; +import static java.math.BigInteger.ONE; + +/** + * An ASIC-resistant Proof-of-Work scheme based on the Generalized Birthday Problem (GBP), + * as described in this paper and used + * in ZCash and some other cryptocurrencies. Like + * Hashcash but unlike many other + * memory-hard and ASIC resistant PoW schemes, it is asymmetric, meaning that it + * supports fast verification. This makes it suitable for DoS attack protection.

+ *
+ * The Generalized Birthday Problem is an attempt to find 2k + * n-bit hashes (out of a list of length N) which XOR to zero. When N + * equals 21+n/(k+1), this has at least a handful of solutions on + * average, which can be found using Wagner's Algorithm, as described in the paper and + * implemented here. The rough idea is to split each hash into k+1 + * n/(k+1)-bit blocks and place them into a table to look for collisions on a given + * block. All (partially) colliding pairs are XORed together and used to build a new table + * of roughly the same size as the last, before moving on to the next block and repeating + * the process until a full collision can be found. Keeping track of the tuple of hashes + * used to form each intermediate XOR result (which doubles in length each iteration) + * gives the final solution. The table-based approach needed to find solutions to the GBP + * makes it a memory-hard problem.

+ *
+ * In this implementation and the reference + * C++ implementation included with + * the paper, the hash function BLAKE2b is used to supply 256 bits, which is shortened and + * split into k+1 32-bit blocks. The blocks are masked to provide n/(k+1) + * bits each and n bits in total. This allows working with 32-bit integers + * throughout, for efficiency. + */ +@SuppressWarnings("UnstableApiUsage") +public class Equihash { + private static final int HASH_BIT_LENGTH = 256; + + private final int k, N; + private final int tableCapacity; + private final int inputNum, inputBits; + private final int[] hashUpperBound; + + public Equihash(int n, int k, double difficulty) { + checkArgument(k > 0 && k < HASH_BIT_LENGTH / 32, + "Tree depth k must be a positive integer less than %s.", + HASH_BIT_LENGTH / 32); + checkArgument(n > 0 && n < HASH_BIT_LENGTH && n % (k + 1) == 0, + "Collision bit count n must be a positive multiple of k + 1 and less than %s.", + HASH_BIT_LENGTH); + checkArgument(n / (k + 1) < 30, + "Sub-collision bit count n / (k + 1) must be less than 30, got %s.", + n / (k + 1)); + this.k = k; + inputNum = 1 << k; + inputBits = n / (k + 1) + 1; + N = 1 << inputBits; + tableCapacity = (int) (N * 1.1); + hashUpperBound = hashUpperBound(difficulty); + } + + @VisibleForTesting + static int[] hashUpperBound(double difficulty) { + return Utilities.bytesToIntsBE(Utilities.copyRightAligned( + inverseDifficultyMinusOne(difficulty).toByteArray(), HASH_BIT_LENGTH / 8 + )); + } + + private static BigInteger inverseDifficultyMinusOne(double difficulty) { + checkArgument(difficulty >= 1.0, "Difficulty must be at least 1."); + int exponent = Math.getExponent(difficulty) - 52; + var mantissa = BigInteger.valueOf((long) Math.scalb(difficulty, -exponent)); + var inverse = ONE.shiftLeft(HASH_BIT_LENGTH - exponent).add(mantissa).subtract(ONE).divide(mantissa); + return inverse.subtract(ONE).max(BigInteger.ZERO); + } + + public Puzzle puzzle(byte[] seed) { + return new Puzzle(seed); + } + + public class Puzzle { + private final byte[] seed; + + private Puzzle(byte[] seed) { + this.seed = seed; + } + + @ToString + public class Solution { + private final long nonce; + private final int[] inputs; + + private Solution(long nonce, int... inputs) { + this.nonce = nonce; + this.inputs = inputs; + } + + public boolean verify() { + return withHashPrefix(seed, nonce).verify(inputs); + } + + public byte[] serialize() { + int bitLen = 64 + inputNum * inputBits; + int byteLen = (bitLen + 7) / 8; + + byte[] paddedBytes = new byte[byteLen + 3 & -4]; + IntBuffer intBuffer = ByteBuffer.wrap(paddedBytes).asIntBuffer(); + intBuffer.put((int) (nonce >> 32)).put((int) nonce); + int off = 64; + long buf = 0; + + for (int v : inputs) { + off -= inputBits; + buf |= UnsignedInts.toLong(v) << off; + if (off <= 32) { + intBuffer.put((int) (buf >> 32)); + buf <<= 32; + off += 32; + } + } + if (off < 64) { + intBuffer.put((int) (buf >> 32)); + } + return (byteLen & 3) == 0 ? paddedBytes : Arrays.copyOf(paddedBytes, byteLen); + } + } + + public Solution deserializeSolution(byte[] bytes) { + int bitLen = 64 + inputNum * inputBits; + int byteLen = (bitLen + 7) / 8; + checkArgument(bytes.length == byteLen, + "Incorrect solution byte length. Expected %s but got %s.", + byteLen, bytes.length); + checkArgument(byteLen == 0 || (byte) (bytes[byteLen - 1] << ((bitLen + 7 & 7) + 1)) == 0, + "Nonzero padding bits found at end of solution byte array."); + + byte[] paddedBytes = (byteLen & 3) == 0 ? bytes : Arrays.copyOf(bytes, byteLen + 3 & -4); + IntBuffer intBuffer = ByteBuffer.wrap(paddedBytes).asIntBuffer(); + long nonce = ((long) intBuffer.get() << 32) | UnsignedInts.toLong(intBuffer.get()); + int[] inputs = new int[inputNum]; + int off = 0; + long buf = 0; + + for (int i = 0; i < inputs.length; i++) { + if (off < inputBits) { + buf = buf << 32 | UnsignedInts.toLong(intBuffer.get()); + off += 32; + } + off -= inputBits; + inputs[i] = (int) (buf >>> off) & (N - 1); + } + return new Solution(nonce, inputs); + } + + public Solution findSolution() { + Optional inputs; + for (int nonce = 0; ; nonce++) { + if ((inputs = withHashPrefix(seed, nonce).findInputs()).isPresent()) { + return new Solution(nonce, inputs.get()); + } + } + } + } + + private WithHashPrefix withHashPrefix(byte[] seed, long nonce) { + return new WithHashPrefix(Bytes.concat(seed, Longs.toByteArray(nonce))); + } + + private class WithHashPrefix { + private final byte[] prefixBytes; + + private WithHashPrefix(byte[] prefixBytes) { + this.prefixBytes = prefixBytes; + } + + private int[] hashInputs(int... inputs) { + var digest = new Blake2bDigest(HASH_BIT_LENGTH); + digest.update(prefixBytes, 0, prefixBytes.length); + byte[] inputBytes = Utilities.intsToBytesBE(inputs); + digest.update(inputBytes, 0, inputBytes.length); + byte[] outputBytes = new byte[HASH_BIT_LENGTH / 8]; + digest.doFinal(outputBytes, 0); + return Utilities.bytesToIntsBE(outputBytes); + } + + Optional findInputs() { + var table = computeAllHashes(); + for (int i = 0; i < k; i++) { + table = findCollisions(table, i + 1 < k); + } + for (int i = 0; i < table.numRows; i++) { + if (table.getRow(i).stream().distinct().count() == inputNum) { + int[] inputs = sortInputs(table.getRow(i).toArray()); + if (testDifficultyCondition(inputs)) { + return Optional.of(inputs); + } + } + } + return Optional.empty(); + } + + private XorTable computeAllHashes() { + var tableValues = ImmutableIntArray.builder((k + 2) * N); + for (int i = 0; i < N; i++) { + int[] hash = hashInputs(i); + for (int j = 0; j <= k; j++) { + tableValues.add(hash[j] & (N / 2 - 1)); + } + tableValues.add(i); + } + return new XorTable(k + 1, 1, tableValues.build()); + } + + private boolean testDifficultyCondition(int[] inputs) { + int[] difficultyHash = hashInputs(inputs); + return UnsignedInts.lexicographicalComparator().compare(difficultyHash, hashUpperBound) <= 0; + } + + boolean verify(int[] inputs) { + if (inputs.length != inputNum || Arrays.stream(inputs).distinct().count() < inputNum) { + return false; + } + if (Arrays.stream(inputs).anyMatch(i -> i < 0 || i >= N)) { + return false; + } + if (!Arrays.equals(inputs, sortInputs(inputs))) { + return false; + } + if (!testDifficultyCondition(inputs)) { + return false; + } + int[] hashBlockSums = new int[k + 1]; + for (int i = 0; i < inputs.length; i++) { + int[] hash = hashInputs(inputs[i]); + for (int j = 0; j <= k; j++) { + hashBlockSums[j] ^= hash[j] & (N / 2 - 1); + } + for (int ii = i + 1 + inputNum, j = 0; (ii & 1) == 0; ii /= 2, j++) { + if (hashBlockSums[j] != 0) { + return false; + } + } + } + return true; + } + } + + private static class XorTable { + private final int hashWidth, indexTupleWidth, rowWidth, numRows; + private final ImmutableIntArray values; + + XorTable(int hashWidth, int indexTupleWidth, ImmutableIntArray values) { + this.hashWidth = hashWidth; + this.indexTupleWidth = indexTupleWidth; + this.values = values; + rowWidth = hashWidth + indexTupleWidth; + numRows = (values.length() + rowWidth - 1) / rowWidth; + } + + ImmutableIntArray getRow(int index) { + return values.subArray(index * rowWidth, index * rowWidth + hashWidth + indexTupleWidth); + } + } + + // Apply a single iteration of Wagner's Algorithm. + private XorTable findCollisions(XorTable table, boolean isPartial) { + int newHashWidth = isPartial ? table.hashWidth - 1 : 0; + int newIndexTupleWidth = table.indexTupleWidth * 2; + int newRowWidth = newHashWidth + newIndexTupleWidth; + var newTableValues = ImmutableIntArray.builder( + newRowWidth * (isPartial ? tableCapacity : 10)); + + ListMultimap indexMultimap = MultimapBuilder.hashKeys().arrayListValues().build(); + for (int i = 0; i < table.numRows; i++) { + var row = table.getRow(i); + var collisionIndices = indexMultimap.get(row.get(0)); + collisionIndices.forEach(ii -> { + var collidingRow = table.getRow(ii); + if (isPartial) { + for (int j = 1; j < table.hashWidth; j++) { + newTableValues.add(collidingRow.get(j) ^ row.get(j)); + } + } else if (!collidingRow.subArray(1, table.hashWidth).equals(row.subArray(1, table.hashWidth))) { + return; + } + newTableValues.addAll(collidingRow.subArray(table.hashWidth, collidingRow.length())); + newTableValues.addAll(row.subArray(table.hashWidth, row.length())); + }); + indexMultimap.put(row.get(0), i); + } + return new XorTable(newHashWidth, newIndexTupleWidth, newTableValues.build()); + } + + private static int[] sortInputs(int[] inputs) { + Deque sublistStack = new ArrayDeque<>(); + int[] topSublist; + for (int input : inputs) { + topSublist = new int[]{input}; + while (!sublistStack.isEmpty() && sublistStack.peek().length == topSublist.length) { + topSublist = UnsignedInts.lexicographicalComparator().compare(sublistStack.peek(), topSublist) < 0 + ? Ints.concat(sublistStack.pop(), topSublist) + : Ints.concat(topSublist, sublistStack.pop()); + } + sublistStack.push(topSublist); + } + return sublistStack.pop(); + } +} diff --git a/common/src/main/java/bisq/common/util/Utilities.java b/common/src/main/java/bisq/common/util/Utilities.java index 03b29aa35e1..7e665833c13 100644 --- a/common/src/main/java/bisq/common/util/Utilities.java +++ b/common/src/main/java/bisq/common/util/Utilities.java @@ -42,6 +42,7 @@ import java.net.URI; import java.net.URISyntaxException; +import java.nio.ByteBuffer; import java.nio.file.Paths; import java.io.File; @@ -525,6 +526,26 @@ public static int byteArrayToInteger(byte[] bytes) { return result; } + public static byte[] copyRightAligned(byte[] src, int newLength) { + byte[] dest = new byte[newLength]; + int srcPos = Math.max(src.length - newLength, 0); + int destPos = Math.max(newLength - src.length, 0); + System.arraycopy(src, srcPos, dest, destPos, newLength - destPos); + return dest; + } + + public static byte[] intsToBytesBE(int[] ints) { + byte[] bytes = new byte[ints.length * 4]; + ByteBuffer.wrap(bytes).asIntBuffer().put(ints); + return bytes; + } + + public static int[] bytesToIntsBE(byte[] bytes) { + int[] ints = new int[bytes.length / 4]; + ByteBuffer.wrap(bytes).asIntBuffer().get(ints); + return ints; + } + // Helper to filter unique elements by key public static Predicate distinctByKey(Function keyExtractor) { Map map = new ConcurrentHashMap<>(); diff --git a/common/src/test/java/bisq/common/crypto/EquihashTest.java b/common/src/test/java/bisq/common/crypto/EquihashTest.java new file mode 100644 index 00000000000..6143a54c605 --- /dev/null +++ b/common/src/test/java/bisq/common/crypto/EquihashTest.java @@ -0,0 +1,71 @@ +/* + * This file is part of Bisq. + * + * Bisq is free software: you can redistribute it and/or modify it + * under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or (at + * your option) any later version. + * + * Bisq is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public + * License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with Bisq. If not, see . + */ + +package bisq.common.crypto; + +import bisq.common.crypto.Equihash.Puzzle.Solution; + +import com.google.common.base.Strings; + +import java.util.Arrays; +import java.util.stream.Collectors; + +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +public class EquihashTest { + @Test + public void testHashUpperBound() { + assertEquals("ffffffff ffffffff ffffffff ffffffff ffffffff ffffffff ffffffff ffffffff", hub(1)); + assertEquals("aaaaaaaa aaaaaaaa aaaaaaaa aaaaaaaa aaaaaaaa aaaaaaaa aaaaaaaa aaaaaaaa", hub(1.5)); + assertEquals("7fffffff ffffffff ffffffff ffffffff ffffffff ffffffff ffffffff ffffffff", hub(2)); + assertEquals("55555555 55555555 55555555 55555555 55555555 55555555 55555555 55555555", hub(3)); + assertEquals("3fffffff ffffffff ffffffff ffffffff ffffffff ffffffff ffffffff ffffffff", hub(4)); + assertEquals("33333333 33333333 33333333 33333333 33333333 33333333 33333333 33333333", hub(5)); + assertEquals("051eb851 eb851eb8 51eb851e b851eb85 1eb851eb 851eb851 eb851eb8 51eb851e", hub(50.0)); + assertEquals("0083126e 978d4fdf 3b645a1c ac083126 e978d4fd f3b645a1 cac08312 6e978d4f", hub(500.0)); + assertEquals("00000000 00000000 2f394219 248446ba a23d2ec7 29af3d61 0607aa01 67dd94ca", hub(1.0e20)); + assertEquals("00000000 00000000 00000000 00000000 ffffffff ffffffff ffffffff ffffffff", hub(0x1.0p128)); + assertEquals("00000000 00000000 00000000 00000000 00000000 00000000 00000000 00000000", hub(Double.POSITIVE_INFINITY)); + } + + @Test + public void testFindSolution() { + Equihash equihash = new Equihash(90, 5, 5.0); + byte[] seed = new byte[64]; + Solution solution = equihash.puzzle(seed).findSolution(); + + byte[] solutionBytes = solution.serialize(); + Solution roundTrippedSolution = equihash.puzzle(seed).deserializeSolution(solutionBytes); + + assertTrue(solution.verify()); + assertEquals(72, solutionBytes.length); + assertEquals(solution.toString(), roundTrippedSolution.toString()); + } + + private static String hub(double difficulty) { + return hexString(Equihash.hashUpperBound(difficulty)); + } + + private static String hexString(int[] ints) { + return Arrays.stream(ints) + .mapToObj(n -> Strings.padStart(Integer.toHexString(n), 8, '0')) + .collect(Collectors.joining(" ")); + } +} From a5f5a557759a94339d90aa471353d6bf8dd2fd9d Mon Sep 17 00:00:00 2001 From: Steven Barclay Date: Tue, 23 Nov 2021 07:34:52 +0000 Subject: [PATCH 04/15] Add Equihash.IntListMultimap (private) class for speedup Provide a (vastly cut down) drop-in replacement for the Guava multimap instance 'indexMultimap', of type 'ListMultimap', used to map table row indices to block values, to detect collisions at a given block position (that is, in a given table column). The replacement stores (multi-)mappings from ints to ints in a flat int- array, only spilling over to a ListMultimap if there are more than 4 values added for a given key. This vastly reduces the amount of boxing and memory usage when running 'Equihash::findCollisions' to build up the next table as part of Wagner's algorithm. --- .../java/bisq/common/crypto/Equihash.java | 34 ++++++++++++++++++- 1 file changed, 33 insertions(+), 1 deletion(-) diff --git a/common/src/main/java/bisq/common/crypto/Equihash.java b/common/src/main/java/bisq/common/crypto/Equihash.java index 87f0b736341..bbd85408ddb 100644 --- a/common/src/main/java/bisq/common/crypto/Equihash.java +++ b/common/src/main/java/bisq/common/crypto/Equihash.java @@ -39,6 +39,7 @@ import java.util.Arrays; import java.util.Deque; import java.util.Optional; +import java.util.stream.IntStream; import lombok.ToString; @@ -303,6 +304,37 @@ ImmutableIntArray getRow(int index) { } } + private static class IntListMultimap { + final int[] shortLists; + final ListMultimap overspillMultimap; + + IntListMultimap(int keyUpperBound) { + shortLists = new int[keyUpperBound * 4]; + overspillMultimap = MultimapBuilder.hashKeys().arrayListValues().build(); + } + + IntStream get(int key) { + if (shortLists[key * 4 + 3] == 0) { + return IntStream.range(0, 4).map(i -> ~shortLists[key * 4 + i]).takeWhile(i -> i >= 0); + } + return IntStream.concat( + IntStream.range(0, 4).map(i -> ~shortLists[key * 4 + i]), + overspillMultimap.get(key).stream().mapToInt(i -> i) + ); + } + + // assumes non-negative values only: + void put(int key, int value) { + for (int i = 0; i < 4; i++) { + if (shortLists[key * 4 + i] == 0) { + shortLists[key * 4 + i] = ~value; + return; + } + } + overspillMultimap.put(key, value); + } + } + // Apply a single iteration of Wagner's Algorithm. private XorTable findCollisions(XorTable table, boolean isPartial) { int newHashWidth = isPartial ? table.hashWidth - 1 : 0; @@ -311,7 +343,7 @@ private XorTable findCollisions(XorTable table, boolean isPartial) { var newTableValues = ImmutableIntArray.builder( newRowWidth * (isPartial ? tableCapacity : 10)); - ListMultimap indexMultimap = MultimapBuilder.hashKeys().arrayListValues().build(); + var indexMultimap = new IntListMultimap(N / 2); for (int i = 0; i < table.numRows; i++) { var row = table.getRow(i); var collisionIndices = indexMultimap.get(row.get(0)); From 6bb0f27807f7b94d4c998c76fa441a7ff2b04a1e Mon Sep 17 00:00:00 2001 From: Steven Barclay Date: Tue, 23 Nov 2021 07:57:07 +0000 Subject: [PATCH 05/15] Further Equihash optimisation: avoid lambda+stream in tight loop Manually iterate over colliding table rows using a while- loop and a custom 'PrimitiveIterator.OfInt' implementation, instead of a foreach lambda called on an IntStream, in 'Equihash::findCollisions'. Profiling shows that this results in a slight speedup. --- .../java/bisq/common/crypto/Equihash.java | 45 +++++++++++++------ 1 file changed, 32 insertions(+), 13 deletions(-) diff --git a/common/src/main/java/bisq/common/crypto/Equihash.java b/common/src/main/java/bisq/common/crypto/Equihash.java index bbd85408ddb..d78e13315b3 100644 --- a/common/src/main/java/bisq/common/crypto/Equihash.java +++ b/common/src/main/java/bisq/common/crypto/Equihash.java @@ -38,8 +38,10 @@ import java.util.ArrayDeque; import java.util.Arrays; import java.util.Deque; +import java.util.Iterator; +import java.util.NoSuchElementException; import java.util.Optional; -import java.util.stream.IntStream; +import java.util.PrimitiveIterator; import lombok.ToString; @@ -313,14 +315,31 @@ private static class IntListMultimap { overspillMultimap = MultimapBuilder.hashKeys().arrayListValues().build(); } - IntStream get(int key) { - if (shortLists[key * 4 + 3] == 0) { - return IntStream.range(0, 4).map(i -> ~shortLists[key * 4 + i]).takeWhile(i -> i >= 0); - } - return IntStream.concat( - IntStream.range(0, 4).map(i -> ~shortLists[key * 4 + i]), - overspillMultimap.get(key).stream().mapToInt(i -> i) - ); + PrimitiveIterator.OfInt get(int key) { + return new PrimitiveIterator.OfInt() { + int i; + Iterator overspillIterator; + + private Iterator overspillIterator() { + if (overspillIterator == null) { + overspillIterator = overspillMultimap.get(i).iterator(); + } + return overspillIterator; + } + + @Override + public int nextInt() { + if (!hasNext()) { + throw new NoSuchElementException(); + } + return i < 4 ? ~shortLists[key * 4 + i++] : overspillIterator().next(); + } + + @Override + public boolean hasNext() { + return i < 4 && shortLists[key * 4 + i] < 0 || i == 4 && overspillIterator().hasNext(); + } + }; } // assumes non-negative values only: @@ -347,18 +366,18 @@ private XorTable findCollisions(XorTable table, boolean isPartial) { for (int i = 0; i < table.numRows; i++) { var row = table.getRow(i); var collisionIndices = indexMultimap.get(row.get(0)); - collisionIndices.forEach(ii -> { - var collidingRow = table.getRow(ii); + while (collisionIndices.hasNext()) { + var collidingRow = table.getRow(collisionIndices.nextInt()); if (isPartial) { for (int j = 1; j < table.hashWidth; j++) { newTableValues.add(collidingRow.get(j) ^ row.get(j)); } } else if (!collidingRow.subArray(1, table.hashWidth).equals(row.subArray(1, table.hashWidth))) { - return; + continue; } newTableValues.addAll(collidingRow.subArray(table.hashWidth, collidingRow.length())); newTableValues.addAll(row.subArray(table.hashWidth, row.length())); - }); + } indexMultimap.put(row.get(0), i); } return new XorTable(newHashWidth, newIndexTupleWidth, newTableValues.build()); From 52f4981f1514963f877ce71986c3dde21d6bafd6 Mon Sep 17 00:00:00 2001 From: Steven Barclay Date: Tue, 23 Nov 2021 08:54:55 +0000 Subject: [PATCH 06/15] Further Equihash optimisation: partial parallelisation Run the initial XorTable fillup in 'Equihash::computeAllHashes' in parallel, using a parallel stream, to get an easy speed up. (The solver spends about half its time computing BLAKE2b hashes before iteratively building tables of partial collisions using 'Equihash::findCollisions'.) As part of this, replace the use of 'java.nio.ByteBuffer' array wrapping in 'Utilities::(bytesToIntsBE|intsToBytesBE)' with manual for-loops, as profiling reveals an unexpected bottleneck in the former when used in a multithreaded setting. (Lock contention somewhere in unsafe code?) --- .../src/main/java/bisq/common/crypto/Equihash.java | 13 +++++-------- .../src/main/java/bisq/common/util/Utilities.java | 14 +++++++++++--- 2 files changed, 16 insertions(+), 11 deletions(-) diff --git a/common/src/main/java/bisq/common/crypto/Equihash.java b/common/src/main/java/bisq/common/crypto/Equihash.java index d78e13315b3..e8eec1697d6 100644 --- a/common/src/main/java/bisq/common/crypto/Equihash.java +++ b/common/src/main/java/bisq/common/crypto/Equihash.java @@ -42,6 +42,7 @@ import java.util.NoSuchElementException; import java.util.Optional; import java.util.PrimitiveIterator; +import java.util.stream.IntStream; import lombok.ToString; @@ -244,15 +245,11 @@ Optional findInputs() { } private XorTable computeAllHashes() { - var tableValues = ImmutableIntArray.builder((k + 2) * N); - for (int i = 0; i < N; i++) { + var tableValues = IntStream.range(0, N).flatMap(i -> { int[] hash = hashInputs(i); - for (int j = 0; j <= k; j++) { - tableValues.add(hash[j] & (N / 2 - 1)); - } - tableValues.add(i); - } - return new XorTable(k + 1, 1, tableValues.build()); + return IntStream.range(0, k + 2).map(j -> j <= k ? hash[j] & (N / 2 - 1) : i); + }); + return new XorTable(k + 1, 1, ImmutableIntArray.copyOf(tableValues.parallel())); } private boolean testDifficultyCondition(int[] inputs) { diff --git a/common/src/main/java/bisq/common/util/Utilities.java b/common/src/main/java/bisq/common/util/Utilities.java index 7e665833c13..fca323ab4c8 100644 --- a/common/src/main/java/bisq/common/util/Utilities.java +++ b/common/src/main/java/bisq/common/util/Utilities.java @@ -20,6 +20,7 @@ import org.bitcoinj.core.Utils; import com.google.common.base.Splitter; +import com.google.common.primitives.Ints; import com.google.common.util.concurrent.FutureCallback; import com.google.common.util.concurrent.ListeningExecutorService; import com.google.common.util.concurrent.MoreExecutors; @@ -42,7 +43,6 @@ import java.net.URI; import java.net.URISyntaxException; -import java.nio.ByteBuffer; import java.nio.file.Paths; import java.io.File; @@ -536,13 +536,21 @@ public static byte[] copyRightAligned(byte[] src, int newLength) { public static byte[] intsToBytesBE(int[] ints) { byte[] bytes = new byte[ints.length * 4]; - ByteBuffer.wrap(bytes).asIntBuffer().put(ints); + int i = 0; + for (int v : ints) { + bytes[i++] = (byte) (v >> 24); + bytes[i++] = (byte) (v >> 16); + bytes[i++] = (byte) (v >> 8); + bytes[i++] = (byte) v; + } return bytes; } public static int[] bytesToIntsBE(byte[] bytes) { int[] ints = new int[bytes.length / 4]; - ByteBuffer.wrap(bytes).asIntBuffer().get(ints); + for (int i = 0, j = 0; i < bytes.length / 4; i++) { + ints[i] = Ints.fromBytes(bytes[j++], bytes[j++], bytes[j++], bytes[j++]); + } return ints; } From d8d8ec3f97a8bfa2d9777e32dc276c48ff801f58 Mon Sep 17 00:00:00 2001 From: Steven Barclay Date: Wed, 24 Nov 2021 15:07:42 +0000 Subject: [PATCH 07/15] Add PoW version(-list) fields to ProofOfWork & Filter Add a numeric version field to the 'ProofOfWork' protobuf object, along with a list of allowed version numbers, 'enabled_pow_versions', to the filter. The versions are taken to be in order of preference from most to least preferred when creating a PoW, with an empty list signifying use of the default algorithm only (that is, version 0: Hashcash). An explicit list is used instead of an upper & lower version bound, in case a new PoW algorithm (or changed algorithm params) turns out to provide worse resistance than an earlier version. (The fields are unused for now, to be enabled in a later commit.) --- .../bisq/common/crypto/HashCashService.java | 2 +- .../java/bisq/common/crypto/ProofOfWork.java | 36 +++++++++++-------- .../main/java/bisq/core/filter/Filter.java | 11 ++++++ .../resources/i18n/displayStrings.properties | 1 + .../core/user/UserPayloadModelVOTest.java | 1 + .../core/util/FeeReceiverSelectorTest.java | 1 + .../main/overlays/windows/FilterWindow.java | 23 ++++++------ proto/src/main/proto/pb.proto | 2 ++ 8 files changed, 49 insertions(+), 28 deletions(-) diff --git a/common/src/main/java/bisq/common/crypto/HashCashService.java b/common/src/main/java/bisq/common/crypto/HashCashService.java index eabba30f8d6..ed0e30f0c56 100644 --- a/common/src/main/java/bisq/common/crypto/HashCashService.java +++ b/common/src/main/java/bisq/common/crypto/HashCashService.java @@ -98,7 +98,7 @@ static CompletableFuture mint(byte[] payload, result = toSha256Hash(payload, challenge, ++counter); } while (!testDifficulty.test(result, difficulty)); - ProofOfWork proofOfWork = new ProofOfWork(payload, counter, challenge, difficulty, System.currentTimeMillis() - ts); + ProofOfWork proofOfWork = new ProofOfWork(payload, counter, challenge, difficulty, System.currentTimeMillis() - ts, 0); log.info("Completed minting proofOfWork: {}", proofOfWork); return proofOfWork; }); diff --git a/common/src/main/java/bisq/common/crypto/ProofOfWork.java b/common/src/main/java/bisq/common/crypto/ProofOfWork.java index 8f88e67a289..333e061c211 100644 --- a/common/src/main/java/bisq/common/crypto/ProofOfWork.java +++ b/common/src/main/java/bisq/common/crypto/ProofOfWork.java @@ -38,51 +38,55 @@ public final class ProofOfWork implements NetworkPayload { private final byte[] difficulty; @Getter private final long duration; + @Getter + private final int version; public ProofOfWork(byte[] payload, long counter, byte[] challenge, int difficulty, - long duration) { + long duration, + int version) { this(payload, counter, challenge, BigInteger.valueOf(difficulty).toByteArray(), - duration); + duration, + version); } public ProofOfWork(byte[] payload, long counter, byte[] challenge, BigInteger difficulty, - long duration) { + long duration, + int version) { this(payload, counter, challenge, difficulty.toByteArray(), - duration); + duration, + version); } - public ProofOfWork(byte[] payload, - long counter, - byte[] challenge, - byte[] difficulty, - long duration) { + private ProofOfWork(byte[] payload, + long counter, + byte[] challenge, + byte[] difficulty, + long duration, + int version) { this.payload = payload; this.counter = counter; this.challenge = challenge; this.difficulty = difficulty; this.duration = duration; + this.version = version; } public int getNumLeadingZeros() { return new BigInteger(difficulty).intValue(); } - public BigInteger getTarget() { - return new BigInteger(difficulty); - } - /////////////////////////////////////////////////////////////////////////////////////////// // PROTO BUFFER @@ -96,6 +100,7 @@ public protobuf.ProofOfWork toProtoMessage() { .setChallenge(ByteString.copyFrom(challenge)) .setDifficulty(ByteString.copyFrom(difficulty)) .setDuration(duration) + .setVersion(version) .build(); } @@ -105,7 +110,8 @@ public static ProofOfWork fromProto(protobuf.ProofOfWork proto) { proto.getCounter(), proto.getChallenge().toByteArray(), proto.getDifficulty().toByteArray(), - proto.getDuration() + proto.getDuration(), + proto.getVersion() ); } @@ -115,8 +121,8 @@ public String toString() { return "ProofOfWork{" + ",\r\n counter=" + counter + ",\r\n numLeadingZeros=" + getNumLeadingZeros() + - ",\r\n target=" + getTarget() + ",\r\n duration=" + duration + + ",\r\n version=" + version + "\r\n}"; } } diff --git a/core/src/main/java/bisq/core/filter/Filter.java b/core/src/main/java/bisq/core/filter/Filter.java index 3af0b8c832e..8ed55006c5d 100644 --- a/core/src/main/java/bisq/core/filter/Filter.java +++ b/core/src/main/java/bisq/core/filter/Filter.java @@ -108,6 +108,8 @@ public final class Filter implements ProtectedStoragePayload, ExpirablePayload { // Number of leading zeros for pow for BSQ swap offers. Difficulty of 8 requires 0.856 ms in average, 15 about 100 ms. // See ProofOfWorkTest for more info. private final int powDifficulty; + // Enabled PoW version numbers in reverse order of preference, starting with 0 for Hashcash. + private final List enabledPowVersions; // Added at v 1.8.0 // BSQ fee gets updated in proposals repo (e.g. https://github.com/bisq-network/proposals/issues/345) @@ -148,6 +150,7 @@ static Filter cloneWithSig(Filter filter, String signatureAsBase64) { filter.isDisableApi(), filter.isDisablePowMessage(), filter.getPowDifficulty(), + filter.getEnabledPowVersions(), filter.getMakerFeeBtc(), filter.getTakerFeeBtc(), filter.getMakerFeeBsq(), @@ -186,6 +189,7 @@ static Filter cloneWithoutSig(Filter filter) { filter.isDisableApi(), filter.isDisablePowMessage(), filter.getPowDifficulty(), + filter.getEnabledPowVersions(), filter.getMakerFeeBtc(), filter.getTakerFeeBtc(), filter.getMakerFeeBsq(), @@ -219,6 +223,7 @@ public Filter(List bannedOfferIds, boolean disableApi, boolean disablePowMessage, int powDifficulty, + List enabledPowVersions, long makerFeeBtc, long takerFeeBtc, long makerFeeBsq, @@ -253,6 +258,7 @@ public Filter(List bannedOfferIds, disableApi, disablePowMessage, powDifficulty, + enabledPowVersions, makerFeeBtc, takerFeeBtc, makerFeeBsq, @@ -295,6 +301,7 @@ public Filter(List bannedOfferIds, boolean disableApi, boolean disablePowMessage, int powDifficulty, + List enabledPowVersions, long makerFeeBtc, long takerFeeBtc, long makerFeeBsq, @@ -329,6 +336,7 @@ public Filter(List bannedOfferIds, this.disableApi = disableApi; this.disablePowMessage = disablePowMessage; this.powDifficulty = powDifficulty; + this.enabledPowVersions = enabledPowVersions; this.makerFeeBtc = makerFeeBtc; this.takerFeeBtc = takerFeeBtc; this.makerFeeBsq = makerFeeBsq; @@ -376,6 +384,7 @@ public protobuf.StoragePayload toProtoMessage() { .setDisableApi(disableApi) .setDisablePowMessage(disablePowMessage) .setPowDifficulty(powDifficulty) + .addAllEnabledPowVersions(enabledPowVersions) .setMakerFeeBtc(makerFeeBtc) .setTakerFeeBtc(takerFeeBtc) .setMakerFeeBsq(makerFeeBsq) @@ -422,6 +431,7 @@ public static Filter fromProto(protobuf.Filter proto) { proto.getDisableApi(), proto.getDisablePowMessage(), proto.getPowDifficulty(), + proto.getEnabledPowVersionsList(), proto.getMakerFeeBtc(), proto.getTakerFeeBtc(), proto.getMakerFeeBsq(), @@ -473,6 +483,7 @@ public String toString() { ",\n disableApi=" + disableApi + ",\n disablePowMessage=" + disablePowMessage + ",\n powDifficulty=" + powDifficulty + + ",\n enabledPowVersions=" + enabledPowVersions + ",\n makerFeeBtc=" + makerFeeBtc + ",\n takerFeeBtc=" + takerFeeBtc + ",\n makerFeeBsq=" + makerFeeBsq + diff --git a/core/src/main/resources/i18n/displayStrings.properties b/core/src/main/resources/i18n/displayStrings.properties index 1cc275d92bf..89ea1742bc9 100644 --- a/core/src/main/resources/i18n/displayStrings.properties +++ b/core/src/main/resources/i18n/displayStrings.properties @@ -2782,6 +2782,7 @@ filterWindow.disableApi=Disable API filterWindow.disableMempoolValidation=Disable Mempool Validation filterWindow.disablePowMessage=Disable messages requiring Proof of Work filterWindow.powDifficulty=Proof of work difficulty (BSQ swap offers) +filterWindow.enabledPowVersions=Enabled proof of work versions (comma sep. integers) filterWindow.makerFeeBtc=Min. BTC maker fee (e.g. 0.001) filterWindow.takerFeeBtc=Min. BTC taker fee (e.g. 0.007) filterWindow.makerFeeBsq=Min. BSQ maker fee (e.g. 15.14) diff --git a/core/src/test/java/bisq/core/user/UserPayloadModelVOTest.java b/core/src/test/java/bisq/core/user/UserPayloadModelVOTest.java index 3c953583027..42509b95f2d 100644 --- a/core/src/test/java/bisq/core/user/UserPayloadModelVOTest.java +++ b/core/src/test/java/bisq/core/user/UserPayloadModelVOTest.java @@ -72,6 +72,7 @@ public void testRoundtripFull() { false, false, 0, + Lists.newArrayList(), 0, 0, 0, diff --git a/core/src/test/java/bisq/core/util/FeeReceiverSelectorTest.java b/core/src/test/java/bisq/core/util/FeeReceiverSelectorTest.java index 2dcbd5b1a19..8ddf70cb153 100644 --- a/core/src/test/java/bisq/core/util/FeeReceiverSelectorTest.java +++ b/core/src/test/java/bisq/core/util/FeeReceiverSelectorTest.java @@ -133,6 +133,7 @@ private static Filter filterWithReceivers(List btcFeeReceiverAddresses) false, false, 0, + Lists.newArrayList(), 0, 0, 0, diff --git a/desktop/src/main/java/bisq/desktop/main/overlays/windows/FilterWindow.java b/desktop/src/main/java/bisq/desktop/main/overlays/windows/FilterWindow.java index d393de487df..e88d303e70b 100644 --- a/desktop/src/main/java/bisq/desktop/main/overlays/windows/FilterWindow.java +++ b/desktop/src/main/java/bisq/desktop/main/overlays/windows/FilterWindow.java @@ -41,7 +41,8 @@ import javax.inject.Named; -import org.apache.commons.lang3.StringUtils; +import com.google.common.base.Joiner; +import com.google.common.base.Splitter; import javafx.scene.Scene; import javafx.scene.control.Button; @@ -55,9 +56,6 @@ import javafx.geometry.HPos; import javafx.geometry.Insets; -import javafx.collections.FXCollections; - -import java.util.Arrays; import java.util.Collection; import java.util.HashSet; import java.util.List; @@ -190,6 +188,8 @@ private void addContent() { InputTextField powDifficultyTF = addInputTextField(gridPane, ++rowIndex, Res.get("filterWindow.powDifficulty")); powDifficultyTF.setText("0"); + InputTextField enabledPowVersionsTF = addInputTextField(gridPane, ++rowIndex, + Res.get("filterWindow.enabledPowVersions")); InputTextField makerFeeBtcTF = addInputTextField(gridPane, ++rowIndex, Res.get("filterWindow.makerFeeBtc")); InputTextField takerFeeBtcTF = addInputTextField(gridPane, ++rowIndex, @@ -217,6 +217,7 @@ private void addContent() { setupFieldFromList(btcNodesTF, filter.getBtcNodes()); setupFieldFromList(bannedPrivilegedDevPubKeysTF, filter.getBannedPrivilegedDevPubKeys()); setupFieldFromList(autoConfExplorersTF, filter.getBannedAutoConfExplorers()); + setupFieldFromList(enabledPowVersionsTF, filter.getEnabledPowVersions()); preventPublicBtcNetworkCheckBox.setSelected(filter.isPreventPublicBtcNetwork()); disableDaoCheckBox.setSelected(filter.isDisableDao()); @@ -270,6 +271,7 @@ private void addContent() { disableApiCheckBox.isSelected(), disablePowMessage.isSelected(), Integer.parseInt(powDifficultyTF.getText()), + readAsList(enabledPowVersionsTF).stream().map(Integer::parseInt).collect(Collectors.toList()), ParsingUtils.parseToCoin(makerFeeBtcTF.getText(), btcFormatter).value, ParsingUtils.parseToCoin(takerFeeBtcTF.getText(), btcFormatter).value, ParsingUtils.parseToCoin(makerFeeBsqTF.getText(), bsqFormatter).value, @@ -325,9 +327,10 @@ private void addDevFilter(Button removeFilterMessageButton, String privKeyString hide(); } - private void setupFieldFromList(InputTextField field, Collection values) { - if (values != null) - field.setText(String.join(", ", values)); + private void setupFieldFromList(InputTextField field, Collection values) { + if (values != null) { + field.setText(Joiner.on(", ").join(values)); + } } private void setupFieldFromPaymentAccountFiltersList(InputTextField field, List values) { @@ -349,11 +352,7 @@ private void setupFieldFromPaymentAccountFiltersList(InputTextField field, List< } private List readAsList(InputTextField field) { - if (field.getText().isEmpty()) { - return FXCollections.emptyObservableList(); - } else { - return Arrays.asList(StringUtils.deleteWhitespace(field.getText()).split(",")); - } + return Splitter.on(',').trimResults().omitEmptyStrings().splitToList(field.getText()); } private List readAsPaymentAccountFiltersList(InputTextField field) { diff --git a/proto/src/main/proto/pb.proto b/proto/src/main/proto/pb.proto index 0382145fd5f..549319f6ba4 100644 --- a/proto/src/main/proto/pb.proto +++ b/proto/src/main/proto/pb.proto @@ -770,6 +770,7 @@ message Filter { bool disable_mempool_validation = 28; bool disable_pow_message = 29; int32 pow_difficulty = 30; + repeated int32 enabled_pow_versions = 35; int64 maker_fee_btc = 31; int64 taker_fee_btc = 32; int64 maker_fee_bsq = 33; @@ -883,6 +884,7 @@ message ProofOfWork { bytes challenge = 3; bytes difficulty = 4; int64 duration = 5; + int32 version = 6; } message AccountAgeWitness { From cb7481d21fe68dc0e8fffff116fe0db8507b60b7 Mon Sep 17 00:00:00 2001 From: Steven Barclay Date: Thu, 25 Nov 2021 11:10:02 +0000 Subject: [PATCH 08/15] Add difficulty adjustment & benchmarking to Equihash(Test) Provide a utility method, 'Equihash::adjustDifficulty', to linearise and normalise the expected time taken to solve a puzzle, as a function of the provided difficulty, by taking into account the fact that there could be 0, 1, 2 or more puzzle solutions for any given nonce. (Wagner's algorithm is supposed to give 2 solutions on average, but the observed number is fewer, possibly due to duplicate removal.) For tractability, assume that the solution count has a Poisson distribution, which seems to have good agreement with the tests. Also add some (disabled) benchmarks to EquihashTest. These reveal an Equihash-90-5 solution time of ~146ms per puzzle per unit difficulty on a Core i3 laptop, with a verification time of ~50 microseconds. --- .../java/bisq/common/crypto/Equihash.java | 40 ++++-- .../java/bisq/common/crypto/EquihashTest.java | 136 +++++++++++++++++- 2 files changed, 163 insertions(+), 13 deletions(-) diff --git a/common/src/main/java/bisq/common/crypto/Equihash.java b/common/src/main/java/bisq/common/crypto/Equihash.java index e8eec1697d6..576f930b994 100644 --- a/common/src/main/java/bisq/common/crypto/Equihash.java +++ b/common/src/main/java/bisq/common/crypto/Equihash.java @@ -43,6 +43,7 @@ import java.util.Optional; import java.util.PrimitiveIterator; import java.util.stream.IntStream; +import java.util.stream.Stream; import lombok.ToString; @@ -80,6 +81,10 @@ @SuppressWarnings("UnstableApiUsage") public class Equihash { private static final int HASH_BIT_LENGTH = 256; + /** Observed mean solution count per nonce for Equihash-n-4 puzzles with unit difficulty. */ + public static final double EQUIHASH_n_4_MEAN_SOLUTION_COUNT_PER_NONCE = 1.63; + /** Observed mean solution count per nonce for Equihash-n-5 puzzles with unit difficulty. */ + public static final double EQUIHASH_n_5_MEAN_SOLUTION_COUNT_PER_NONCE = 1.34; private final int k, N; private final int tableCapacity; @@ -119,6 +124,13 @@ private static BigInteger inverseDifficultyMinusOne(double difficulty) { return inverse.subtract(ONE).max(BigInteger.ZERO); } + /** Adjust the provided difficulty to take the variable number of puzzle solutions per + * nonce into account, so that the expected number of attempts needed to solve a given + * puzzle equals the reciprocal of the provided difficulty. */ + public static double adjustDifficulty(double realDifficulty, double meanSolutionCountPerNonce) { + return Math.max(-meanSolutionCountPerNonce / Math.log1p(-1.0 / Math.max(realDifficulty, 1.0)), 1.0); + } + public Puzzle puzzle(byte[] seed) { return new Puzzle(seed); } @@ -205,6 +217,14 @@ public Solution findSolution() { } } } + + @VisibleForTesting + int countAllSolutionsForNonce(long nonce) { + return (int) withHashPrefix(seed, nonce).streamInputsHits() + .map(ImmutableIntArray::copyOf) + .distinct() + .count(); + } } private WithHashPrefix withHashPrefix(byte[] seed, long nonce) { @@ -228,20 +248,20 @@ private int[] hashInputs(int... inputs) { return Utilities.bytesToIntsBE(outputBytes); } - Optional findInputs() { + Stream streamInputsHits() { var table = computeAllHashes(); for (int i = 0; i < k; i++) { table = findCollisions(table, i + 1 < k); } - for (int i = 0; i < table.numRows; i++) { - if (table.getRow(i).stream().distinct().count() == inputNum) { - int[] inputs = sortInputs(table.getRow(i).toArray()); - if (testDifficultyCondition(inputs)) { - return Optional.of(inputs); - } - } - } - return Optional.empty(); + return IntStream.range(0, table.numRows) + .mapToObj(table::getRow) + .filter(row -> row.stream().distinct().count() == inputNum) + .map(row -> sortInputs(row.toArray())) + .filter(this::testDifficultyCondition); + } + + Optional findInputs() { + return streamInputsHits().findFirst(); } private XorTable computeAllHashes() { diff --git a/common/src/test/java/bisq/common/crypto/EquihashTest.java b/common/src/test/java/bisq/common/crypto/EquihashTest.java index 6143a54c605..60611568de4 100644 --- a/common/src/test/java/bisq/common/crypto/EquihashTest.java +++ b/common/src/test/java/bisq/common/crypto/EquihashTest.java @@ -18,14 +18,23 @@ package bisq.common.crypto; import bisq.common.crypto.Equihash.Puzzle.Solution; +import bisq.common.util.Utilities; +import com.google.common.base.Stopwatch; import com.google.common.base.Strings; +import com.google.common.collect.ConcurrentHashMultiset; +import com.google.common.collect.ImmutableMultiset; +import com.google.common.collect.Multiset; import java.util.Arrays; import java.util.stream.Collectors; +import java.util.stream.IntStream; +import org.junit.Ignore; import org.junit.Test; +import static bisq.common.crypto.Equihash.EQUIHASH_n_5_MEAN_SOLUTION_COUNT_PER_NONCE; +import static java.lang.Double.POSITIVE_INFINITY; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; @@ -42,13 +51,27 @@ public void testHashUpperBound() { assertEquals("0083126e 978d4fdf 3b645a1c ac083126 e978d4fd f3b645a1 cac08312 6e978d4f", hub(500.0)); assertEquals("00000000 00000000 2f394219 248446ba a23d2ec7 29af3d61 0607aa01 67dd94ca", hub(1.0e20)); assertEquals("00000000 00000000 00000000 00000000 ffffffff ffffffff ffffffff ffffffff", hub(0x1.0p128)); - assertEquals("00000000 00000000 00000000 00000000 00000000 00000000 00000000 00000000", hub(Double.POSITIVE_INFINITY)); + assertEquals("00000000 00000000 00000000 00000000 00000000 00000000 00000000 00000000", hub(POSITIVE_INFINITY)); + } + + @Test + public void testAdjustDifficulty() { + assertEquals(1.0, Equihash.adjustDifficulty(0.0, 1.34), 0.0001); + assertEquals(1.0, Equihash.adjustDifficulty(0.5, 1.34), 0.0001); + assertEquals(1.0, Equihash.adjustDifficulty(1.0, 1.34), 0.0001); + assertEquals(1.0, Equihash.adjustDifficulty(1.2, 1.34), 0.0001); + assertEquals(1.22, Equihash.adjustDifficulty(1.5, 1.34), 0.01); + assertEquals(1.93, Equihash.adjustDifficulty(2.0, 1.34), 0.01); + assertEquals(2.62, Equihash.adjustDifficulty(2.5, 1.34), 0.01); + assertEquals(3.30, Equihash.adjustDifficulty(3.0, 1.34), 0.01); + assertEquals(134.0, Equihash.adjustDifficulty(100.0, 1.34), 1.0); + assertEquals(Equihash.adjustDifficulty(POSITIVE_INFINITY, 1.34), POSITIVE_INFINITY, 1.0); } @Test public void testFindSolution() { - Equihash equihash = new Equihash(90, 5, 5.0); - byte[] seed = new byte[64]; + Equihash equihash = new Equihash(90, 5, 2.0); + byte[] seed = new byte[32]; Solution solution = equihash.puzzle(seed).findSolution(); byte[] solutionBytes = solution.serialize(); @@ -59,6 +82,113 @@ public void testFindSolution() { assertEquals(solution.toString(), roundTrippedSolution.toString()); } + @Test + @Ignore + public void benchmarkFindSolution() { + // On Intel Core i3 CPU M 330 @ 2.13GHz ... + // + // For Equihash-90-5 with real difficulty 2.0, adjusted difficulty 1.933211354791211 ... + // Total elapsed solution time: 292789 ms + // Mean time to solve one puzzle: 292 ms + // Puzzle solution time per unit difficulty: 146 ms + // + double adjustedDifficulty = Equihash.adjustDifficulty(2.0, EQUIHASH_n_5_MEAN_SOLUTION_COUNT_PER_NONCE); + Equihash equihash = new Equihash(90, 5, adjustedDifficulty); + + Stopwatch stopwatch = Stopwatch.createStarted(); + for (int i = 0; i < 1000; i++) { + byte[] seed = Utilities.intsToBytesBE(new int[]{0, 0, 0, 0, 0, 0, 0, i}); + equihash.puzzle(seed).findSolution(); + } + stopwatch.stop(); + var duration = stopwatch.elapsed(); + + System.out.println("For Equihash-90-5 with real difficulty 2.0, adjusted difficulty " + adjustedDifficulty + " ..."); + System.out.println("Total elapsed solution time: " + duration.toMillis() + " ms"); + System.out.println("Mean time to solve one puzzle: " + duration.dividedBy(1000).toMillis() + " ms"); + System.out.println("Puzzle solution time per unit difficulty: " + duration.dividedBy(2000).toMillis() + " ms"); + } + + @Test + @Ignore + public void benchmarkVerify() { + // On Intel Core i3 CPU M 330 @ 2.13GHz ... + // + // For Equihash-90-5 ... + // Total elapsed verification time: 50046 ms + // Mean time to verify one solution: 50046 ns + // + Equihash equihash = new Equihash(90, 5, 1.0); + byte[] seed = new byte[32]; + Solution solution = equihash.puzzle(seed).findSolution(); + + Stopwatch stopwatch = Stopwatch.createStarted(); + for (int i = 0; i < 1_000_000; i++) { + solution.verify(); + } + stopwatch.stop(); + var duration = stopwatch.elapsed(); + + System.out.println("For Equihash-90-5 ..."); + System.out.println("Total elapsed verification time: " + duration.toMillis() + " ms"); + System.out.println("Mean time to verify one solution: " + duration.dividedBy(1_000_000).toNanos() + " ns"); + } + + private static final int SAMPLE_NO = 10000; + + @Test + @Ignore + public void solutionCountPerNonceStats() { + // For Equihash-60-4... + // Got puzzle solution count mean: 1.6161 + // Got expected count stats: [0 x 1987, 1 x 3210, 2 x 2595, 3 x 1398, 4 x 564, 5 x 183, 6 x 49, 7 x 11, 8 x 3] + // Got actual count stats: [0 x 2014, 1 x 3230, 2 x 2546, 3 x 1395, 4 x 543, 5 x 191, 6 x 50, 7 x 24, 8 x 4, 9 x 3] + // + // For Equihash-70-4... + // Got puzzle solution count mean: 1.6473 + // Got expected count stats: [0 x 1926, 1 x 3172, 2 x 2613, 3 x 1434, 4 x 591, 5 x 195, 6 x 53, 7 x 13, 8 x 2, 9] + // Got actual count stats: [0 x 1958, 1 x 3172, 2 x 2584, 3 x 1413, 4 x 585, 5 x 204, 6 x 61, 7 x 17, 8 x 5, 9] + // + // For Equihash-90-5... + // Got puzzle solution count mean: 1.3419 + // Got expected count stats: [0 x 2613, 1 x 3508, 2 x 2353, 3 x 1052, 4 x 353, 5 x 95, 6 x 21, 7 x 4, 8] + // Got actual count stats: [0 x 2698, 1 x 3446, 2 x 2311, 3 x 1045, 4 x 352, 5 x 104, 6 x 33, 7 x 5, 8 x 3, 9, 10, 12] + // + // For Equihash-96-5... + // Got puzzle solution count mean: 1.3363 + // Got expected count stats: [0 x 2628, 1 x 3512, 2 x 2347, 3 x 1045, 4 x 349, 5 x 93, 6 x 21, 7 x 4, 8] + // Got actual count stats: [0 x 2708, 1 x 3409, 2 x 2344, 3 x 1048, 4 x 368, 5 x 94, 6 x 23, 7 x 6] + // + Equihash equihash = new Equihash(90, 5, 1.0); + byte[] seed = new byte[32]; + + Multiset stats = ConcurrentHashMultiset.create(); + IntStream.range(0, SAMPLE_NO).parallel().forEach(nonce -> + stats.add(equihash.puzzle(seed).countAllSolutionsForNonce(nonce))); + + double mean = (stats.entrySet().stream() + .mapToInt(entry -> entry.getElement() * entry.getCount()) + .sum()) / (double) SAMPLE_NO; + + System.out.println("For Equihash-90-5..."); + System.out.println("Got puzzle solution count mean: " + mean); + System.out.println("Got expected count stats: " + expectedStatsFromPoissonDistribution(mean)); + System.out.println("Got actual count stats: " + stats); + } + + private Multiset expectedStatsFromPoissonDistribution(double mean) { + var setBuilder = ImmutableMultiset.builder(); + double prob = Math.exp(-mean), roundError = 0.0; + for (int i = 0, total = 0; total < SAMPLE_NO; i++) { + int n = (int) (roundError + prob * SAMPLE_NO + 0.5); + setBuilder.addCopies(i, n); + roundError += prob * SAMPLE_NO - n; + total += n; + prob *= mean / (i + 1); + } + return setBuilder.build(); + } + private static String hub(double difficulty) { return hexString(Equihash.hashUpperBound(difficulty)); } From 647fc862d40d04543e2f402d0cf7015908a66d12 Mon Sep 17 00:00:00 2001 From: Steven Barclay Date: Thu, 25 Nov 2021 11:44:04 +0000 Subject: [PATCH 09/15] Fix method name typo: redoProofOfWorkAndRepublish --- .../offer/bsq_swap/OpenBsqSwapOfferService.java | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/core/src/main/java/bisq/core/offer/bsq_swap/OpenBsqSwapOfferService.java b/core/src/main/java/bisq/core/offer/bsq_swap/OpenBsqSwapOfferService.java index b55a1b70e44..3f202abc300 100644 --- a/core/src/main/java/bisq/core/offer/bsq_swap/OpenBsqSwapOfferService.java +++ b/core/src/main/java/bisq/core/offer/bsq_swap/OpenBsqSwapOfferService.java @@ -247,7 +247,7 @@ public void activateOpenOffer(OpenOffer openOffer, ResultHandler resultHandler, ErrorMessageHandler errorMessageHandler) { if (isProofOfWorkInvalid(openOffer.getOffer())) { - redoProofOrWorkAndRepublish(openOffer); + redoProofOfWorkAndRepublish(openOffer); return; } @@ -265,7 +265,7 @@ void requestPersistence() { void enableBsqSwapOffer(OpenOffer openOffer) { if (isProofOfWorkInvalid(openOffer.getOffer())) { - redoProofOrWorkAndRepublish(openOffer); + redoProofOfWorkAndRepublish(openOffer); return; } @@ -297,7 +297,7 @@ private void onOpenOffersAdded(List list) { .forEach(openOffer -> { if (isProofOfWorkInvalid(openOffer.getOffer())) { // Avoiding ConcurrentModificationException - UserThread.execute(() -> redoProofOrWorkAndRepublish(openOffer)); + UserThread.execute(() -> redoProofOfWorkAndRepublish(openOffer)); } else { OpenBsqSwapOffer openBsqSwapOffer = new OpenBsqSwapOffer(openOffer, this, @@ -333,7 +333,7 @@ private void onProofOfWorkDifficultyChanged() { .filter(openBsqSwapOffer -> isProofOfWorkInvalid(openBsqSwapOffer.getOffer())) .forEach(openBsqSwapOffer -> { // Avoiding ConcurrentModificationException - UserThread.execute(() -> redoProofOrWorkAndRepublish(openBsqSwapOffer.getOpenOffer())); + UserThread.execute(() -> redoProofOfWorkAndRepublish(openBsqSwapOffer.getOpenOffer())); }); } @@ -342,8 +342,8 @@ private void onProofOfWorkDifficultyChanged() { // Proof of work /////////////////////////////////////////////////////////////////////////////////////////// - private void redoProofOrWorkAndRepublish(OpenOffer openOffer) { - // This triggers our onOpenOffersRemoved handler so we dont handle removal here + private void redoProofOfWorkAndRepublish(OpenOffer openOffer) { + // This triggers our onOpenOffersRemoved handler so we don't handle removal here openOfferManager.removeOpenOffer(openOffer); String newOfferId = OfferUtil.getOfferIdWithMutationCounter(openOffer.getId()); @@ -374,7 +374,7 @@ private void redoProofOrWorkAndRepublish(OpenOffer openOffer) { if (!newOpenOffer.isDeactivated()) { openOfferManager.maybeRepublishOffer(newOpenOffer); } - // This triggers our onOpenOffersAdded handler so we dont handle adding to our list here + // This triggers our onOpenOffersAdded handler so we don't handle adding to our list here openOfferManager.addOpenBsqSwapOffer(newOpenOffer); }); }); From 0c94e232f8b511a4d9cca9e3031c219726f4b514 Mon Sep 17 00:00:00 2001 From: Steven Barclay Date: Thu, 25 Nov 2021 16:57:04 +0000 Subject: [PATCH 10/15] Add+enable service to mint/verify Equihash proofs of work Add an abstract base class, 'ProofOfWorkService', for the existing PoW implementation 'HashCashService' and a new 'EquihashProofOfWorkService' PoW implementation based on Equihash-90-5 (which has 72 byte solutions & 5-10 MB peak memory usage). Since the current 'ProofOfWork' protobuf object only provides a 64-bit counter field to hold the puzzle solution (as that is all Hashcash requires), repurpose the 'payload' field to hold the Equihash puzzle solution bytes, with the 'challenge' field equal to the puzzle seed: the SHA256 hash of the offerId & makerAddress. Use a difficulty scale factor of 3e-5 (derived from benchmarking) to try to make the average Hashcash & Equihash puzzle solution times roughly equal for any given log-difficulty/numLeadingZeros integer chosen in the filter. NOTE: An empty enabled-version-list in the filter defaults to Hashcash (= version 0) only. The new Equihash-90-5 PoW scheme is version 1. --- .../crypto/EquihashProofOfWorkService.java | 76 +++++++++++++++++++ .../bisq/common/crypto/HashCashService.java | 44 +++++++---- .../common/crypto/ProofOfWorkService.java | 72 ++++++++++++++++++ .../common/crypto/HashCashServiceTest.java | 4 +- .../java/bisq/core/filter/FilterManager.java | 17 ++++- .../bsq_swap/OpenBsqSwapOfferService.java | 29 ++++--- 6 files changed, 213 insertions(+), 29 deletions(-) create mode 100644 common/src/main/java/bisq/common/crypto/EquihashProofOfWorkService.java create mode 100644 common/src/main/java/bisq/common/crypto/ProofOfWorkService.java diff --git a/common/src/main/java/bisq/common/crypto/EquihashProofOfWorkService.java b/common/src/main/java/bisq/common/crypto/EquihashProofOfWorkService.java new file mode 100644 index 00000000000..25da45688ec --- /dev/null +++ b/common/src/main/java/bisq/common/crypto/EquihashProofOfWorkService.java @@ -0,0 +1,76 @@ +/* + * This file is part of Bisq. + * + * Bisq is free software: you can redistribute it and/or modify it + * under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or (at + * your option) any later version. + * + * Bisq is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public + * License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with Bisq. If not, see . + */ + +package bisq.common.crypto; + +import com.google.common.primitives.Longs; + +import org.apache.commons.lang3.StringUtils; + +import java.util.Arrays; +import java.util.concurrent.CompletableFuture; + +import lombok.extern.slf4j.Slf4j; + +import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.base.Preconditions.checkNotNull; + +@Slf4j +public class EquihashProofOfWorkService extends ProofOfWorkService { + /** Rough cost of one Hashcash iteration compared to solving an Equihash-90-5 puzzle of unit difficulty. */ + private static final double DIFFICULTY_SCALE_FACTOR = 3.0e-5; + + EquihashProofOfWorkService(int version) { + super(version); + } + + @Override + public CompletableFuture mint(String itemId, byte[] challenge, int log2Difficulty) { + double difficulty = adjustedDifficulty(log2Difficulty); + log.info("Got adjusted difficulty: {}", difficulty); + + return CompletableFuture.supplyAsync(() -> { + long ts = System.currentTimeMillis(); + byte[] solution = new Equihash(90, 5, difficulty).puzzle(challenge).findSolution().serialize(); + long counter = Longs.fromByteArray(Arrays.copyOf(solution, 8)); + var proofOfWork = new ProofOfWork(solution, counter, challenge, log2Difficulty, + System.currentTimeMillis() - ts, getVersion()); + log.info("Completed minting proofOfWork: {}", proofOfWork); + return proofOfWork; + }); + } + + @Override + public byte[] getChallenge(String itemId, String ownerId) { + checkArgument(!StringUtils.contains(itemId, '\0')); + checkArgument(!StringUtils.contains(ownerId, '\0')); + return Hash.getSha256Hash(checkNotNull(itemId) + "\0" + checkNotNull(ownerId)); + } + + @Override + boolean verify(ProofOfWork proofOfWork) { + double difficulty = adjustedDifficulty(proofOfWork.getNumLeadingZeros()); + + var puzzle = new Equihash(90, 5, difficulty).puzzle(proofOfWork.getChallenge()); + return puzzle.deserializeSolution(proofOfWork.getPayload()).verify(); + } + + private static double adjustedDifficulty(int log2Difficulty) { + return Equihash.adjustDifficulty(Math.scalb(DIFFICULTY_SCALE_FACTOR, log2Difficulty), + Equihash.EQUIHASH_n_5_MEAN_SOLUTION_COUNT_PER_NONCE); + } +} diff --git a/common/src/main/java/bisq/common/crypto/HashCashService.java b/common/src/main/java/bisq/common/crypto/HashCashService.java index ed0e30f0c56..bebbdd8246b 100644 --- a/common/src/main/java/bisq/common/crypto/HashCashService.java +++ b/common/src/main/java/bisq/common/crypto/HashCashService.java @@ -35,40 +35,56 @@ * See https://www.hashcash.org/papers/hashcash.pdf */ @Slf4j -public class HashCashService { +public class HashCashService extends ProofOfWorkService { // Default validations. Custom implementations might use tolerance. private static final BiPredicate isChallengeValid = Arrays::equals; private static final BiPredicate isDifficultyValid = Integer::equals; - public static CompletableFuture mint(byte[] payload, - byte[] challenge, - int difficulty) { + HashCashService() { + super(0); + } + + @Override + public CompletableFuture mint(String itemId, byte[] challenge, int log2Difficulty) { + byte[] payload = getBytes(itemId); + return mint(payload, challenge, log2Difficulty); + } + + @Override + public byte[] getChallenge(String itemId, String ownerId) { + return getBytes(itemId + ownerId); + } + + static CompletableFuture mint(byte[] payload, + byte[] challenge, + int difficulty) { return HashCashService.mint(payload, challenge, difficulty, HashCashService::testDifficulty); } - public static boolean verify(ProofOfWork proofOfWork) { + @Override + boolean verify(ProofOfWork proofOfWork) { return verify(proofOfWork, proofOfWork.getChallenge(), proofOfWork.getNumLeadingZeros()); } - public static boolean verify(ProofOfWork proofOfWork, - byte[] controlChallenge, - int controlDifficulty) { + static boolean verify(ProofOfWork proofOfWork, + byte[] controlChallenge, + int controlDifficulty) { return HashCashService.verify(proofOfWork, controlChallenge, controlDifficulty, HashCashService::testDifficulty); } - public static boolean verify(ProofOfWork proofOfWork, - byte[] controlChallenge, - int controlDifficulty, - BiPredicate challengeValidation, - BiPredicate difficultyValidation) { + static boolean verify(ProofOfWork proofOfWork, + byte[] controlChallenge, + int controlDifficulty, + BiPredicate challengeValidation, + BiPredicate difficultyValidation) { return HashCashService.verify(proofOfWork, controlChallenge, controlDifficulty, @@ -139,7 +155,7 @@ private static boolean verify(ProofOfWork proofOfWork, BiPredicate. + */ + +package bisq.common.crypto; + +import com.google.common.base.Preconditions; + +import java.util.Optional; +import java.util.concurrent.CompletableFuture; +import java.util.function.BiPredicate; + +import lombok.Getter; + +public abstract class ProofOfWorkService { + private static class InstanceHolder { + private static final ProofOfWorkService[] INSTANCES = { + new HashCashService(), + new EquihashProofOfWorkService(1) + }; + } + + public static Optional forVersion(int version) { + return version >= 0 && version < InstanceHolder.INSTANCES.length ? + Optional.of(InstanceHolder.INSTANCES[version]) : Optional.empty(); + } + + @Getter + private final int version; + + ProofOfWorkService(int version) { + this.version = version; + } + + public abstract CompletableFuture mint(String itemId, byte[] challenge, int log2Difficulty); + + public abstract byte[] getChallenge(String itemId, String ownerId); + + abstract boolean verify(ProofOfWork proofOfWork); + + public CompletableFuture mint(String itemId, String ownerId, int log2Difficulty) { + return mint(itemId, getChallenge(itemId, ownerId), log2Difficulty); + } + + public boolean verify(ProofOfWork proofOfWork, + String itemId, + String ownerId, + int controlLog2Difficulty, + BiPredicate challengeValidation, + BiPredicate difficultyValidation) { + + Preconditions.checkArgument(proofOfWork.getVersion() == version); + + byte[] controlChallenge = getChallenge(itemId, ownerId); + return challengeValidation.test(proofOfWork.getChallenge(), controlChallenge) && + difficultyValidation.test(proofOfWork.getNumLeadingZeros(), controlLog2Difficulty) && + verify(proofOfWork); + } +} diff --git a/common/src/test/java/bisq/common/crypto/HashCashServiceTest.java b/common/src/test/java/bisq/common/crypto/HashCashServiceTest.java index 115483d25fb..9c4fcb1f53c 100644 --- a/common/src/test/java/bisq/common/crypto/HashCashServiceTest.java +++ b/common/src/test/java/bisq/common/crypto/HashCashServiceTest.java @@ -36,7 +36,7 @@ public void testNumberOfLeadingZeros() { @Test public void testDiffIncrease() throws ExecutionException, InterruptedException { StringBuilder stringBuilder = new StringBuilder(); - for (int i = 0; i < 12; i++) { + for (int i = 0; i < 9; i++) { run(i, stringBuilder); } log.info(stringBuilder.toString()); @@ -70,7 +70,7 @@ private void run(int difficulty, StringBuilder stringBuilder) throws ExecutionEx double size = tokens.size(); long ts2 = System.currentTimeMillis(); long averageCounter = Math.round(tokens.stream().mapToLong(ProofOfWork::getCounter).average().orElse(0)); - boolean allValid = tokens.stream().allMatch(HashCashService::verify); + boolean allValid = tokens.stream().allMatch(new HashCashService()::verify); assertTrue(allValid); double time1 = (System.currentTimeMillis() - ts) / size; double time2 = (System.currentTimeMillis() - ts2) / size; diff --git a/core/src/main/java/bisq/core/filter/FilterManager.java b/core/src/main/java/bisq/core/filter/FilterManager.java index 3ecc11a7465..4c0b74f44f8 100644 --- a/core/src/main/java/bisq/core/filter/FilterManager.java +++ b/core/src/main/java/bisq/core/filter/FilterManager.java @@ -37,8 +37,9 @@ import bisq.common.app.Version; import bisq.common.config.Config; import bisq.common.config.ConfigFileEditor; -import bisq.common.crypto.HashCashService; +import bisq.common.crypto.ProofOfWorkService; import bisq.common.crypto.KeyRing; +import bisq.common.crypto.ProofOfWork; import org.bitcoinj.core.ECKey; import org.bitcoinj.core.Sha256Hash; @@ -493,13 +494,23 @@ public boolean isProofOfWorkValid(Offer offer) { } checkArgument(offer.getBsqSwapOfferPayload().isPresent(), "Offer payload must be BsqSwapOfferPayload"); - return HashCashService.verify(offer.getBsqSwapOfferPayload().get().getProofOfWork(), - HashCashService.getBytes(offer.getId() + offer.getOwnerNodeAddress().toString()), + ProofOfWork pow = offer.getBsqSwapOfferPayload().get().getProofOfWork(); + var service = ProofOfWorkService.forVersion(pow.getVersion()); + if (!service.isPresent() || !getEnabledPowVersions().contains(pow.getVersion())) { + return false; + } + return service.get().verify(offer.getBsqSwapOfferPayload().get().getProofOfWork(), + offer.getId(), offer.getOwnerNodeAddress().toString(), filter.getPowDifficulty(), challengeValidation, difficultyValidation); } + public List getEnabledPowVersions() { + Filter filter = getFilter(); + return filter != null && !filter.getEnabledPowVersions().isEmpty() ? filter.getEnabledPowVersions() : List.of(0); + } + /////////////////////////////////////////////////////////////////////////////////////////// // Private diff --git a/core/src/main/java/bisq/core/offer/bsq_swap/OpenBsqSwapOfferService.java b/core/src/main/java/bisq/core/offer/bsq_swap/OpenBsqSwapOfferService.java index 3f202abc300..962983b8209 100644 --- a/core/src/main/java/bisq/core/offer/bsq_swap/OpenBsqSwapOfferService.java +++ b/core/src/main/java/bisq/core/offer/bsq_swap/OpenBsqSwapOfferService.java @@ -43,7 +43,7 @@ import bisq.common.UserThread; import bisq.common.app.Version; -import bisq.common.crypto.HashCashService; +import bisq.common.crypto.ProofOfWorkService; import bisq.common.crypto.PubKeyRing; import bisq.common.handlers.ErrorMessageHandler; import bisq.common.handlers.ResultHandler; @@ -61,6 +61,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.NoSuchElementException; import java.util.Objects; import java.util.concurrent.TimeUnit; import java.util.function.Consumer; @@ -195,13 +196,11 @@ public void requestNewOffer(String offerId, amount.value, minAmount.value); - NodeAddress makerAddress = p2PService.getAddress(); + NodeAddress makerAddress = Objects.requireNonNull(p2PService.getAddress()); offerUtil.validateBasicOfferData(PaymentMethod.BSQ_SWAP, "BSQ"); - byte[] payload = HashCashService.getBytes(offerId); - byte[] challenge = HashCashService.getBytes(offerId + Objects.requireNonNull(makerAddress)); - int difficulty = getPowDifficulty(); - HashCashService.mint(payload, challenge, difficulty) + int log2Difficulty = getPowDifficulty(); + getPowService().mint(offerId, makerAddress.getFullAddress(), log2Difficulty) .whenComplete((proofOfWork, throwable) -> { // We got called from a non user thread... UserThread.execute(() -> { @@ -347,11 +346,9 @@ private void redoProofOfWorkAndRepublish(OpenOffer openOffer) { openOfferManager.removeOpenOffer(openOffer); String newOfferId = OfferUtil.getOfferIdWithMutationCounter(openOffer.getId()); - byte[] payload = HashCashService.getBytes(newOfferId); NodeAddress nodeAddress = Objects.requireNonNull(openOffer.getOffer().getMakerNodeAddress()); - byte[] challenge = HashCashService.getBytes(newOfferId + nodeAddress); - int difficulty = getPowDifficulty(); - HashCashService.mint(payload, challenge, difficulty) + int log2Difficulty = getPowDifficulty(); + getPowService().mint(newOfferId, nodeAddress.getFullAddress(), log2Difficulty) .whenComplete((proofOfWork, throwable) -> { // We got called from a non user thread... UserThread.execute(() -> { @@ -393,4 +390,16 @@ private boolean isProofOfWorkInvalid(Offer offer) { private int getPowDifficulty() { return filterManager.getFilter() != null ? filterManager.getFilter().getPowDifficulty() : 0; } + + private ProofOfWorkService getPowService() { + var service = filterManager.getEnabledPowVersions().stream() + .flatMap(v -> ProofOfWorkService.forVersion(v).stream()) + .findFirst(); + if (!service.isPresent()) { + // We cannot exit normally, else we get caught in an infinite loop generating invalid PoWs. + throw new NoSuchElementException("Could not find a suitable PoW version to use."); + } + log.info("Selected PoW version {}, service instance {}", service.get().getVersion(), service.get()); + return service.get(); + } } From e0595aa284ce18ab7602f1126d23014da336ef69 Mon Sep 17 00:00:00 2001 From: Steven Barclay Date: Fri, 26 Nov 2021 04:06:24 +0000 Subject: [PATCH 11/15] Store difficulty as floating point in Filter & PoW Change the type of the 'difficulty' field in the Filter & ProofOfWork proto objects from int32/bytes to double and make it use a linear scale, in place of the original logarithmic scale which counts the (effective) number of required zeros. This allows fine-grained difficulty control for Equihash, though for Hashcash it simply rounds up to the nearest power of 2 internally. NOTE: This is a breaking change to PoW & filter serialisation (unlike the earlier PR commits), as the proto field version nums aren't updated. --- .../crypto/EquihashProofOfWorkService.java | 20 ++++---- .../bisq/common/crypto/HashCashService.java | 41 ++++++++++------- .../java/bisq/common/crypto/ProofOfWork.java | 46 +++---------------- .../common/crypto/ProofOfWorkService.java | 12 ++--- .../common/crypto/HashCashServiceTest.java | 16 ++++++- .../main/java/bisq/core/filter/Filter.java | 11 +++-- .../java/bisq/core/filter/FilterManager.java | 2 +- .../bsq_swap/OpenBsqSwapOfferService.java | 12 ++--- .../main/overlays/windows/FilterWindow.java | 4 +- proto/src/main/proto/pb.proto | 4 +- 10 files changed, 77 insertions(+), 91 deletions(-) diff --git a/common/src/main/java/bisq/common/crypto/EquihashProofOfWorkService.java b/common/src/main/java/bisq/common/crypto/EquihashProofOfWorkService.java index 25da45688ec..70ad4f3e0b4 100644 --- a/common/src/main/java/bisq/common/crypto/EquihashProofOfWorkService.java +++ b/common/src/main/java/bisq/common/crypto/EquihashProofOfWorkService.java @@ -31,7 +31,7 @@ @Slf4j public class EquihashProofOfWorkService extends ProofOfWorkService { - /** Rough cost of one Hashcash iteration compared to solving an Equihash-90-5 puzzle of unit difficulty. */ + /** Rough cost of two Hashcash iterations compared to solving an Equihash-90-5 puzzle of unit difficulty. */ private static final double DIFFICULTY_SCALE_FACTOR = 3.0e-5; EquihashProofOfWorkService(int version) { @@ -39,15 +39,15 @@ public class EquihashProofOfWorkService extends ProofOfWorkService { } @Override - public CompletableFuture mint(String itemId, byte[] challenge, int log2Difficulty) { - double difficulty = adjustedDifficulty(log2Difficulty); - log.info("Got adjusted difficulty: {}", difficulty); + public CompletableFuture mint(String itemId, byte[] challenge, double difficulty) { + double scaledDifficulty = scaledDifficulty(difficulty); + log.info("Got scaled & adjusted difficulty: {}", scaledDifficulty); return CompletableFuture.supplyAsync(() -> { long ts = System.currentTimeMillis(); - byte[] solution = new Equihash(90, 5, difficulty).puzzle(challenge).findSolution().serialize(); + byte[] solution = new Equihash(90, 5, scaledDifficulty).puzzle(challenge).findSolution().serialize(); long counter = Longs.fromByteArray(Arrays.copyOf(solution, 8)); - var proofOfWork = new ProofOfWork(solution, counter, challenge, log2Difficulty, + var proofOfWork = new ProofOfWork(solution, counter, challenge, difficulty, System.currentTimeMillis() - ts, getVersion()); log.info("Completed minting proofOfWork: {}", proofOfWork); return proofOfWork; @@ -63,14 +63,14 @@ public byte[] getChallenge(String itemId, String ownerId) { @Override boolean verify(ProofOfWork proofOfWork) { - double difficulty = adjustedDifficulty(proofOfWork.getNumLeadingZeros()); + double scaledDifficulty = scaledDifficulty(proofOfWork.getDifficulty()); - var puzzle = new Equihash(90, 5, difficulty).puzzle(proofOfWork.getChallenge()); + var puzzle = new Equihash(90, 5, scaledDifficulty).puzzle(proofOfWork.getChallenge()); return puzzle.deserializeSolution(proofOfWork.getPayload()).verify(); } - private static double adjustedDifficulty(int log2Difficulty) { - return Equihash.adjustDifficulty(Math.scalb(DIFFICULTY_SCALE_FACTOR, log2Difficulty), + private static double scaledDifficulty(double difficulty) { + return Equihash.adjustDifficulty(DIFFICULTY_SCALE_FACTOR * difficulty, Equihash.EQUIHASH_n_5_MEAN_SOLUTION_COUNT_PER_NONCE); } } diff --git a/common/src/main/java/bisq/common/crypto/HashCashService.java b/common/src/main/java/bisq/common/crypto/HashCashService.java index bebbdd8246b..092957817dd 100644 --- a/common/src/main/java/bisq/common/crypto/HashCashService.java +++ b/common/src/main/java/bisq/common/crypto/HashCashService.java @@ -45,9 +45,9 @@ public class HashCashService extends ProofOfWorkService { } @Override - public CompletableFuture mint(String itemId, byte[] challenge, int log2Difficulty) { + public CompletableFuture mint(String itemId, byte[] challenge, double difficulty) { byte[] payload = getBytes(itemId); - return mint(payload, challenge, log2Difficulty); + return mint(payload, challenge, difficulty); } @Override @@ -57,7 +57,7 @@ public byte[] getChallenge(String itemId, String ownerId) { static CompletableFuture mint(byte[] payload, byte[] challenge, - int difficulty) { + double difficulty) { return HashCashService.mint(payload, challenge, difficulty, @@ -68,33 +68,33 @@ static CompletableFuture mint(byte[] payload, boolean verify(ProofOfWork proofOfWork) { return verify(proofOfWork, proofOfWork.getChallenge(), - proofOfWork.getNumLeadingZeros()); + toNumLeadingZeros(proofOfWork.getDifficulty())); } static boolean verify(ProofOfWork proofOfWork, byte[] controlChallenge, - int controlDifficulty) { + int controlLog2Difficulty) { return HashCashService.verify(proofOfWork, controlChallenge, - controlDifficulty, + controlLog2Difficulty, HashCashService::testDifficulty); } static boolean verify(ProofOfWork proofOfWork, byte[] controlChallenge, - int controlDifficulty, + int controlLog2Difficulty, BiPredicate challengeValidation, BiPredicate difficultyValidation) { return HashCashService.verify(proofOfWork, controlChallenge, - controlDifficulty, + controlLog2Difficulty, challengeValidation, difficultyValidation, HashCashService::testDifficulty); } - private static boolean testDifficulty(byte[] result, long difficulty) { - return HashCashService.numberOfLeadingZeros(result) > difficulty; + private static boolean testDifficulty(byte[] result, int log2Difficulty) { + return HashCashService.numberOfLeadingZeros(result) > log2Difficulty; } @@ -104,16 +104,17 @@ private static boolean testDifficulty(byte[] result, long difficulty) { static CompletableFuture mint(byte[] payload, byte[] challenge, - int difficulty, + double difficulty, BiPredicate testDifficulty) { return CompletableFuture.supplyAsync(() -> { long ts = System.currentTimeMillis(); + int log2Difficulty = toNumLeadingZeros(difficulty); byte[] result; long counter = 0; do { result = toSha256Hash(payload, challenge, ++counter); } - while (!testDifficulty.test(result, difficulty)); + while (!testDifficulty.test(result, log2Difficulty)); ProofOfWork proofOfWork = new ProofOfWork(payload, counter, challenge, difficulty, System.currentTimeMillis() - ts, 0); log.info("Completed minting proofOfWork: {}", proofOfWork); return proofOfWork; @@ -122,11 +123,11 @@ static CompletableFuture mint(byte[] payload, static boolean verify(ProofOfWork proofOfWork, byte[] controlChallenge, - int controlDifficulty, + int controlLog2Difficulty, BiPredicate testDifficulty) { return verify(proofOfWork, controlChallenge, - controlDifficulty, + controlLog2Difficulty, HashCashService.isChallengeValid, HashCashService.isDifficultyValid, testDifficulty); @@ -134,12 +135,12 @@ static boolean verify(ProofOfWork proofOfWork, static boolean verify(ProofOfWork proofOfWork, byte[] controlChallenge, - int controlDifficulty, + int controlLog2Difficulty, BiPredicate challengeValidation, BiPredicate difficultyValidation, BiPredicate testDifficulty) { return challengeValidation.test(proofOfWork.getChallenge(), controlChallenge) && - difficultyValidation.test(proofOfWork.getNumLeadingZeros(), controlDifficulty) && + difficultyValidation.test(toNumLeadingZeros(proofOfWork.getDifficulty()), controlLog2Difficulty) && verify(proofOfWork, testDifficulty); } @@ -147,7 +148,7 @@ private static boolean verify(ProofOfWork proofOfWork, BiPredicate>> 1); } + + // round up to nearest power-of-two and take the base-2 log + @VisibleForTesting + static int toNumLeadingZeros(double difficulty) { + return Math.getExponent(Math.max(Math.nextDown(difficulty), 0.5)) + 1; + } } diff --git a/common/src/main/java/bisq/common/crypto/ProofOfWork.java b/common/src/main/java/bisq/common/crypto/ProofOfWork.java index 333e061c211..e28cd10056b 100644 --- a/common/src/main/java/bisq/common/crypto/ProofOfWork.java +++ b/common/src/main/java/bisq/common/crypto/ProofOfWork.java @@ -21,8 +21,6 @@ import com.google.protobuf.ByteString; -import java.math.BigInteger; - import lombok.EqualsAndHashCode; import lombok.Getter; @@ -34,8 +32,8 @@ public final class ProofOfWork implements NetworkPayload { private final long counter; @Getter private final byte[] challenge; - // We want to support BigInteger value for difficulty as well so we store it as byte array - private final byte[] difficulty; + @Getter + private final double difficulty; @Getter private final long duration; @Getter @@ -44,37 +42,9 @@ public final class ProofOfWork implements NetworkPayload { public ProofOfWork(byte[] payload, long counter, byte[] challenge, - int difficulty, + double difficulty, long duration, int version) { - this(payload, - counter, - challenge, - BigInteger.valueOf(difficulty).toByteArray(), - duration, - version); - } - - public ProofOfWork(byte[] payload, - long counter, - byte[] challenge, - BigInteger difficulty, - long duration, - int version) { - this(payload, - counter, - challenge, - difficulty.toByteArray(), - duration, - version); - } - - private ProofOfWork(byte[] payload, - long counter, - byte[] challenge, - byte[] difficulty, - long duration, - int version) { this.payload = payload; this.counter = counter; this.challenge = challenge; @@ -83,10 +53,6 @@ private ProofOfWork(byte[] payload, this.version = version; } - public int getNumLeadingZeros() { - return new BigInteger(difficulty).intValue(); - } - /////////////////////////////////////////////////////////////////////////////////////////// // PROTO BUFFER @@ -98,7 +64,7 @@ public protobuf.ProofOfWork toProtoMessage() { .setPayload(ByteString.copyFrom(payload)) .setCounter(counter) .setChallenge(ByteString.copyFrom(challenge)) - .setDifficulty(ByteString.copyFrom(difficulty)) + .setDifficulty(difficulty) .setDuration(duration) .setVersion(version) .build(); @@ -109,7 +75,7 @@ public static ProofOfWork fromProto(protobuf.ProofOfWork proto) { proto.getPayload().toByteArray(), proto.getCounter(), proto.getChallenge().toByteArray(), - proto.getDifficulty().toByteArray(), + proto.getDifficulty(), proto.getDuration(), proto.getVersion() ); @@ -120,7 +86,7 @@ public static ProofOfWork fromProto(protobuf.ProofOfWork proto) { public String toString() { return "ProofOfWork{" + ",\r\n counter=" + counter + - ",\r\n numLeadingZeros=" + getNumLeadingZeros() + + ",\r\n difficulty=" + difficulty + ",\r\n duration=" + duration + ",\r\n version=" + version + "\r\n}"; diff --git a/common/src/main/java/bisq/common/crypto/ProofOfWorkService.java b/common/src/main/java/bisq/common/crypto/ProofOfWorkService.java index 419c6587efa..6977b5b6a23 100644 --- a/common/src/main/java/bisq/common/crypto/ProofOfWorkService.java +++ b/common/src/main/java/bisq/common/crypto/ProofOfWorkService.java @@ -45,28 +45,28 @@ public static Optional forVersion(int version) { this.version = version; } - public abstract CompletableFuture mint(String itemId, byte[] challenge, int log2Difficulty); + public abstract CompletableFuture mint(String itemId, byte[] challenge, double difficulty); public abstract byte[] getChallenge(String itemId, String ownerId); abstract boolean verify(ProofOfWork proofOfWork); - public CompletableFuture mint(String itemId, String ownerId, int log2Difficulty) { - return mint(itemId, getChallenge(itemId, ownerId), log2Difficulty); + public CompletableFuture mint(String itemId, String ownerId, double difficulty) { + return mint(itemId, getChallenge(itemId, ownerId), difficulty); } public boolean verify(ProofOfWork proofOfWork, String itemId, String ownerId, - int controlLog2Difficulty, + double controlDifficulty, BiPredicate challengeValidation, - BiPredicate difficultyValidation) { + BiPredicate difficultyValidation) { Preconditions.checkArgument(proofOfWork.getVersion() == version); byte[] controlChallenge = getChallenge(itemId, ownerId); return challengeValidation.test(proofOfWork.getChallenge(), controlChallenge) && - difficultyValidation.test(proofOfWork.getNumLeadingZeros(), controlLog2Difficulty) && + difficultyValidation.test(proofOfWork.getDifficulty(), controlDifficulty) && verify(proofOfWork); } } diff --git a/common/src/test/java/bisq/common/crypto/HashCashServiceTest.java b/common/src/test/java/bisq/common/crypto/HashCashServiceTest.java index 9c4fcb1f53c..30c4fe98dcb 100644 --- a/common/src/test/java/bisq/common/crypto/HashCashServiceTest.java +++ b/common/src/test/java/bisq/common/crypto/HashCashServiceTest.java @@ -32,6 +32,17 @@ public void testNumberOfLeadingZeros() { assertEquals(9, HashCashService.numberOfLeadingZeros(new byte[]{Byte.parseByte("00000000", 2), Byte.parseByte("01010000", 2)})); } + @Test + public void testToNumLeadingZeros() { + assertEquals(0, HashCashService.toNumLeadingZeros(-1.0)); + assertEquals(0, HashCashService.toNumLeadingZeros(0.0)); + assertEquals(0, HashCashService.toNumLeadingZeros(1.0)); + assertEquals(1, HashCashService.toNumLeadingZeros(1.1)); + assertEquals(1, HashCashService.toNumLeadingZeros(2.0)); + assertEquals(8, HashCashService.toNumLeadingZeros(256.0)); + assertEquals(1024, HashCashService.toNumLeadingZeros(Double.POSITIVE_INFINITY)); + } + // @Ignore @Test public void testDiffIncrease() throws ExecutionException, InterruptedException { @@ -58,7 +69,8 @@ public void testDiffIncrease() throws ExecutionException, InterruptedException { //Minting 1000 tokens with 13 leading zeros took 25.276 ms per token and 16786 iterations in average. Verification took 0.002 ms per token. } - private void run(int difficulty, StringBuilder stringBuilder) throws ExecutionException, InterruptedException { + private void run(int log2Difficulty, StringBuilder stringBuilder) throws ExecutionException, InterruptedException { + double difficulty = Math.scalb(1.0, log2Difficulty); int numTokens = 1000; byte[] payload = RandomStringUtils.random(50, true, true).getBytes(StandardCharsets.UTF_8); long ts = System.currentTimeMillis(); @@ -75,7 +87,7 @@ private void run(int difficulty, StringBuilder stringBuilder) throws ExecutionEx double time1 = (System.currentTimeMillis() - ts) / size; double time2 = (System.currentTimeMillis() - ts2) / size; stringBuilder.append("\nMinting ").append(numTokens) - .append(" tokens with ").append(difficulty) + .append(" tokens with > ").append(log2Difficulty) .append(" leading zeros took ").append(time1) .append(" ms per token and ").append(averageCounter) .append(" iterations in average. Verification took ").append(time2) diff --git a/core/src/main/java/bisq/core/filter/Filter.java b/core/src/main/java/bisq/core/filter/Filter.java index 8ed55006c5d..70f088eb9b7 100644 --- a/core/src/main/java/bisq/core/filter/Filter.java +++ b/core/src/main/java/bisq/core/filter/Filter.java @@ -105,9 +105,10 @@ public final class Filter implements ProtectedStoragePayload, ExpirablePayload { // added at BsqSwap release private final boolean disablePowMessage; - // Number of leading zeros for pow for BSQ swap offers. Difficulty of 8 requires 0.856 ms in average, 15 about 100 ms. - // See ProofOfWorkTest for more info. - private final int powDifficulty; + // 2 ** effective-number-of-leading-zeros for pow for BSQ swap offers, when using Hashcash (= version 0), and + // a similar difficulty for Equihash (= versions 1) or later schemes. Difficulty of 2 ** 8 (= 256) requires + // 0.856 ms in average, 2 ** 15 (= 32768) about 100 ms. See HashCashServiceTest for more info. + private final double powDifficulty; // Enabled PoW version numbers in reverse order of preference, starting with 0 for Hashcash. private final List enabledPowVersions; @@ -222,7 +223,7 @@ public Filter(List bannedOfferIds, boolean disableMempoolValidation, boolean disableApi, boolean disablePowMessage, - int powDifficulty, + double powDifficulty, List enabledPowVersions, long makerFeeBtc, long takerFeeBtc, @@ -300,7 +301,7 @@ public Filter(List bannedOfferIds, boolean disableMempoolValidation, boolean disableApi, boolean disablePowMessage, - int powDifficulty, + double powDifficulty, List enabledPowVersions, long makerFeeBtc, long takerFeeBtc, diff --git a/core/src/main/java/bisq/core/filter/FilterManager.java b/core/src/main/java/bisq/core/filter/FilterManager.java index 4c0b74f44f8..73db4d36353 100644 --- a/core/src/main/java/bisq/core/filter/FilterManager.java +++ b/core/src/main/java/bisq/core/filter/FilterManager.java @@ -90,7 +90,7 @@ public class FilterManager { private final BiPredicate challengeValidation = Arrays::equals; // We only require a new pow if difficulty has increased - private final BiPredicate difficultyValidation = + private final BiPredicate difficultyValidation = (value, controlValue) -> value - controlValue >= 0; diff --git a/core/src/main/java/bisq/core/offer/bsq_swap/OpenBsqSwapOfferService.java b/core/src/main/java/bisq/core/offer/bsq_swap/OpenBsqSwapOfferService.java index 962983b8209..70735a57064 100644 --- a/core/src/main/java/bisq/core/offer/bsq_swap/OpenBsqSwapOfferService.java +++ b/core/src/main/java/bisq/core/offer/bsq_swap/OpenBsqSwapOfferService.java @@ -199,8 +199,8 @@ public void requestNewOffer(String offerId, NodeAddress makerAddress = Objects.requireNonNull(p2PService.getAddress()); offerUtil.validateBasicOfferData(PaymentMethod.BSQ_SWAP, "BSQ"); - int log2Difficulty = getPowDifficulty(); - getPowService().mint(offerId, makerAddress.getFullAddress(), log2Difficulty) + double difficulty = getPowDifficulty(); + getPowService().mint(offerId, makerAddress.getFullAddress(), difficulty) .whenComplete((proofOfWork, throwable) -> { // We got called from a non user thread... UserThread.execute(() -> { @@ -347,8 +347,8 @@ private void redoProofOfWorkAndRepublish(OpenOffer openOffer) { String newOfferId = OfferUtil.getOfferIdWithMutationCounter(openOffer.getId()); NodeAddress nodeAddress = Objects.requireNonNull(openOffer.getOffer().getMakerNodeAddress()); - int log2Difficulty = getPowDifficulty(); - getPowService().mint(newOfferId, nodeAddress.getFullAddress(), log2Difficulty) + double difficulty = getPowDifficulty(); + getPowService().mint(newOfferId, nodeAddress.getFullAddress(), difficulty) .whenComplete((proofOfWork, throwable) -> { // We got called from a non user thread... UserThread.execute(() -> { @@ -387,8 +387,8 @@ private boolean isProofOfWorkInvalid(Offer offer) { } - private int getPowDifficulty() { - return filterManager.getFilter() != null ? filterManager.getFilter().getPowDifficulty() : 0; + private double getPowDifficulty() { + return filterManager.getFilter() != null ? filterManager.getFilter().getPowDifficulty() : 0.0; } private ProofOfWorkService getPowService() { diff --git a/desktop/src/main/java/bisq/desktop/main/overlays/windows/FilterWindow.java b/desktop/src/main/java/bisq/desktop/main/overlays/windows/FilterWindow.java index e88d303e70b..658e84acd33 100644 --- a/desktop/src/main/java/bisq/desktop/main/overlays/windows/FilterWindow.java +++ b/desktop/src/main/java/bisq/desktop/main/overlays/windows/FilterWindow.java @@ -187,7 +187,7 @@ private void addContent() { Res.get("filterWindow.disablePowMessage")); InputTextField powDifficultyTF = addInputTextField(gridPane, ++rowIndex, Res.get("filterWindow.powDifficulty")); - powDifficultyTF.setText("0"); + powDifficultyTF.setText("0.0"); InputTextField enabledPowVersionsTF = addInputTextField(gridPane, ++rowIndex, Res.get("filterWindow.enabledPowVersions")); InputTextField makerFeeBtcTF = addInputTextField(gridPane, ++rowIndex, @@ -270,7 +270,7 @@ private void addContent() { disableMempoolValidationCheckBox.isSelected(), disableApiCheckBox.isSelected(), disablePowMessage.isSelected(), - Integer.parseInt(powDifficultyTF.getText()), + Double.parseDouble(powDifficultyTF.getText()), readAsList(enabledPowVersionsTF).stream().map(Integer::parseInt).collect(Collectors.toList()), ParsingUtils.parseToCoin(makerFeeBtcTF.getText(), btcFormatter).value, ParsingUtils.parseToCoin(takerFeeBtcTF.getText(), btcFormatter).value, diff --git a/proto/src/main/proto/pb.proto b/proto/src/main/proto/pb.proto index 549319f6ba4..522c8714fca 100644 --- a/proto/src/main/proto/pb.proto +++ b/proto/src/main/proto/pb.proto @@ -769,7 +769,7 @@ message Filter { bool disable_api = 27; bool disable_mempool_validation = 28; bool disable_pow_message = 29; - int32 pow_difficulty = 30; + double pow_difficulty = 30; repeated int32 enabled_pow_versions = 35; int64 maker_fee_btc = 31; int64 taker_fee_btc = 32; @@ -882,7 +882,7 @@ message ProofOfWork { bytes payload = 1; int64 counter = 2; bytes challenge = 3; - bytes difficulty = 4; + double difficulty = 4; int64 duration = 5; int32 version = 6; } From e383edc38ad2df7d122585083c290c5d7b526eba Mon Sep 17 00:00:00 2001 From: Steven Barclay Date: Sat, 27 Nov 2021 05:59:41 +0000 Subject: [PATCH 12/15] Inline predicates to simplify HashCashService & FilterManager Remove all the 'challengeValidation', 'difficultyValidation' and 'testDifficulty' BiPredicate method params from 'HashCashService' & 'ProofOfWorkService', to simplify the API. These were originally included to aid testing, but turned out to be unnecessary. Patches committed on behalf of @chimp1984. --- .../bisq/common/crypto/HashCashService.java | 104 +++--------------- .../common/crypto/ProofOfWorkService.java | 10 +- .../common/crypto/HashCashServiceTest.java | 2 +- .../java/bisq/core/filter/FilterManager.java | 14 +-- 4 files changed, 21 insertions(+), 109 deletions(-) diff --git a/common/src/main/java/bisq/common/crypto/HashCashService.java b/common/src/main/java/bisq/common/crypto/HashCashService.java index 092957817dd..939c705d4e9 100644 --- a/common/src/main/java/bisq/common/crypto/HashCashService.java +++ b/common/src/main/java/bisq/common/crypto/HashCashService.java @@ -22,24 +22,18 @@ import java.nio.charset.StandardCharsets; -import java.util.Arrays; import java.util.concurrent.CompletableFuture; -import java.util.function.BiPredicate; import lombok.extern.slf4j.Slf4j; /** * HashCash implementation for proof of work - * It doubles required work by difficulty increase (adding one leading zero). + * It doubles required work by log2Difficulty increase (adding one leading zero). * * See https://www.hashcash.org/papers/hashcash.pdf */ @Slf4j public class HashCashService extends ProofOfWorkService { - // Default validations. Custom implementations might use tolerance. - private static final BiPredicate isChallengeValid = Arrays::equals; - private static final BiPredicate isDifficultyValid = Integer::equals; - HashCashService() { super(0); } @@ -50,105 +44,35 @@ public CompletableFuture mint(String itemId, byte[] challenge, doub return mint(payload, challenge, difficulty); } - @Override - public byte[] getChallenge(String itemId, String ownerId) { - return getBytes(itemId + ownerId); - } - - static CompletableFuture mint(byte[] payload, + public CompletableFuture mint(byte[] payload, byte[] challenge, double difficulty) { - return HashCashService.mint(payload, - challenge, - difficulty, - HashCashService::testDifficulty); - } - - @Override - boolean verify(ProofOfWork proofOfWork) { - return verify(proofOfWork, - proofOfWork.getChallenge(), - toNumLeadingZeros(proofOfWork.getDifficulty())); - } - - static boolean verify(ProofOfWork proofOfWork, - byte[] controlChallenge, - int controlLog2Difficulty) { - return HashCashService.verify(proofOfWork, - controlChallenge, - controlLog2Difficulty, - HashCashService::testDifficulty); - } - - static boolean verify(ProofOfWork proofOfWork, - byte[] controlChallenge, - int controlLog2Difficulty, - BiPredicate challengeValidation, - BiPredicate difficultyValidation) { - return HashCashService.verify(proofOfWork, - controlChallenge, - controlLog2Difficulty, - challengeValidation, - difficultyValidation, - HashCashService::testDifficulty); - } - - private static boolean testDifficulty(byte[] result, int log2Difficulty) { - return HashCashService.numberOfLeadingZeros(result) > log2Difficulty; - } - - - /////////////////////////////////////////////////////////////////////////////////////////// - // Generic - /////////////////////////////////////////////////////////////////////////////////////////// - - static CompletableFuture mint(byte[] payload, - byte[] challenge, - double difficulty, - BiPredicate testDifficulty) { return CompletableFuture.supplyAsync(() -> { long ts = System.currentTimeMillis(); int log2Difficulty = toNumLeadingZeros(difficulty); - byte[] result; + byte[] hash; long counter = 0; do { - result = toSha256Hash(payload, challenge, ++counter); + hash = toSha256Hash(payload, challenge, ++counter); } - while (!testDifficulty.test(result, log2Difficulty)); + while (numberOfLeadingZeros(hash) <= log2Difficulty); ProofOfWork proofOfWork = new ProofOfWork(payload, counter, challenge, difficulty, System.currentTimeMillis() - ts, 0); log.info("Completed minting proofOfWork: {}", proofOfWork); return proofOfWork; }); } - static boolean verify(ProofOfWork proofOfWork, - byte[] controlChallenge, - int controlLog2Difficulty, - BiPredicate testDifficulty) { - return verify(proofOfWork, - controlChallenge, - controlLog2Difficulty, - HashCashService.isChallengeValid, - HashCashService.isDifficultyValid, - testDifficulty); - } - - static boolean verify(ProofOfWork proofOfWork, - byte[] controlChallenge, - int controlLog2Difficulty, - BiPredicate challengeValidation, - BiPredicate difficultyValidation, - BiPredicate testDifficulty) { - return challengeValidation.test(proofOfWork.getChallenge(), controlChallenge) && - difficultyValidation.test(toNumLeadingZeros(proofOfWork.getDifficulty()), controlLog2Difficulty) && - verify(proofOfWork, testDifficulty); - } - - private static boolean verify(ProofOfWork proofOfWork, BiPredicate testDifficulty) { - byte[] hash = HashCashService.toSha256Hash(proofOfWork.getPayload(), + @Override + boolean verify(ProofOfWork proofOfWork) { + byte[] hash = toSha256Hash(proofOfWork.getPayload(), proofOfWork.getChallenge(), proofOfWork.getCounter()); - return testDifficulty.test(hash, toNumLeadingZeros(proofOfWork.getDifficulty())); + return numberOfLeadingZeros(hash) > toNumLeadingZeros(proofOfWork.getDifficulty()); + } + + @Override + public byte[] getChallenge(String itemId, String ownerId) { + return getBytes(itemId + ownerId); } diff --git a/common/src/main/java/bisq/common/crypto/ProofOfWorkService.java b/common/src/main/java/bisq/common/crypto/ProofOfWorkService.java index 6977b5b6a23..4ad02d35616 100644 --- a/common/src/main/java/bisq/common/crypto/ProofOfWorkService.java +++ b/common/src/main/java/bisq/common/crypto/ProofOfWorkService.java @@ -19,9 +19,9 @@ import com.google.common.base.Preconditions; +import java.util.Arrays; import java.util.Optional; import java.util.concurrent.CompletableFuture; -import java.util.function.BiPredicate; import lombok.Getter; @@ -58,15 +58,13 @@ public CompletableFuture mint(String itemId, String ownerId, double public boolean verify(ProofOfWork proofOfWork, String itemId, String ownerId, - double controlDifficulty, - BiPredicate challengeValidation, - BiPredicate difficultyValidation) { + double controlDifficulty) { Preconditions.checkArgument(proofOfWork.getVersion() == version); byte[] controlChallenge = getChallenge(itemId, ownerId); - return challengeValidation.test(proofOfWork.getChallenge(), controlChallenge) && - difficultyValidation.test(proofOfWork.getDifficulty(), controlDifficulty) && + return Arrays.equals(proofOfWork.getChallenge(), controlChallenge) && + proofOfWork.getDifficulty() >= controlDifficulty && verify(proofOfWork); } } diff --git a/common/src/test/java/bisq/common/crypto/HashCashServiceTest.java b/common/src/test/java/bisq/common/crypto/HashCashServiceTest.java index 30c4fe98dcb..22eb5d1e283 100644 --- a/common/src/test/java/bisq/common/crypto/HashCashServiceTest.java +++ b/common/src/test/java/bisq/common/crypto/HashCashServiceTest.java @@ -77,7 +77,7 @@ private void run(int log2Difficulty, StringBuilder stringBuilder) throws Executi List tokens = new ArrayList<>(); for (int i = 0; i < numTokens; i++) { byte[] challenge = UUID.randomUUID().toString().getBytes(StandardCharsets.UTF_8); - tokens.add(HashCashService.mint(payload, challenge, difficulty).get()); + tokens.add(new HashCashService().mint(payload, challenge, difficulty).get()); } double size = tokens.size(); long ts2 = System.currentTimeMillis(); diff --git a/core/src/main/java/bisq/core/filter/FilterManager.java b/core/src/main/java/bisq/core/filter/FilterManager.java index 73db4d36353..cacfa9e8063 100644 --- a/core/src/main/java/bisq/core/filter/FilterManager.java +++ b/core/src/main/java/bisq/core/filter/FilterManager.java @@ -37,9 +37,9 @@ import bisq.common.app.Version; import bisq.common.config.Config; import bisq.common.config.ConfigFileEditor; -import bisq.common.crypto.ProofOfWorkService; import bisq.common.crypto.KeyRing; import bisq.common.crypto.ProofOfWork; +import bisq.common.crypto.ProofOfWorkService; import org.bitcoinj.core.ECKey; import org.bitcoinj.core.Sha256Hash; @@ -58,7 +58,6 @@ import java.math.BigInteger; -import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.Date; @@ -66,7 +65,6 @@ import java.util.List; import java.util.Set; import java.util.concurrent.CopyOnWriteArrayList; -import java.util.function.BiPredicate; import java.util.function.Consumer; import java.lang.reflect.Method; @@ -88,12 +86,6 @@ public class FilterManager { private static final String BANNED_SEED_NODES = "bannedSeedNodes"; private static final String BANNED_BTC_NODES = "bannedBtcNodes"; - private final BiPredicate challengeValidation = Arrays::equals; - // We only require a new pow if difficulty has increased - private final BiPredicate difficultyValidation = - (value, controlValue) -> value - controlValue >= 0; - - /////////////////////////////////////////////////////////////////////////////////////////// // Listener /////////////////////////////////////////////////////////////////////////////////////////// @@ -501,9 +493,7 @@ public boolean isProofOfWorkValid(Offer offer) { } return service.get().verify(offer.getBsqSwapOfferPayload().get().getProofOfWork(), offer.getId(), offer.getOwnerNodeAddress().toString(), - filter.getPowDifficulty(), - challengeValidation, - difficultyValidation); + filter.getPowDifficulty()); } public List getEnabledPowVersions() { From 0ee6175fb2f908d90acd283b727216fc4d3de1f9 Mon Sep 17 00:00:00 2001 From: Steven Barclay Date: Tue, 30 Nov 2021 21:26:17 +0000 Subject: [PATCH 13/15] Fix bug in Equihash.IntListMultimap & adjust constants Fix a trivial bug in the iterator returned by 'IntListMultimap::get', caused by mistaken use of the iterator index in place of the key when doing lookups into the overspill map. This was causing puzzle solutions to be invalid about 3% of the time, as well as substantially reducing the average number of solutions found per nonce. As the fix increases the mean solution count per nonce to the correct value of 2.0 predicted by the paper (regardless of puzzle params k & n), inline the affected constants to simplify 'Equihash::adjustDifficulty'. --- .../java/bisq/common/crypto/Equihash.java | 12 ++-- .../crypto/EquihashProofOfWorkService.java | 3 +- .../java/bisq/common/crypto/EquihashTest.java | 56 +++++++++---------- 3 files changed, 34 insertions(+), 37 deletions(-) diff --git a/common/src/main/java/bisq/common/crypto/Equihash.java b/common/src/main/java/bisq/common/crypto/Equihash.java index 576f930b994..dc191617793 100644 --- a/common/src/main/java/bisq/common/crypto/Equihash.java +++ b/common/src/main/java/bisq/common/crypto/Equihash.java @@ -81,10 +81,8 @@ @SuppressWarnings("UnstableApiUsage") public class Equihash { private static final int HASH_BIT_LENGTH = 256; - /** Observed mean solution count per nonce for Equihash-n-4 puzzles with unit difficulty. */ - public static final double EQUIHASH_n_4_MEAN_SOLUTION_COUNT_PER_NONCE = 1.63; - /** Observed mean solution count per nonce for Equihash-n-5 puzzles with unit difficulty. */ - public static final double EQUIHASH_n_5_MEAN_SOLUTION_COUNT_PER_NONCE = 1.34; + /** Mean solution count per nonce for Equihash puzzles with unit difficulty. */ + private static final double MEAN_SOLUTION_COUNT_PER_NONCE = 2.0; private final int k, N; private final int tableCapacity; @@ -127,8 +125,8 @@ private static BigInteger inverseDifficultyMinusOne(double difficulty) { /** Adjust the provided difficulty to take the variable number of puzzle solutions per * nonce into account, so that the expected number of attempts needed to solve a given * puzzle equals the reciprocal of the provided difficulty. */ - public static double adjustDifficulty(double realDifficulty, double meanSolutionCountPerNonce) { - return Math.max(-meanSolutionCountPerNonce / Math.log1p(-1.0 / Math.max(realDifficulty, 1.0)), 1.0); + public static double adjustDifficulty(double realDifficulty) { + return Math.max(-MEAN_SOLUTION_COUNT_PER_NONCE / Math.log1p(-1.0 / Math.max(realDifficulty, 1.0)), 1.0); } public Puzzle puzzle(byte[] seed) { @@ -339,7 +337,7 @@ PrimitiveIterator.OfInt get(int key) { private Iterator overspillIterator() { if (overspillIterator == null) { - overspillIterator = overspillMultimap.get(i).iterator(); + overspillIterator = overspillMultimap.get(key).iterator(); } return overspillIterator; } diff --git a/common/src/main/java/bisq/common/crypto/EquihashProofOfWorkService.java b/common/src/main/java/bisq/common/crypto/EquihashProofOfWorkService.java index 70ad4f3e0b4..ac947418c26 100644 --- a/common/src/main/java/bisq/common/crypto/EquihashProofOfWorkService.java +++ b/common/src/main/java/bisq/common/crypto/EquihashProofOfWorkService.java @@ -70,7 +70,6 @@ boolean verify(ProofOfWork proofOfWork) { } private static double scaledDifficulty(double difficulty) { - return Equihash.adjustDifficulty(DIFFICULTY_SCALE_FACTOR * difficulty, - Equihash.EQUIHASH_n_5_MEAN_SOLUTION_COUNT_PER_NONCE); + return Equihash.adjustDifficulty(DIFFICULTY_SCALE_FACTOR * difficulty); } } diff --git a/common/src/test/java/bisq/common/crypto/EquihashTest.java b/common/src/test/java/bisq/common/crypto/EquihashTest.java index 60611568de4..16b3510ba23 100644 --- a/common/src/test/java/bisq/common/crypto/EquihashTest.java +++ b/common/src/test/java/bisq/common/crypto/EquihashTest.java @@ -33,7 +33,6 @@ import org.junit.Ignore; import org.junit.Test; -import static bisq.common.crypto.Equihash.EQUIHASH_n_5_MEAN_SOLUTION_COUNT_PER_NONCE; import static java.lang.Double.POSITIVE_INFINITY; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; @@ -56,16 +55,17 @@ public void testHashUpperBound() { @Test public void testAdjustDifficulty() { - assertEquals(1.0, Equihash.adjustDifficulty(0.0, 1.34), 0.0001); - assertEquals(1.0, Equihash.adjustDifficulty(0.5, 1.34), 0.0001); - assertEquals(1.0, Equihash.adjustDifficulty(1.0, 1.34), 0.0001); - assertEquals(1.0, Equihash.adjustDifficulty(1.2, 1.34), 0.0001); - assertEquals(1.22, Equihash.adjustDifficulty(1.5, 1.34), 0.01); - assertEquals(1.93, Equihash.adjustDifficulty(2.0, 1.34), 0.01); - assertEquals(2.62, Equihash.adjustDifficulty(2.5, 1.34), 0.01); - assertEquals(3.30, Equihash.adjustDifficulty(3.0, 1.34), 0.01); - assertEquals(134.0, Equihash.adjustDifficulty(100.0, 1.34), 1.0); - assertEquals(Equihash.adjustDifficulty(POSITIVE_INFINITY, 1.34), POSITIVE_INFINITY, 1.0); + assertEquals(1.0, Equihash.adjustDifficulty(0.0), 0.0001); + assertEquals(1.0, Equihash.adjustDifficulty(0.5), 0.0001); + assertEquals(1.0, Equihash.adjustDifficulty(1.0), 0.0001); + assertEquals(1.0, Equihash.adjustDifficulty(1.1), 0.0001); + assertEquals(1.12, Equihash.adjustDifficulty(1.2), 0.01); + assertEquals(1.83, Equihash.adjustDifficulty(1.5), 0.01); + assertEquals(2.89, Equihash.adjustDifficulty(2.0), 0.01); + assertEquals(3.92, Equihash.adjustDifficulty(2.5), 0.01); + assertEquals(4.93, Equihash.adjustDifficulty(3.0), 0.01); + assertEquals(200.0, Equihash.adjustDifficulty(100.0), 1.5); + assertEquals(Equihash.adjustDifficulty(POSITIVE_INFINITY), POSITIVE_INFINITY, 1.0); } @Test @@ -87,12 +87,12 @@ public void testFindSolution() { public void benchmarkFindSolution() { // On Intel Core i3 CPU M 330 @ 2.13GHz ... // - // For Equihash-90-5 with real difficulty 2.0, adjusted difficulty 1.933211354791211 ... - // Total elapsed solution time: 292789 ms - // Mean time to solve one puzzle: 292 ms - // Puzzle solution time per unit difficulty: 146 ms + // For Equihash-90-5 with real difficulty 2.0, adjusted difficulty 2.8853900817779268 ... + // Total elapsed solution time: 279583 ms + // Mean time to solve one puzzle: 279 ms + // Puzzle solution time per unit difficulty: 139 ms // - double adjustedDifficulty = Equihash.adjustDifficulty(2.0, EQUIHASH_n_5_MEAN_SOLUTION_COUNT_PER_NONCE); + double adjustedDifficulty = Equihash.adjustDifficulty(2.0); Equihash equihash = new Equihash(90, 5, adjustedDifficulty); Stopwatch stopwatch = Stopwatch.createStarted(); @@ -140,24 +140,24 @@ public void benchmarkVerify() { @Ignore public void solutionCountPerNonceStats() { // For Equihash-60-4... - // Got puzzle solution count mean: 1.6161 - // Got expected count stats: [0 x 1987, 1 x 3210, 2 x 2595, 3 x 1398, 4 x 564, 5 x 183, 6 x 49, 7 x 11, 8 x 3] - // Got actual count stats: [0 x 2014, 1 x 3230, 2 x 2546, 3 x 1395, 4 x 543, 5 x 191, 6 x 50, 7 x 24, 8 x 4, 9 x 3] + // Got puzzle solution count mean: 1.9797 + // Got expected count stats: [0 x 1381, 1 x 2734, 2 x 2707, 3 x 1786, 4 x 884, 5 x 350, 6 x 115, 7 x 33, 8 x 8, 9 x 2] + // Got actual count stats: [0 x 1413, 1 x 2704, 2 x 2699, 3 x 1813, 4 x 866, 5 x 330, 6 x 115, 7 x 35, 8 x 19, 9 x 5, 10] // // For Equihash-70-4... - // Got puzzle solution count mean: 1.6473 - // Got expected count stats: [0 x 1926, 1 x 3172, 2 x 2613, 3 x 1434, 4 x 591, 5 x 195, 6 x 53, 7 x 13, 8 x 2, 9] - // Got actual count stats: [0 x 1958, 1 x 3172, 2 x 2584, 3 x 1413, 4 x 585, 5 x 204, 6 x 61, 7 x 17, 8 x 5, 9] + // Got puzzle solution count mean: 1.9988 + // Got expected count stats: [0 x 1355, 1 x 2708, 2 x 2707, 3 x 1803, 4 x 902, 5 x 360, 6 x 120, 7 x 34, 8 x 9, 9 x 2] + // Got actual count stats: [0 x 1362, 1 x 2690, 2 x 2720, 3 x 1826, 4 x 870, 5 x 353, 6 x 129, 7 x 41, 8 x 7, 9 x 2] // // For Equihash-90-5... - // Got puzzle solution count mean: 1.3419 - // Got expected count stats: [0 x 2613, 1 x 3508, 2 x 2353, 3 x 1052, 4 x 353, 5 x 95, 6 x 21, 7 x 4, 8] - // Got actual count stats: [0 x 2698, 1 x 3446, 2 x 2311, 3 x 1045, 4 x 352, 5 x 104, 6 x 33, 7 x 5, 8 x 3, 9, 10, 12] + // Got puzzle solution count mean: 1.9921 + // Got expected count stats: [0 x 1364, 1 x 2717, 2 x 2707, 3 x 1797, 4 x 896, 5 x 356, 6 x 119, 7 x 33, 8 x 9, 9 x 2] + // Got actual count stats: [0 x 1379, 1 x 2709, 2 x 2729, 3 x 1750, 4 x 900, 5 x 362, 6 x 119, 7 x 39, 8 x 11, 9, 10] // // For Equihash-96-5... - // Got puzzle solution count mean: 1.3363 - // Got expected count stats: [0 x 2628, 1 x 3512, 2 x 2347, 3 x 1045, 4 x 349, 5 x 93, 6 x 21, 7 x 4, 8] - // Got actual count stats: [0 x 2708, 1 x 3409, 2 x 2344, 3 x 1048, 4 x 368, 5 x 94, 6 x 23, 7 x 6] + // Got puzzle solution count mean: 1.9997 + // Got expected count stats: [0 x 1354, 1 x 2707, 2 x 2707, 3 x 1804, 4 x 902, 5 x 360, 6 x 121, 7 x 34, 8 x 9, 9 x 2] + // Got actual count stats: [0 x 1405, 1 x 2621, 2 x 2733, 3 x 1802, 4 x 928, 5 x 342, 6 x 123, 7 x 29, 8 x 13, 9 x 3, 10] // Equihash equihash = new Equihash(90, 5, 1.0); byte[] seed = new byte[32]; From 24d2a7f2228c1e546ae3bd8adecac16c7750d2dc Mon Sep 17 00:00:00 2001 From: Steven Barclay Date: Mon, 6 Dec 2021 22:55:14 +0000 Subject: [PATCH 14/15] Code cleanup: proto fields, duplicated expr & null char separator 1. Reorder the PoW fields in the 'Filter' proto by field index, instead of contextually. 2. Deduplicate expression for 'pow' & replace if-block with boolean op to simplify 'FilterManager::isProofOfWorkValid'. 3. Avoid slightly confusing use of null char as a separator to prevent hashing collisions in 'EquihashProofOfWorkService::getChallenge'. Use comma separator and escape the 'itemId' & 'ownerId' arguments instead. (based on PR #5858 review comments) --- .../common/crypto/EquihashProofOfWorkService.java | 11 +++-------- .../src/main/java/bisq/core/filter/FilterManager.java | 11 +++-------- proto/src/main/proto/pb.proto | 8 ++++---- 3 files changed, 10 insertions(+), 20 deletions(-) diff --git a/common/src/main/java/bisq/common/crypto/EquihashProofOfWorkService.java b/common/src/main/java/bisq/common/crypto/EquihashProofOfWorkService.java index ac947418c26..e31d1d4c451 100644 --- a/common/src/main/java/bisq/common/crypto/EquihashProofOfWorkService.java +++ b/common/src/main/java/bisq/common/crypto/EquihashProofOfWorkService.java @@ -19,16 +19,11 @@ import com.google.common.primitives.Longs; -import org.apache.commons.lang3.StringUtils; - import java.util.Arrays; import java.util.concurrent.CompletableFuture; import lombok.extern.slf4j.Slf4j; -import static com.google.common.base.Preconditions.checkArgument; -import static com.google.common.base.Preconditions.checkNotNull; - @Slf4j public class EquihashProofOfWorkService extends ProofOfWorkService { /** Rough cost of two Hashcash iterations compared to solving an Equihash-90-5 puzzle of unit difficulty. */ @@ -56,9 +51,9 @@ public CompletableFuture mint(String itemId, byte[] challenge, doub @Override public byte[] getChallenge(String itemId, String ownerId) { - checkArgument(!StringUtils.contains(itemId, '\0')); - checkArgument(!StringUtils.contains(ownerId, '\0')); - return Hash.getSha256Hash(checkNotNull(itemId) + "\0" + checkNotNull(ownerId)); + String escapedItemId = itemId.replace(" ", " "); + String escapedOwnerId = ownerId.replace(" ", " "); + return Hash.getSha256Hash(escapedItemId + ", " + escapedOwnerId); } @Override diff --git a/core/src/main/java/bisq/core/filter/FilterManager.java b/core/src/main/java/bisq/core/filter/FilterManager.java index cacfa9e8063..2e57d702b29 100644 --- a/core/src/main/java/bisq/core/filter/FilterManager.java +++ b/core/src/main/java/bisq/core/filter/FilterManager.java @@ -484,16 +484,11 @@ public boolean isProofOfWorkValid(Offer offer) { if (filter == null) { return true; } - checkArgument(offer.getBsqSwapOfferPayload().isPresent(), - "Offer payload must be BsqSwapOfferPayload"); + checkArgument(offer.getBsqSwapOfferPayload().isPresent(), "Offer payload must be BsqSwapOfferPayload"); ProofOfWork pow = offer.getBsqSwapOfferPayload().get().getProofOfWork(); var service = ProofOfWorkService.forVersion(pow.getVersion()); - if (!service.isPresent() || !getEnabledPowVersions().contains(pow.getVersion())) { - return false; - } - return service.get().verify(offer.getBsqSwapOfferPayload().get().getProofOfWork(), - offer.getId(), offer.getOwnerNodeAddress().toString(), - filter.getPowDifficulty()); + return service.isPresent() && getEnabledPowVersions().contains(pow.getVersion()) && + service.get().verify(pow, offer.getId(), offer.getOwnerNodeAddress().toString(), filter.getPowDifficulty()); } public List getEnabledPowVersions() { diff --git a/proto/src/main/proto/pb.proto b/proto/src/main/proto/pb.proto index 522c8714fca..b5a04d28305 100644 --- a/proto/src/main/proto/pb.proto +++ b/proto/src/main/proto/pb.proto @@ -85,8 +85,8 @@ message NetworkEnvelope { ShareBuyerPaymentAccountMessage share_buyer_payment_account_message = 54; // Added at 1.7.0 SellersBsqSwapRequest sellers_bsq_swap_request = 55; - BuyersBsqSwapRequest buyers_bsq_swap_request= 56; - BsqSwapTxInputsMessage bsq_swap_tx_inputs_message= 57; + BuyersBsqSwapRequest buyers_bsq_swap_request = 56; + BsqSwapTxInputsMessage bsq_swap_tx_inputs_message = 57; BsqSwapFinalizeTxRequest bsq_swap_finalize_tx_request = 58; BsqSwapFinalizedTxMessage bsq_swap_finalized_tx_message = 59; } @@ -770,11 +770,11 @@ message Filter { bool disable_mempool_validation = 28; bool disable_pow_message = 29; double pow_difficulty = 30; - repeated int32 enabled_pow_versions = 35; int64 maker_fee_btc = 31; int64 taker_fee_btc = 32; int64 maker_fee_bsq = 33; int64 taker_fee_bsq = 34; + repeated int32 enabled_pow_versions = 35; } // Deprecated @@ -2038,7 +2038,7 @@ message BaseBlock { } message BsqBlockStore { - repeated BaseBlock blocks = 1; + repeated BaseBlock blocks = 1; } message RawBlock { From db6025a08dcbe267de4da93e3635ad3ab65ea10a Mon Sep 17 00:00:00 2001 From: Steven Barclay Date: Tue, 7 Dec 2021 02:22:33 +0000 Subject: [PATCH 15/15] Add separate ProofOfWork.solution proto field for Equihash Avoid repurposing the 'ProofOfWork.payload' field for Equihash puzzle solutions, as that may be of later use in interactive PoW schemes such as P2P network DoS protection (where the challenge may be a random nonce instead of derived from the offer ID). Instead, make the payload the UTF-8 bytes of the offer ID, just as with Hashcash. Also, make the puzzle seed the SHA-256 hash of the payload concatenated with the challenge, instead of just the 256-bit challenge on its own, so that the PoW is tied to a particular payload and cannot be reused for other payloads in the case of future randomly chosen challenges. --- .../crypto/EquihashProofOfWorkService.java | 19 +++++++++++++------ .../bisq/common/crypto/HashCashService.java | 9 +++------ .../java/bisq/common/crypto/ProofOfWork.java | 6 ++++++ .../common/crypto/ProofOfWorkService.java | 14 ++++++++++---- proto/src/main/proto/pb.proto | 3 ++- 5 files changed, 34 insertions(+), 17 deletions(-) diff --git a/common/src/main/java/bisq/common/crypto/EquihashProofOfWorkService.java b/common/src/main/java/bisq/common/crypto/EquihashProofOfWorkService.java index e31d1d4c451..56aad6aa47f 100644 --- a/common/src/main/java/bisq/common/crypto/EquihashProofOfWorkService.java +++ b/common/src/main/java/bisq/common/crypto/EquihashProofOfWorkService.java @@ -17,6 +17,7 @@ package bisq.common.crypto; +import com.google.common.primitives.Bytes; import com.google.common.primitives.Longs; import java.util.Arrays; @@ -34,21 +35,26 @@ public class EquihashProofOfWorkService extends ProofOfWorkService { } @Override - public CompletableFuture mint(String itemId, byte[] challenge, double difficulty) { + public CompletableFuture mint(byte[] payload, byte[] challenge, double difficulty) { double scaledDifficulty = scaledDifficulty(difficulty); log.info("Got scaled & adjusted difficulty: {}", scaledDifficulty); return CompletableFuture.supplyAsync(() -> { long ts = System.currentTimeMillis(); - byte[] solution = new Equihash(90, 5, scaledDifficulty).puzzle(challenge).findSolution().serialize(); + byte[] seed = getSeed(payload, challenge); + byte[] solution = new Equihash(90, 5, scaledDifficulty).puzzle(seed).findSolution().serialize(); long counter = Longs.fromByteArray(Arrays.copyOf(solution, 8)); - var proofOfWork = new ProofOfWork(solution, counter, challenge, difficulty, - System.currentTimeMillis() - ts, getVersion()); + var proofOfWork = new ProofOfWork(payload, counter, challenge, difficulty, + System.currentTimeMillis() - ts, solution, getVersion()); log.info("Completed minting proofOfWork: {}", proofOfWork); return proofOfWork; }); } + private byte[] getSeed(byte[] payload, byte[] challenge) { + return Hash.getSha256Hash(Bytes.concat(payload, challenge)); + } + @Override public byte[] getChallenge(String itemId, String ownerId) { String escapedItemId = itemId.replace(" ", " "); @@ -60,8 +66,9 @@ public byte[] getChallenge(String itemId, String ownerId) { boolean verify(ProofOfWork proofOfWork) { double scaledDifficulty = scaledDifficulty(proofOfWork.getDifficulty()); - var puzzle = new Equihash(90, 5, scaledDifficulty).puzzle(proofOfWork.getChallenge()); - return puzzle.deserializeSolution(proofOfWork.getPayload()).verify(); + byte[] seed = getSeed(proofOfWork.getPayload(), proofOfWork.getChallenge()); + var puzzle = new Equihash(90, 5, scaledDifficulty).puzzle(seed); + return puzzle.deserializeSolution(proofOfWork.getSolution()).verify(); } private static double scaledDifficulty(double difficulty) { diff --git a/common/src/main/java/bisq/common/crypto/HashCashService.java b/common/src/main/java/bisq/common/crypto/HashCashService.java index 939c705d4e9..a894f37fb71 100644 --- a/common/src/main/java/bisq/common/crypto/HashCashService.java +++ b/common/src/main/java/bisq/common/crypto/HashCashService.java @@ -39,11 +39,6 @@ public class HashCashService extends ProofOfWorkService { } @Override - public CompletableFuture mint(String itemId, byte[] challenge, double difficulty) { - byte[] payload = getBytes(itemId); - return mint(payload, challenge, difficulty); - } - public CompletableFuture mint(byte[] payload, byte[] challenge, double difficulty) { @@ -56,7 +51,9 @@ public CompletableFuture mint(byte[] payload, hash = toSha256Hash(payload, challenge, ++counter); } while (numberOfLeadingZeros(hash) <= log2Difficulty); - ProofOfWork proofOfWork = new ProofOfWork(payload, counter, challenge, difficulty, System.currentTimeMillis() - ts, 0); + byte[] solution = Longs.toByteArray(counter); + ProofOfWork proofOfWork = new ProofOfWork(payload, counter, challenge, difficulty, + System.currentTimeMillis() - ts, solution, 0); log.info("Completed minting proofOfWork: {}", proofOfWork); return proofOfWork; }); diff --git a/common/src/main/java/bisq/common/crypto/ProofOfWork.java b/common/src/main/java/bisq/common/crypto/ProofOfWork.java index e28cd10056b..3fd1ab685b7 100644 --- a/common/src/main/java/bisq/common/crypto/ProofOfWork.java +++ b/common/src/main/java/bisq/common/crypto/ProofOfWork.java @@ -37,6 +37,8 @@ public final class ProofOfWork implements NetworkPayload { @Getter private final long duration; @Getter + private final byte[] solution; + @Getter private final int version; public ProofOfWork(byte[] payload, @@ -44,12 +46,14 @@ public ProofOfWork(byte[] payload, byte[] challenge, double difficulty, long duration, + byte[] solution, int version) { this.payload = payload; this.counter = counter; this.challenge = challenge; this.difficulty = difficulty; this.duration = duration; + this.solution = solution; this.version = version; } @@ -66,6 +70,7 @@ public protobuf.ProofOfWork toProtoMessage() { .setChallenge(ByteString.copyFrom(challenge)) .setDifficulty(difficulty) .setDuration(duration) + .setSolution(ByteString.copyFrom(solution)) .setVersion(version) .build(); } @@ -77,6 +82,7 @@ public static ProofOfWork fromProto(protobuf.ProofOfWork proto) { proto.getChallenge().toByteArray(), proto.getDifficulty(), proto.getDuration(), + proto.getSolution().toByteArray(), proto.getVersion() ); } diff --git a/common/src/main/java/bisq/common/crypto/ProofOfWorkService.java b/common/src/main/java/bisq/common/crypto/ProofOfWorkService.java index 4ad02d35616..3042f5b048f 100644 --- a/common/src/main/java/bisq/common/crypto/ProofOfWorkService.java +++ b/common/src/main/java/bisq/common/crypto/ProofOfWorkService.java @@ -19,6 +19,8 @@ import com.google.common.base.Preconditions; +import java.nio.charset.StandardCharsets; + import java.util.Arrays; import java.util.Optional; import java.util.concurrent.CompletableFuture; @@ -45,14 +47,18 @@ public static Optional forVersion(int version) { this.version = version; } - public abstract CompletableFuture mint(String itemId, byte[] challenge, double difficulty); - - public abstract byte[] getChallenge(String itemId, String ownerId); + public abstract CompletableFuture mint(byte[] payload, byte[] challenge, double difficulty); abstract boolean verify(ProofOfWork proofOfWork); + public byte[] getPayload(String itemId) { + return itemId.getBytes(StandardCharsets.UTF_8); + } + + public abstract byte[] getChallenge(String itemId, String ownerId); + public CompletableFuture mint(String itemId, String ownerId, double difficulty) { - return mint(itemId, getChallenge(itemId, ownerId), difficulty); + return mint(getPayload(itemId), getChallenge(itemId, ownerId), difficulty); } public boolean verify(ProofOfWork proofOfWork, diff --git a/proto/src/main/proto/pb.proto b/proto/src/main/proto/pb.proto index b5a04d28305..7f9b4d435fc 100644 --- a/proto/src/main/proto/pb.proto +++ b/proto/src/main/proto/pb.proto @@ -884,7 +884,8 @@ message ProofOfWork { bytes challenge = 3; double difficulty = 4; int64 duration = 5; - int32 version = 6; + bytes solution = 6; + int32 version = 7; } message AccountAgeWitness {