From 9404e8fa87cc51073823aa2207c23134c2c1d876 Mon Sep 17 00:00:00 2001 From: chimp1984 Date: Wed, 7 Oct 2020 09:43:39 -0500 Subject: [PATCH 01/59] Add NO_ADDRESS_PRE_FIX capability. Add `boolean contains(Capability capability)` method --- common/src/main/java/bisq/common/app/Capabilities.java | 4 ++++ common/src/main/java/bisq/common/app/Capability.java | 3 ++- .../main/java/bisq/core/setup/CoreNetworkCapabilities.java | 3 ++- 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/common/src/main/java/bisq/common/app/Capabilities.java b/common/src/main/java/bisq/common/app/Capabilities.java index 25913c9b0a7..bf95c15c48d 100644 --- a/common/src/main/java/bisq/common/app/Capabilities.java +++ b/common/src/main/java/bisq/common/app/Capabilities.java @@ -98,6 +98,10 @@ public boolean containsAll(Capability... capabilities) { return this.capabilities.containsAll(Arrays.asList(capabilities)); } + public boolean contains(Capability capability) { + return this.capabilities.contains(capability); + } + public boolean isEmpty() { return capabilities.isEmpty(); } diff --git a/common/src/main/java/bisq/common/app/Capability.java b/common/src/main/java/bisq/common/app/Capability.java index 33040bbfa40..9379c9dd392 100644 --- a/common/src/main/java/bisq/common/app/Capability.java +++ b/common/src/main/java/bisq/common/app/Capability.java @@ -40,5 +40,6 @@ public enum Capability { SIGNED_ACCOUNT_AGE_WITNESS, // Supports the signed account age witness feature MEDIATION, // Supports mediation feature REFUND_AGENT, // Supports refund agents - TRADE_STATISTICS_HASH_UPDATE // We changed the hash method in 1.2.0 and that requires update to 1.2.2 for handling it correctly, otherwise the seed nodes have to process too much data. + TRADE_STATISTICS_HASH_UPDATE, // We changed the hash method in 1.2.0 and that requires update to 1.2.2 for handling it correctly, otherwise the seed nodes have to process too much data. + NO_ADDRESS_PRE_FIX // At 1.4.0 we removed the prefix filter for mailbox messages. If a peer has that capability we do not sent the prefix. } diff --git a/core/src/main/java/bisq/core/setup/CoreNetworkCapabilities.java b/core/src/main/java/bisq/core/setup/CoreNetworkCapabilities.java index fcd03280337..fd951d98592 100644 --- a/core/src/main/java/bisq/core/setup/CoreNetworkCapabilities.java +++ b/core/src/main/java/bisq/core/setup/CoreNetworkCapabilities.java @@ -39,7 +39,8 @@ static void setSupportedCapabilities(Config config) { Capability.MEDIATION, Capability.SIGNED_ACCOUNT_AGE_WITNESS, Capability.REFUND_AGENT, - Capability.TRADE_STATISTICS_HASH_UPDATE + Capability.TRADE_STATISTICS_HASH_UPDATE, + Capability.NO_ADDRESS_PRE_FIX ); if (config.daoActivated) { From e73a4b4ae44529816407838c70ee79ea704dd138 Mon Sep 17 00:00:00 2001 From: chimp1984 Date: Wed, 7 Oct 2020 09:44:13 -0500 Subject: [PATCH 02/59] Cleanups Remove debug log, remove annotation --- .../bisq/network/p2p/DecryptedDirectMessageListener.java | 2 +- .../main/java/bisq/network/p2p/storage/P2PDataStorage.java | 6 +----- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/p2p/src/main/java/bisq/network/p2p/DecryptedDirectMessageListener.java b/p2p/src/main/java/bisq/network/p2p/DecryptedDirectMessageListener.java index c3eb61e7b41..2747bcedb27 100644 --- a/p2p/src/main/java/bisq/network/p2p/DecryptedDirectMessageListener.java +++ b/p2p/src/main/java/bisq/network/p2p/DecryptedDirectMessageListener.java @@ -19,5 +19,5 @@ public interface DecryptedDirectMessageListener { - void onDirectMessage(DecryptedMessageWithPubKey decryptedMessageWithPubKey, @SuppressWarnings("UnusedParameters") NodeAddress peerNodeAddress); + void onDirectMessage(DecryptedMessageWithPubKey decryptedMessageWithPubKey, NodeAddress peerNodeAddress); } diff --git a/p2p/src/main/java/bisq/network/p2p/storage/P2PDataStorage.java b/p2p/src/main/java/bisq/network/p2p/storage/P2PDataStorage.java index f412e5146a4..7b9e3714cea 100644 --- a/p2p/src/main/java/bisq/network/p2p/storage/P2PDataStorage.java +++ b/p2p/src/main/java/bisq/network/p2p/storage/P2PDataStorage.java @@ -637,12 +637,8 @@ private boolean addProtectedStorageEntry(ProtectedStorageEntry protectedStorageE // To avoid that expired data get stored and broadcast we check early for expire date. if (protectedStorageEntry.isExpired(clock)) { String peer = sender != null ? sender.getFullAddress() : "sender is null"; - log.warn("We received an expired protectedStorageEntry from peer {}. ProtectedStoragePayload={}", + log.debug("We received an expired protectedStorageEntry from peer {}. ProtectedStoragePayload={}", peer, protectedStorageEntry.getProtectedStoragePayload().getClass().getSimpleName()); - log.debug("Expired protectedStorageEntry from peer {}. getCreationTimeStamp={}, protectedStorageEntry={}", - peer, - new Date(protectedStorageEntry.getCreationTimeStamp()), - protectedStorageEntry); return false; } From 6f7dfcf4efcdbdd42d2ac56fa1618a4df2b2bed9 Mon Sep 17 00:00:00 2001 From: chimp1984 Date: Wed, 7 Oct 2020 11:02:20 -0500 Subject: [PATCH 03/59] Make onRemoved default in interface --- .../bisq/network/p2p/storage/HashMapChangedListener.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/p2p/src/main/java/bisq/network/p2p/storage/HashMapChangedListener.java b/p2p/src/main/java/bisq/network/p2p/storage/HashMapChangedListener.java index ce483889703..b225b25f553 100644 --- a/p2p/src/main/java/bisq/network/p2p/storage/HashMapChangedListener.java +++ b/p2p/src/main/java/bisq/network/p2p/storage/HashMapChangedListener.java @@ -24,6 +24,7 @@ public interface HashMapChangedListener { void onAdded(Collection protectedStorageEntries); - @SuppressWarnings("UnusedParameters") - void onRemoved(Collection protectedStorageEntries); + default void onRemoved(Collection protectedStorageEntries) { + // Often we are only interested in added data as there is no use case for remove + } } From 40f9cfb7c59a448067843717afdb44224edfdadb Mon Sep 17 00:00:00 2001 From: chimp1984 Date: Wed, 7 Oct 2020 11:49:47 -0500 Subject: [PATCH 04/59] Add methods for getting peers capabilities --- .../java/bisq/network/p2p/peers/PeerManager.java | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/p2p/src/main/java/bisq/network/p2p/peers/PeerManager.java b/p2p/src/main/java/bisq/network/p2p/peers/PeerManager.java index e8cf818336d..f12c95b9fd7 100644 --- a/p2p/src/main/java/bisq/network/p2p/peers/PeerManager.java +++ b/p2p/src/main/java/bisq/network/p2p/peers/PeerManager.java @@ -32,6 +32,7 @@ import bisq.common.Timer; import bisq.common.UserThread; import bisq.common.app.Capabilities; +import bisq.common.app.Capability; import bisq.common.config.Config; import bisq.common.persistence.PersistenceManager; import bisq.common.proto.persistable.PersistedDataHost; @@ -206,6 +207,21 @@ private void setConnectionLimits(int maxConnections) { } + public boolean peerHasCapability(NodeAddress peersNodeAddress, Capability capability) { + return findPeersCapabilities(peersNodeAddress) + .map(capabilities -> capabilities.contains(capability)) + .orElse(false); + } + + // TODO persist Capabilities + public Optional findPeersCapabilities(NodeAddress peersNodeAddress) { + return networkNode.getConfirmedConnections().stream() + .filter(c -> c.getPeersNodeAddressProperty().get() != null) + .filter(c -> c.getPeersNodeAddressProperty().get().equals(peersNodeAddress)) + .map(Connection::getCapabilities) + .findAny(); + } + /////////////////////////////////////////////////////////////////////////////////////////// // ConnectionListener implementation /////////////////////////////////////////////////////////////////////////////////////////// From 8aec30615922f368e98904fa9c9370cad141876c Mon Sep 17 00:00:00 2001 From: chimp1984 Date: Wed, 7 Oct 2020 11:52:17 -0500 Subject: [PATCH 05/59] Remove verification for address prefix Set address prefix to empty bytes in case we know that peer has capability (updated version) Batch process mailbox messages in a thread. Refactor handling of mailbox messages --- .../java/bisq/network/p2p/P2PService.java | 262 ++++++++++-------- .../p2p/PrefixedSealedAndSignedMessage.java | 4 + 2 files changed, 149 insertions(+), 117 deletions(-) diff --git a/p2p/src/main/java/bisq/network/p2p/P2PService.java b/p2p/src/main/java/bisq/network/p2p/P2PService.java index 4d2a5ef9025..9abde7bf16d 100644 --- a/p2p/src/main/java/bisq/network/p2p/P2PService.java +++ b/p2p/src/main/java/bisq/network/p2p/P2PService.java @@ -46,6 +46,7 @@ import bisq.network.p2p.storage.payload.ProtectedStoragePayload; import bisq.common.UserThread; +import bisq.common.app.Capability; import bisq.common.crypto.CryptoException; import bisq.common.crypto.KeyRing; import bisq.common.crypto.PubKeyRing; @@ -53,12 +54,15 @@ import bisq.common.proto.network.NetworkEnvelope; import bisq.common.proto.persistable.PersistedDataHost; import bisq.common.util.Tuple2; +import bisq.common.util.Utilities; import com.google.inject.Inject; import com.google.common.annotations.VisibleForTesting; import com.google.common.util.concurrent.FutureCallback; import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.ListeningExecutorService; import com.google.common.util.concurrent.MoreExecutors; import com.google.common.util.concurrent.SettableFuture; @@ -74,16 +78,20 @@ import java.security.PublicKey; -import java.util.Arrays; +import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Optional; +import java.util.Random; import java.util.Set; import java.util.UUID; import java.util.concurrent.CopyOnWriteArraySet; import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -164,7 +172,6 @@ public P2PService(NetworkNode networkNode, this.networkNode.addConnectionListener(this); this.networkNode.addMessageListener(this); - this.p2PDataStorage.addHashMapChangedListener(this); this.requestDataManager.addListener(this); // We need to have both the initial data delivered and the hidden service published @@ -197,13 +204,11 @@ public void start(@Nullable P2PServiceListener listener) { public void onAllServicesInitialized() { if (networkNode.getNodeAddress() != null) { - maybeProcessAllMailboxEntries(); myNodeAddress = networkNode.getNodeAddress(); } else { // If our HS is still not published networkNode.nodeAddressProperty().addListener((observable, oldValue, newValue) -> { if (newValue != null) { - maybeProcessAllMailboxEntries(); myNodeAddress = networkNode.getNodeAddress(); } }); @@ -284,7 +289,8 @@ public void onTorNodeReady() { if (!seedNodesAvailable) { isBootstrapped = true; - maybeProcessAllMailboxEntries(); + // As we do not expect a updated data request response we start here listening + addHashMapChangedListenerAndApply(); p2pServiceListeners.stream().forEach(P2PServiceListener::onNoSeedNodeAvailable); } } @@ -346,7 +352,9 @@ public void onPreliminaryDataReceived() { public void onUpdatedDataReceived() { if (!isBootstrapped) { isBootstrapped = true; - maybeProcessAllMailboxEntries(); + // Only now we start listening and processing. The p2PDataStorage is our cache for data we have received + // so far. + addHashMapChangedListenerAndApply(); p2pServiceListeners.stream().forEach(P2PServiceListener::onUpdatedDataReceived); p2PDataStorage.onBootstrapComplete(); } @@ -398,58 +406,120 @@ public void onError(Throwable throwable) { @Override public void onMessage(NetworkEnvelope networkEnvelope, Connection connection) { if (networkEnvelope instanceof PrefixedSealedAndSignedMessage) { - // Seed nodes don't have set the encryptionService + PrefixedSealedAndSignedMessage sealedMsg = (PrefixedSealedAndSignedMessage) networkEnvelope; + connection.setPeerType(Connection.PeerType.DIRECT_MSG_PEER); try { - PrefixedSealedAndSignedMessage prefixedSealedAndSignedMessage = (PrefixedSealedAndSignedMessage) networkEnvelope; - if (verifyAddressPrefixHash(prefixedSealedAndSignedMessage)) { - // We set connectionType to that connection to avoid that is get closed when - // we get too many connection attempts. - connection.setPeerType(Connection.PeerType.DIRECT_MSG_PEER); - - log.debug("Try to decrypt..."); - DecryptedMessageWithPubKey decryptedMessageWithPubKey = encryptionService.decryptAndVerify( - prefixedSealedAndSignedMessage.getSealedAndSigned()); - - log.debug("\n\nDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDD\n" + - "Decrypted SealedAndSignedMessage:\ndecryptedMsgWithPubKey={}" - + "\nDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDD\n", decryptedMessageWithPubKey); - if (connection.getPeersNodeAddressOptional().isPresent()) - decryptedDirectMessageListeners.forEach( - e -> e.onDirectMessage(decryptedMessageWithPubKey, connection.getPeersNodeAddressOptional().get())); - else - log.error("peersNodeAddress is not available at onMessage."); - } else { - log.debug("Wrong receiverAddressMaskHash. The message is not intended for us."); - } + DecryptedMessageWithPubKey decryptedMsg = encryptionService.decryptAndVerify(sealedMsg.getSealedAndSigned()); + connection.getPeersNodeAddressOptional().ifPresentOrElse(nodeAddress -> + decryptedDirectMessageListeners.forEach(e -> e.onDirectMessage(decryptedMsg, nodeAddress)), + () -> { + log.error("peersNodeAddress is expected to be available at onMessage for " + + "processing PrefixedSealedAndSignedMessage."); + }); } catch (CryptoException e) { - log.debug(networkEnvelope.toString()); - log.debug(e.toString()); - log.debug("Decryption of prefixedSealedAndSignedMessage.sealedAndSigned failed. " + - "That is expected if the message is not intended for us."); + log.warn("Decryption of a direct message failed. This is not expected as the " + + "direct message was sent to our node."); } catch (ProtobufferException e) { - log.error("Protobuffer data could not be processed: {}", e.toString()); + log.error("ProtobufferException at decryptAndVerify: {}", e.toString()); + e.getStackTrace(); } } } /////////////////////////////////////////////////////////////////////////////////////////// - // HashMapChangedListener implementation + // HashMapChangedListener implementation for ProtectedStorageEntry items /////////////////////////////////////////////////////////////////////////////////////////// + private void addHashMapChangedListenerAndApply() { + p2PDataStorage.addHashMapChangedListener(this); + onAdded(p2PDataStorage.getMap().values()); + } + @Override public void onAdded(Collection protectedStorageEntries) { - protectedStorageEntries.forEach(protectedStorageEntry -> { - if (protectedStorageEntry instanceof ProtectedMailboxStorageEntry) - processMailboxEntry((ProtectedMailboxStorageEntry) protectedStorageEntry); + Collection entries = protectedStorageEntries.stream() + .filter(e -> e instanceof ProtectedMailboxStorageEntry) + .map(e -> (ProtectedMailboxStorageEntry) e) + .filter(e -> networkNode.getNodeAddress() != null) + .filter(e -> !seedNodeRepository.isSeedNode(networkNode.getNodeAddress())) // Seed nodes don't expect mailbox messages + .collect(Collectors.toSet()); + if (entries.size() > 1) { + threadedBatchProcessMailboxEntries(entries); + } else if (entries.size() == 1) { + processSingleMailboxEntry(entries); + } + } + + private void processSingleMailboxEntry(Collection protectedMailboxStorageEntries) { + var decryptedEntries = new ArrayList<>(getDecryptedEntries(protectedMailboxStorageEntries)); + checkArgument(decryptedEntries.size() == 1); + storeMailboxDataAndNotifyListeners(decryptedEntries.get(0)); + } + + // We run the batch processing of all mailbox messages we have received at startup in a thread to not block the UI. + // For about 1000 messages decryption takes about 1 sec. + private void threadedBatchProcessMailboxEntries(Collection protectedMailboxStorageEntries) { + ListeningExecutorService executor = Utilities.getSingleThreadExecutor("processMailboxEntry-" + new Random().nextInt(1000)); + long ts = System.currentTimeMillis(); + ListenableFuture>> future = executor.submit(() -> { + var decryptedEntries = getDecryptedEntries(protectedMailboxStorageEntries); + log.info("Batch processing of {} mailbox entries took {} ms", + protectedMailboxStorageEntries.size(), + System.currentTimeMillis() - ts); + return decryptedEntries; }); + + Futures.addCallback(future, new FutureCallback<>() { + public void onSuccess(Set> decryptedEntries) { + UserThread.execute(() -> decryptedEntries.forEach(e -> storeMailboxDataAndNotifyListeners(e))); + } + + public void onFailure(@NotNull Throwable throwable) { + log.error(throwable.toString()); + } + }, MoreExecutors.directExecutor()); } - @Override - public void onRemoved(Collection protectedStorageEntries) { - // not used + private Set> getDecryptedEntries(Collection protectedMailboxStorageEntries) { + Set> decryptedEntries = new HashSet<>(); + protectedMailboxStorageEntries.stream() + .map(this::decryptProtectedMailboxStorageEntry) + .filter(Objects::nonNull) + .forEach(decryptedEntries::add); + return decryptedEntries; } + @Nullable + private Tuple2 decryptProtectedMailboxStorageEntry( + ProtectedMailboxStorageEntry protectedMailboxStorageEntry) { + try { + DecryptedMessageWithPubKey decryptedMessageWithPubKey = encryptionService.decryptAndVerify(protectedMailboxStorageEntry + .getMailboxStoragePayload() + .getPrefixedSealedAndSignedMessage() + .getSealedAndSigned()); + checkArgument(decryptedMessageWithPubKey.getNetworkEnvelope() instanceof MailboxMessage); + return new Tuple2<>(protectedMailboxStorageEntry, decryptedMessageWithPubKey); + } catch (CryptoException ignore) { + // Expected if message was not intended for us + } catch (ProtobufferException e) { + log.error(e.toString()); + e.getStackTrace(); + } + return null; + } + + private void storeMailboxDataAndNotifyListeners(Tuple2 tuple2) { + DecryptedMessageWithPubKey decryptedMessageWithPubKey = tuple2.second; + MailboxMessage mailboxMessage = (MailboxMessage) decryptedMessageWithPubKey.getNetworkEnvelope(); + NodeAddress sender = mailboxMessage.getSenderNodeAddress(); + mailboxMap.put(mailboxMessage.getUid(), tuple2); + log.info("Received a {} mailbox message with uid {} and senderAddress {}", + mailboxMessage.getClass().getSimpleName(), mailboxMessage.getUid(), sender); + decryptedMailboxListeners.forEach(e -> e.onMailboxMessageAdded(decryptedMessageWithPubKey, sender)); + } + + /////////////////////////////////////////////////////////////////////////////////////////// // DirectMessages /////////////////////////////////////////////////////////////////////////////////////////// @@ -486,13 +556,16 @@ private void doSendEncryptedDirectMessage(@NotNull NodeAddress peersNodeAddress, log.debug("\n\nEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEE\n" + "Encrypt message:\nmessage={}" + "\nEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEE\n", message); - PrefixedSealedAndSignedMessage prefixedSealedAndSignedMessage = new PrefixedSealedAndSignedMessage( - networkNode.getNodeAddress(), - encryptionService.encryptAndSign(pubKeyRing, message), - peersNodeAddress.getAddressPrefixHash(), - UUID.randomUUID().toString()); - SettableFuture future = networkNode.sendMessage(peersNodeAddress, prefixedSealedAndSignedMessage); - Futures.addCallback(future, new FutureCallback() { + + // Prefix is not needed for direct messages but as old code is doing the verification we still need to + // send it if peer has not updated. + // TODO persist capability + PrefixedSealedAndSignedMessage sealedMsg = getPrefixedSealedAndSignedMessage(peersNodeAddress, + pubKeyRing, + message); + + SettableFuture future = networkNode.sendMessage(peersNodeAddress, sealedMsg); + Futures.addCallback(future, new FutureCallback<>() { @Override public void onSuccess(@Nullable Connection connection) { sendDirectMessageListener.onArrived(); @@ -513,48 +586,30 @@ public void onFailure(@NotNull Throwable throwable) { } } + private PrefixedSealedAndSignedMessage getPrefixedSealedAndSignedMessage(NodeAddress peersNodeAddress, + PubKeyRing pubKeyRing, + NetworkEnvelope message) throws CryptoException { + byte[] addressPrefixHash; + if (peerManager.peerHasCapability(peersNodeAddress, Capability.NO_ADDRESS_PRE_FIX)) { + // The peer has an updated version so we do not need to send the prefix. + // We cannot use null as not updated nodes would get a nullPointer at protobuf serialisation. + addressPrefixHash = new byte[0]; + } else { + addressPrefixHash = peersNodeAddress.getAddressPrefixHash(); + } + return new PrefixedSealedAndSignedMessage( + networkNode.getNodeAddress(), + encryptionService.encryptAndSign(pubKeyRing, message), + addressPrefixHash, + UUID.randomUUID().toString()); + } + /////////////////////////////////////////////////////////////////////////////////////////// // MailboxMessages /////////////////////////////////////////////////////////////////////////////////////////// - private void processMailboxEntry(ProtectedMailboxStorageEntry protectedMailboxStorageEntry) { - NodeAddress nodeAddress = networkNode.getNodeAddress(); - // Seed nodes don't receive mailbox network_messages - if (nodeAddress != null && !seedNodeRepository.isSeedNode(nodeAddress)) { - MailboxStoragePayload mailboxStoragePayload = protectedMailboxStorageEntry.getMailboxStoragePayload(); - PrefixedSealedAndSignedMessage prefixedSealedAndSignedMessage = mailboxStoragePayload.getPrefixedSealedAndSignedMessage(); - if (verifyAddressPrefixHash(prefixedSealedAndSignedMessage)) { - try { - DecryptedMessageWithPubKey decryptedMessageWithPubKey = encryptionService.decryptAndVerify( - prefixedSealedAndSignedMessage.getSealedAndSigned()); - if (decryptedMessageWithPubKey.getNetworkEnvelope() instanceof MailboxMessage) { - MailboxMessage mailboxMessage = (MailboxMessage) decryptedMessageWithPubKey.getNetworkEnvelope(); - NodeAddress senderNodeAddress = mailboxMessage.getSenderNodeAddress(); - checkNotNull(senderNodeAddress, "senderAddress must not be null for mailbox network_messages"); - - mailboxMap.put(mailboxMessage.getUid(), new Tuple2<>(protectedMailboxStorageEntry, decryptedMessageWithPubKey)); - log.info("Received a {} mailbox message with messageUid {} and senderAddress {}", mailboxMessage.getClass().getSimpleName(), mailboxMessage.getUid(), senderNodeAddress); - decryptedMailboxListeners.forEach( - e -> e.onMailboxMessageAdded(decryptedMessageWithPubKey, senderNodeAddress)); - } else { - log.warn("tryDecryptMailboxData: Expected MailboxMessage but got other type. " + - "decryptedMsgWithPubKey.message={}", decryptedMessageWithPubKey.getNetworkEnvelope()); - } - } catch (CryptoException e) { - log.debug(e.toString()); - log.debug("Decryption of prefixedSealedAndSignedMessage.sealedAndSigned failed. " + - "That is expected if the message is not intended for us."); - } catch (ProtobufferException e) { - log.error("Protobuffer data could not be processed: {}", e.toString()); - } - } else { - log.trace("Wrong blurredAddressHash. The message is not intended for us."); - } - } - } - - public void sendEncryptedMailboxMessage(NodeAddress peersNodeAddress, PubKeyRing peersPubKeyRing, + public void sendEncryptedMailboxMessage(NodeAddress peer, PubKeyRing peersPubKeyRing, NetworkEnvelope message, SendMailboxMessageListener sendMailboxMessageListener) { if (peersPubKeyRing == null) { @@ -562,7 +617,7 @@ public void sendEncryptedMailboxMessage(NodeAddress peersNodeAddress, PubKeyRing return; } - checkNotNull(peersNodeAddress, + checkNotNull(peer, "PeerAddress must not be null (sendEncryptedMailboxMessage)"); checkNotNull(networkNode.getNodeAddress(), "My node address must not be null at sendEncryptedMailboxMessage"); @@ -578,7 +633,7 @@ public void sendEncryptedMailboxMessage(NodeAddress peersNodeAddress, PubKeyRing return; } - if (capabilityRequiredAndCapabilityNotSupported(peersNodeAddress, message)) { + if (capabilityRequiredAndCapabilityNotSupported(peer, message)) { sendMailboxMessageListener.onFault("We did not send the EncryptedMailboxMessage " + "because the peer does not support the capability."); return; @@ -589,14 +644,11 @@ public void sendEncryptedMailboxMessage(NodeAddress peersNodeAddress, PubKeyRing "Encrypt message:\nmessage={}" + "\nEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEE\n", message); - PrefixedSealedAndSignedMessage prefixedSealedAndSignedMessage = new PrefixedSealedAndSignedMessage( - networkNode.getNodeAddress(), - encryptionService.encryptAndSign(peersPubKeyRing, message), - peersNodeAddress.getAddressPrefixHash(), - UUID.randomUUID().toString()); + PrefixedSealedAndSignedMessage prefixedSealedAndSignedMessage = getPrefixedSealedAndSignedMessage(peer, + peersPubKeyRing, message); - log.debug("sendEncryptedMailboxMessage msg={}, peersNodeAddress={}", message, peersNodeAddress); - SettableFuture future = networkNode.sendMessage(peersNodeAddress, prefixedSealedAndSignedMessage); + log.debug("sendEncryptedMailboxMessage msg={}, peersNodeAddress={}", message, peer); + SettableFuture future = networkNode.sendMessage(peer, prefixedSealedAndSignedMessage); Futures.addCallback(future, new FutureCallback() { @Override public void onSuccess(@Nullable Connection connection) { @@ -654,15 +706,6 @@ private boolean capabilityRequiredAndCapabilityNotSupported(NodeAddress peersNod } - private void maybeProcessAllMailboxEntries() { - if (isBootstrapped) { - p2PDataStorage.getMap().values().forEach(protectedStorageEntry -> { - if (protectedStorageEntry instanceof ProtectedMailboxStorageEntry) - processMailboxEntry((ProtectedMailboxStorageEntry) protectedStorageEntry); - }); - } - } - private void addMailboxData(MailboxStoragePayload expirableMailboxStoragePayload, PublicKey receiversPublicKey, SendMailboxMessageListener sendMailboxMessageListener) { @@ -892,19 +935,4 @@ public PeerManager getPeerManager() { public KeyRing getKeyRing() { return keyRing; } - - /////////////////////////////////////////////////////////////////////////////////////////// - // Private - /////////////////////////////////////////////////////////////////////////////////////////// - - private boolean verifyAddressPrefixHash(PrefixedSealedAndSignedMessage prefixedSealedAndSignedMessage) { - if (networkNode.getNodeAddress() != null) { - byte[] blurredAddressHash = networkNode.getNodeAddress().getAddressPrefixHash(); - return blurredAddressHash != null && - Arrays.equals(blurredAddressHash, prefixedSealedAndSignedMessage.getAddressPrefixHash()); - } else { - log.debug("myOnionAddress is null at verifyAddressPrefixHash. That is expected at startup."); - return false; - } - } } diff --git a/p2p/src/main/java/bisq/network/p2p/PrefixedSealedAndSignedMessage.java b/p2p/src/main/java/bisq/network/p2p/PrefixedSealedAndSignedMessage.java index d5f626a90d5..9f6177ba596 100644 --- a/p2p/src/main/java/bisq/network/p2p/PrefixedSealedAndSignedMessage.java +++ b/p2p/src/main/java/bisq/network/p2p/PrefixedSealedAndSignedMessage.java @@ -33,7 +33,11 @@ public final class PrefixedSealedAndSignedMessage extends NetworkEnvelope implements MailboxMessage, SendersNodeAddressMessage { private final NodeAddress senderNodeAddress; private final SealedAndSigned sealedAndSigned; + + // From v1.4.0 on that can be an empty byte array. We cannot use null as not updated nodes would get a nullPointer + // at protobuf serialisation. private final byte[] addressPrefixHash; + private final String uid; public PrefixedSealedAndSignedMessage(NodeAddress senderNodeAddress, From 9821dd62719ea354c3d52edff8c4d57568d946d4 Mon Sep 17 00:00:00 2001 From: chimp1984 Date: Wed, 7 Oct 2020 13:03:18 -0500 Subject: [PATCH 06/59] Clear capabilitiesListeners at shutdown Improve logs --- .../bisq/network/p2p/network/Connection.java | 36 ++++++++++++------- 1 file changed, 23 insertions(+), 13 deletions(-) diff --git a/p2p/src/main/java/bisq/network/p2p/network/Connection.java b/p2p/src/main/java/bisq/network/p2p/network/Connection.java index 8b3560e3362..7789d0422b8 100644 --- a/p2p/src/main/java/bisq/network/p2p/network/Connection.java +++ b/p2p/src/main/java/bisq/network/p2p/network/Connection.java @@ -159,6 +159,8 @@ public static int getPermittedMessageSize() { private final List messageTimeStamps = new ArrayList<>(); private final CopyOnWriteArraySet messageListeners = new CopyOnWriteArraySet<>(); private volatile long lastSendTimeStamp = 0; + // We use a weak reference here to ensure that no connection causes a memory leak in case it get closed without + // the shutDown being called. private final CopyOnWriteArraySet> capabilitiesListeners = new CopyOnWriteArraySet<>(); @Getter @@ -514,6 +516,8 @@ private void doShutDown(CloseConnectionReason closeConnectionReason, @Nullable R } finally { protoOutputStream.onConnectionShutdown(); + capabilitiesListeners.clear(); + try { protoInputStream.close(); } catch (IOException e) { @@ -559,7 +563,6 @@ public String toString() { '}'; } - @SuppressWarnings("unused") public String printDetails() { String portInfo; if (socket.getLocalPort() == 0) @@ -783,19 +786,26 @@ && reportInvalidRequest(RuleViolation.WRONG_NETWORK_ID)) { } if (networkEnvelope instanceof SupportedCapabilitiesMessage) { - Capabilities supportedCapabilities = ((SupportedCapabilitiesMessage) networkEnvelope).getSupportedCapabilities(); - if (supportedCapabilities != null) { - if (!capabilities.equals(supportedCapabilities)) { - capabilities.set(supportedCapabilities); + Capabilities capabilities = ((SupportedCapabilitiesMessage) networkEnvelope).getSupportedCapabilities(); + if (capabilities != null) { + if (!this.capabilities.equals(capabilities)) { + this.capabilities.set(capabilities); // Capabilities can be empty. We only check for mandatory if we get some capabilities. - if (!capabilities.isEmpty() && !Capabilities.hasMandatoryCapability(capabilities)) { - String senderNodeAddress = networkEnvelope instanceof SendersNodeAddressMessage ? - ((SendersNodeAddressMessage) networkEnvelope).getSenderNodeAddress().getFullAddress() : - "[unknown address]"; - log.info("We close a connection to old node {}. " + - "Capabilities of old node: {}, networkEnvelope class name={}", - senderNodeAddress, capabilities.prettyPrint(), networkEnvelope.getClass().getSimpleName()); + if (!this.capabilities.isEmpty() && !Capabilities.hasMandatoryCapability(this.capabilities)) { + String senderNodeAddress = getPeersNodeAddressOptional().isPresent() ? + getPeersNodeAddressOptional().get().getFullAddress() : + networkEnvelope instanceof SendersNodeAddressMessage ? + ((SendersNodeAddressMessage) networkEnvelope).getSenderNodeAddress().getFullAddress() : + "[unknown address]"; + + log.info("We close a connection because of " + + "CloseConnectionReason.MANDATORY_CAPABILITIES_NOT_SUPPORTED " + + "to node {}. Capabilities of old node: {}, " + + "networkEnvelope class name={}", + senderNodeAddress, + this.capabilities.prettyPrint(), + networkEnvelope.getClass().getSimpleName()); shutDown(CloseConnectionReason.MANDATORY_CAPABILITIES_NOT_SUPPORTED); return; } @@ -803,7 +813,7 @@ && reportInvalidRequest(RuleViolation.WRONG_NETWORK_ID)) { capabilitiesListeners.forEach(weakListener -> { SupportedCapabilitiesListener supportedCapabilitiesListener = weakListener.get(); if (supportedCapabilitiesListener != null) { - UserThread.execute(() -> supportedCapabilitiesListener.onChanged(supportedCapabilities)); + UserThread.execute(() -> supportedCapabilitiesListener.onChanged(capabilities)); } }); } From 17974f3dcc358c9ca47f28aec97f4cf6c95f39c5 Mon Sep 17 00:00:00 2001 From: chimp1984 Date: Wed, 7 Oct 2020 13:11:29 -0500 Subject: [PATCH 07/59] Refactor: move SupportedCapabilitiesMessage handling code out to a method Return early --- .../bisq/network/p2p/network/Connection.java | 82 ++++++++++++------- 1 file changed, 51 insertions(+), 31 deletions(-) diff --git a/p2p/src/main/java/bisq/network/p2p/network/Connection.java b/p2p/src/main/java/bisq/network/p2p/network/Connection.java index 7789d0422b8..2a28158aac5 100644 --- a/p2p/src/main/java/bisq/network/p2p/network/Connection.java +++ b/p2p/src/main/java/bisq/network/p2p/network/Connection.java @@ -786,37 +786,9 @@ && reportInvalidRequest(RuleViolation.WRONG_NETWORK_ID)) { } if (networkEnvelope instanceof SupportedCapabilitiesMessage) { - Capabilities capabilities = ((SupportedCapabilitiesMessage) networkEnvelope).getSupportedCapabilities(); - if (capabilities != null) { - if (!this.capabilities.equals(capabilities)) { - this.capabilities.set(capabilities); - - // Capabilities can be empty. We only check for mandatory if we get some capabilities. - if (!this.capabilities.isEmpty() && !Capabilities.hasMandatoryCapability(this.capabilities)) { - String senderNodeAddress = getPeersNodeAddressOptional().isPresent() ? - getPeersNodeAddressOptional().get().getFullAddress() : - networkEnvelope instanceof SendersNodeAddressMessage ? - ((SendersNodeAddressMessage) networkEnvelope).getSenderNodeAddress().getFullAddress() : - "[unknown address]"; - - log.info("We close a connection because of " + - "CloseConnectionReason.MANDATORY_CAPABILITIES_NOT_SUPPORTED " + - "to node {}. Capabilities of old node: {}, " + - "networkEnvelope class name={}", - senderNodeAddress, - this.capabilities.prettyPrint(), - networkEnvelope.getClass().getSimpleName()); - shutDown(CloseConnectionReason.MANDATORY_CAPABILITIES_NOT_SUPPORTED); - return; - } - - capabilitiesListeners.forEach(weakListener -> { - SupportedCapabilitiesListener supportedCapabilitiesListener = weakListener.get(); - if (supportedCapabilitiesListener != null) { - UserThread.execute(() -> supportedCapabilitiesListener.onChanged(capabilities)); - } - }); - } + boolean causedShutDown = handleSupportedCapabilitiesMessage(networkEnvelope); + if (causedShutDown) { + return; } } @@ -892,4 +864,52 @@ && reportInvalidRequest(RuleViolation.WRONG_NETWORK_ID)) { handleException(t); } } + + protected boolean handleSupportedCapabilitiesMessage(NetworkEnvelope networkEnvelope) { + Capabilities supportedCapabilities = ((SupportedCapabilitiesMessage) networkEnvelope).getSupportedCapabilities(); + if (supportedCapabilities == null || supportedCapabilities.isEmpty()) { + return false; + } + + if (this.capabilities.equals(supportedCapabilities)) { + return false; + } + + if (!Capabilities.hasMandatoryCapability(supportedCapabilities)) { + log.info("We close a connection because of " + + "CloseConnectionReason.MANDATORY_CAPABILITIES_NOT_SUPPORTED " + + "to node {}. Capabilities of old node: {}, " + + "networkEnvelope class name={}", + getSenderNodeAddressAsString(networkEnvelope), + supportedCapabilities.prettyPrint(), + networkEnvelope.getClass().getSimpleName()); + shutDown(CloseConnectionReason.MANDATORY_CAPABILITIES_NOT_SUPPORTED); + return true; + } + + this.capabilities.set(supportedCapabilities); + + capabilitiesListeners.forEach(weakListener -> { + SupportedCapabilitiesListener supportedCapabilitiesListener = weakListener.get(); + if (supportedCapabilitiesListener != null) { + UserThread.execute(() -> supportedCapabilitiesListener.onChanged(supportedCapabilities)); + } + }); + return false; + } + + @Nullable + private NodeAddress getSenderNodeAddress(NetworkEnvelope networkEnvelope) { + return getPeersNodeAddressOptional().isPresent() ? + getPeersNodeAddressOptional().get() : + networkEnvelope instanceof SendersNodeAddressMessage ? + ((SendersNodeAddressMessage) networkEnvelope).getSenderNodeAddress() : + null; + } + + private String getSenderNodeAddressAsString(NetworkEnvelope networkEnvelope) { + NodeAddress nodeAddress = getSenderNodeAddress(networkEnvelope); + return nodeAddress == null ? "null" : nodeAddress.getFullAddress(); + } + } From 1c07be05074f79b75a0064fee12c6e483762be56 Mon Sep 17 00:00:00 2001 From: chimp1984 Date: Wed, 7 Oct 2020 13:14:37 -0500 Subject: [PATCH 08/59] Use only node address for equals and hashcode Make capabilities final If capability changes we would have had duplicate entries --- .../network/p2p/peers/peerexchange/Peer.java | 25 +++++++++++++------ 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/p2p/src/main/java/bisq/network/p2p/peers/peerexchange/Peer.java b/p2p/src/main/java/bisq/network/p2p/peers/peerexchange/Peer.java index cb3f8d64e7e..8c716c6787a 100644 --- a/p2p/src/main/java/bisq/network/p2p/peers/peerexchange/Peer.java +++ b/p2p/src/main/java/bisq/network/p2p/peers/peerexchange/Peer.java @@ -27,7 +27,6 @@ import java.util.Date; -import lombok.EqualsAndHashCode; import lombok.Getter; import lombok.Setter; import lombok.extern.slf4j.Slf4j; @@ -35,18 +34,15 @@ import javax.annotation.Nullable; @Getter -@EqualsAndHashCode(exclude = {"date"}) // failedConnectionAttempts is transient and therefore excluded anyway @Slf4j public final class Peer implements HasCapabilities, NetworkPayload, PersistablePayload, SupportedCapabilitiesListener { private static final int MAX_FAILED_CONNECTION_ATTEMPTS = 5; private final NodeAddress nodeAddress; private final long date; - // Added in v. 0.7.1 - @Setter transient private int failedConnectionAttempts = 0; - private Capabilities capabilities = new Capabilities(); + private final Capabilities capabilities = new Capabilities(); public Peer(NodeAddress nodeAddress, @Nullable Capabilities supportedCapabilities) { this(nodeAddress, new Date().getTime(), supportedCapabilities); @@ -102,14 +98,29 @@ public void onChanged(Capabilities supportedCapabilities) { } } + // We use only node address for equals and hashcode + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (!(o instanceof Peer)) return false; + + Peer peer = (Peer) o; + + return nodeAddress != null ? nodeAddress.equals(peer.nodeAddress) : peer.nodeAddress == null; + } + + @Override + public int hashCode() { + return nodeAddress != null ? nodeAddress.hashCode() : 0; + } @Override public String toString() { return "Peer{" + "\n nodeAddress=" + nodeAddress + - ",\n supportedCapabilities=" + capabilities + - ",\n failedConnectionAttempts=" + failedConnectionAttempts + ",\n date=" + date + + ",\n failedConnectionAttempts=" + failedConnectionAttempts + + ",\n capabilities=" + capabilities + "\n}"; } } From bf659a1e6d0d235a2e85e0efab74fff39d6682a9 Mon Sep 17 00:00:00 2001 From: chimp1984 Date: Wed, 7 Oct 2020 13:15:49 -0500 Subject: [PATCH 09/59] Pass supportedCapabilities to PeerManager. Not further processed yet, will be done in next commits Cleanups --- p2p/src/main/java/bisq/network/p2p/P2PService.java | 2 -- p2p/src/main/java/bisq/network/p2p/peers/PeerManager.java | 7 ++++++- .../p2p/peers/peerexchange/GetPeersRequestHandler.java | 5 +++-- .../p2p/peers/peerexchange/PeerExchangeHandler.java | 4 +++- .../p2p/peers/peerexchange/PeerExchangeManager.java | 3 ++- 5 files changed, 14 insertions(+), 7 deletions(-) diff --git a/p2p/src/main/java/bisq/network/p2p/P2PService.java b/p2p/src/main/java/bisq/network/p2p/P2PService.java index 9abde7bf16d..b36a67c0a73 100644 --- a/p2p/src/main/java/bisq/network/p2p/P2PService.java +++ b/p2p/src/main/java/bisq/network/p2p/P2PService.java @@ -323,8 +323,6 @@ private void onNetworkReady() { "seedNodeOfPreliminaryDataRequest must be present"); requestDataManager.requestUpdateData(); - /*if (Capabilities.app.containsAll(Capability.SEED_NODE)) - UserThread.runPeriodically(() -> requestDataManager.requestUpdateData(), 1, TimeUnit.HOURS);*/ // If we start up first time we don't have any peers so we need to request from seed node. // As well it can be that the persisted peer list is outdated with dead peers. diff --git a/p2p/src/main/java/bisq/network/p2p/peers/PeerManager.java b/p2p/src/main/java/bisq/network/p2p/peers/PeerManager.java index f12c95b9fd7..15cb5508b80 100644 --- a/p2p/src/main/java/bisq/network/p2p/peers/PeerManager.java +++ b/p2p/src/main/java/bisq/network/p2p/peers/PeerManager.java @@ -464,7 +464,12 @@ public Set getReportedPeers() { return reportedPeers; } - public void addToReportedPeers(Set reportedPeersToAdd, Connection connection) { + public void addToReportedPeers(Set reportedPeersToAdd, + Connection connection, + Capabilities supportedCapabilities) { + + //TODO apply supportedCapabilities + printNewReportedPeers(reportedPeersToAdd); // We check if the reported msg is not violating our rules diff --git a/p2p/src/main/java/bisq/network/p2p/peers/peerexchange/GetPeersRequestHandler.java b/p2p/src/main/java/bisq/network/p2p/peers/peerexchange/GetPeersRequestHandler.java index 2297758ee30..f280587c86b 100644 --- a/p2p/src/main/java/bisq/network/p2p/peers/peerexchange/GetPeersRequestHandler.java +++ b/p2p/src/main/java/bisq/network/p2p/peers/peerexchange/GetPeersRequestHandler.java @@ -130,8 +130,9 @@ public void onFailure(@NotNull Throwable throwable) { } } }, MoreExecutors.directExecutor()); - - peerManager.addToReportedPeers(getPeersRequest.getReportedPeers(), connection); + peerManager.addToReportedPeers(getPeersRequest.getReportedPeers(), + connection, + getPeersRequest.getSupportedCapabilities()); } diff --git a/p2p/src/main/java/bisq/network/p2p/peers/peerexchange/PeerExchangeHandler.java b/p2p/src/main/java/bisq/network/p2p/peers/peerexchange/PeerExchangeHandler.java index d0266c2ea36..df6b685dec7 100644 --- a/p2p/src/main/java/bisq/network/p2p/peers/peerexchange/PeerExchangeHandler.java +++ b/p2p/src/main/java/bisq/network/p2p/peers/peerexchange/PeerExchangeHandler.java @@ -168,7 +168,9 @@ public void onMessage(NetworkEnvelope networkEnvelope, Connection connection) { // Check if the response is for our request if (getPeersResponse.getRequestNonce() == nonce) { - peerManager.addToReportedPeers(getPeersResponse.getReportedPeers(), connection); + peerManager.addToReportedPeers(getPeersResponse.getReportedPeers(), + connection, + getPeersResponse.getSupportedCapabilities()); cleanup(); listener.onComplete(); } else { diff --git a/p2p/src/main/java/bisq/network/p2p/peers/peerexchange/PeerExchangeManager.java b/p2p/src/main/java/bisq/network/p2p/peers/peerexchange/PeerExchangeManager.java index ad8489a1790..8d62ead5a59 100644 --- a/p2p/src/main/java/bisq/network/p2p/peers/peerexchange/PeerExchangeManager.java +++ b/p2p/src/main/java/bisq/network/p2p/peers/peerexchange/PeerExchangeManager.java @@ -224,7 +224,8 @@ public void onFault(String errorMessage, Connection connection) { /////////////////////////////////////////////////////////////////////////////////////////// private void requestReportedPeers(NodeAddress nodeAddress, List remainingNodeAddresses) { - log.debug("requestReportedPeers nodeAddress={}; remainingNodeAddresses.size={}", nodeAddress, remainingNodeAddresses.size()); + log.debug("requestReportedPeers nodeAddress={}; remainingNodeAddresses.size={}", + nodeAddress, remainingNodeAddresses.size()); if (!stopped) { if (!handlerMap.containsKey(nodeAddress)) { PeerExchangeHandler peerExchangeHandler = new PeerExchangeHandler(networkNode, From 2523c2e9146c0c26e4a551716dcbecf77c0a2e8b Mon Sep 17 00:00:00 2001 From: chimp1984 Date: Wed, 7 Oct 2020 13:35:10 -0500 Subject: [PATCH 10/59] Use getSingleThreadListeningExecutor, cleanups --- p2p/src/main/java/bisq/network/p2p/P2PService.java | 13 +++++-------- .../network/p2p/PrefixedSealedAndSignedMessage.java | 7 ++++--- .../java/bisq/network/p2p/network/Connection.java | 1 - .../java/bisq/network/p2p/peers/PeerManager.java | 2 +- 4 files changed, 10 insertions(+), 13 deletions(-) diff --git a/p2p/src/main/java/bisq/network/p2p/P2PService.java b/p2p/src/main/java/bisq/network/p2p/P2PService.java index b36a67c0a73..f129127941a 100644 --- a/p2p/src/main/java/bisq/network/p2p/P2PService.java +++ b/p2p/src/main/java/bisq/network/p2p/P2PService.java @@ -289,7 +289,7 @@ public void onTorNodeReady() { if (!seedNodesAvailable) { isBootstrapped = true; - // As we do not expect a updated data request response we start here listening + // As we do not expect a updated data request response we start here with addHashMapChangedListenerAndApply addHashMapChangedListenerAndApply(); p2pServiceListeners.stream().forEach(P2PServiceListener::onNoSeedNodeAvailable); } @@ -351,7 +351,7 @@ public void onUpdatedDataReceived() { if (!isBootstrapped) { isBootstrapped = true; // Only now we start listening and processing. The p2PDataStorage is our cache for data we have received - // so far. + // after the hidden service was ready. addHashMapChangedListenerAndApply(); p2pServiceListeners.stream().forEach(P2PServiceListener::onUpdatedDataReceived); p2PDataStorage.onBootstrapComplete(); @@ -458,7 +458,7 @@ private void processSingleMailboxEntry(Collection // We run the batch processing of all mailbox messages we have received at startup in a thread to not block the UI. // For about 1000 messages decryption takes about 1 sec. private void threadedBatchProcessMailboxEntries(Collection protectedMailboxStorageEntries) { - ListeningExecutorService executor = Utilities.getSingleThreadExecutor("processMailboxEntry-" + new Random().nextInt(1000)); + ListeningExecutorService executor = Utilities.getSingleThreadListeningExecutor("processMailboxEntry-" + new Random().nextInt(1000)); long ts = System.currentTimeMillis(); ListenableFuture>> future = executor.submit(() -> { var decryptedEntries = getDecryptedEntries(protectedMailboxStorageEntries); @@ -557,7 +557,6 @@ private void doSendEncryptedDirectMessage(@NotNull NodeAddress peersNodeAddress, // Prefix is not needed for direct messages but as old code is doing the verification we still need to // send it if peer has not updated. - // TODO persist capability PrefixedSealedAndSignedMessage sealedMsg = getPrefixedSealedAndSignedMessage(peersNodeAddress, pubKeyRing, message); @@ -615,12 +614,10 @@ public void sendEncryptedMailboxMessage(NodeAddress peer, PubKeyRing peersPubKey return; } - checkNotNull(peer, - "PeerAddress must not be null (sendEncryptedMailboxMessage)"); + checkNotNull(peer, "PeerAddress must not be null (sendEncryptedMailboxMessage)"); checkNotNull(networkNode.getNodeAddress(), "My node address must not be null at sendEncryptedMailboxMessage"); - checkArgument(!keyRing.getPubKeyRing().equals(peersPubKeyRing), - "We got own keyring instead of that from peer"); + checkArgument(!keyRing.getPubKeyRing().equals(peersPubKeyRing), "We got own keyring instead of that from peer"); if (!isBootstrapped()) throw new NetworkNotReadyException(); diff --git a/p2p/src/main/java/bisq/network/p2p/PrefixedSealedAndSignedMessage.java b/p2p/src/main/java/bisq/network/p2p/PrefixedSealedAndSignedMessage.java index 9f6177ba596..bf617dce114 100644 --- a/p2p/src/main/java/bisq/network/p2p/PrefixedSealedAndSignedMessage.java +++ b/p2p/src/main/java/bisq/network/p2p/PrefixedSealedAndSignedMessage.java @@ -34,8 +34,8 @@ public final class PrefixedSealedAndSignedMessage extends NetworkEnvelope implem private final NodeAddress senderNodeAddress; private final SealedAndSigned sealedAndSigned; - // From v1.4.0 on that can be an empty byte array. We cannot use null as not updated nodes would get a nullPointer - // at protobuf serialisation. + // From v1.4.0 on addressPrefixHash can be an empty byte array. + // We cannot make it nullable as not updated nodes would get a nullPointer exception at protobuf serialisation. private final byte[] addressPrefixHash; private final String uid; @@ -75,7 +75,8 @@ public protobuf.NetworkEnvelope toProtoNetworkEnvelope() { .build(); } - public static PrefixedSealedAndSignedMessage fromProto(protobuf.PrefixedSealedAndSignedMessage proto, int messageVersion) { + public static PrefixedSealedAndSignedMessage fromProto(protobuf.PrefixedSealedAndSignedMessage proto, + int messageVersion) { return new PrefixedSealedAndSignedMessage(NodeAddress.fromProto(proto.getNodeAddress()), SealedAndSigned.fromProto(proto.getSealedAndSigned()), proto.getAddressPrefixHash().toByteArray(), diff --git a/p2p/src/main/java/bisq/network/p2p/network/Connection.java b/p2p/src/main/java/bisq/network/p2p/network/Connection.java index 2a28158aac5..6a29c7e0343 100644 --- a/p2p/src/main/java/bisq/network/p2p/network/Connection.java +++ b/p2p/src/main/java/bisq/network/p2p/network/Connection.java @@ -911,5 +911,4 @@ private String getSenderNodeAddressAsString(NetworkEnvelope networkEnvelope) { NodeAddress nodeAddress = getSenderNodeAddress(networkEnvelope); return nodeAddress == null ? "null" : nodeAddress.getFullAddress(); } - } diff --git a/p2p/src/main/java/bisq/network/p2p/peers/PeerManager.java b/p2p/src/main/java/bisq/network/p2p/peers/PeerManager.java index 15cb5508b80..9ec74dc2eb9 100644 --- a/p2p/src/main/java/bisq/network/p2p/peers/PeerManager.java +++ b/p2p/src/main/java/bisq/network/p2p/peers/PeerManager.java @@ -213,7 +213,7 @@ public boolean peerHasCapability(NodeAddress peersNodeAddress, Capability capabi .orElse(false); } - // TODO persist Capabilities + // TODO get Capabilities from peers public Optional findPeersCapabilities(NodeAddress peersNodeAddress) { return networkNode.getConfirmedConnections().stream() .filter(c -> c.getPeersNodeAddressProperty().get() != null) From 765f9ea940fde8ae0b938c352a96461614131629 Mon Sep 17 00:00:00 2001 From: chimp1984 Date: Wed, 7 Oct 2020 13:42:29 -0500 Subject: [PATCH 11/59] Apply code inspection suggestions The check for AckMessage is not needed anymore as we remove the interface from AckMessage --- .../java/bisq/network/p2p/P2PService.java | 28 ++++++++----------- 1 file changed, 11 insertions(+), 17 deletions(-) diff --git a/p2p/src/main/java/bisq/network/p2p/P2PService.java b/p2p/src/main/java/bisq/network/p2p/P2PService.java index f129127941a..b127833fd4d 100644 --- a/p2p/src/main/java/bisq/network/p2p/P2PService.java +++ b/p2p/src/main/java/bisq/network/p2p/P2PService.java @@ -285,13 +285,13 @@ public void onTorNodeReady() { boolean seedNodesAvailable = requestDataManager.requestPreliminaryData(); keepAliveManager.start(); - p2pServiceListeners.stream().forEach(SetupListener::onTorNodeReady); + p2pServiceListeners.forEach(SetupListener::onTorNodeReady); if (!seedNodesAvailable) { isBootstrapped = true; // As we do not expect a updated data request response we start here with addHashMapChangedListenerAndApply addHashMapChangedListenerAndApply(); - p2pServiceListeners.stream().forEach(P2PServiceListener::onNoSeedNodeAvailable); + p2pServiceListeners.forEach(P2PServiceListener::onNoSeedNodeAvailable); } } @@ -301,17 +301,17 @@ public void onHiddenServicePublished() { hiddenServicePublished.set(true); - p2pServiceListeners.stream().forEach(SetupListener::onHiddenServicePublished); + p2pServiceListeners.forEach(SetupListener::onHiddenServicePublished); } @Override public void onSetupFailed(Throwable throwable) { - p2pServiceListeners.stream().forEach(e -> e.onSetupFailed(throwable)); + p2pServiceListeners.forEach(e -> e.onSetupFailed(throwable)); } @Override public void onRequestCustomBridges() { - p2pServiceListeners.stream().forEach(SetupListener::onRequestCustomBridges); + p2pServiceListeners.forEach(SetupListener::onRequestCustomBridges); } // Called from networkReadyBinding @@ -353,24 +353,24 @@ public void onUpdatedDataReceived() { // Only now we start listening and processing. The p2PDataStorage is our cache for data we have received // after the hidden service was ready. addHashMapChangedListenerAndApply(); - p2pServiceListeners.stream().forEach(P2PServiceListener::onUpdatedDataReceived); + p2pServiceListeners.forEach(P2PServiceListener::onUpdatedDataReceived); p2PDataStorage.onBootstrapComplete(); } } @Override public void onNoSeedNodeAvailable() { - p2pServiceListeners.stream().forEach(P2PServiceListener::onNoSeedNodeAvailable); + p2pServiceListeners.forEach(P2PServiceListener::onNoSeedNodeAvailable); } @Override public void onNoPeersAvailable() { - p2pServiceListeners.stream().forEach(P2PServiceListener::onNoPeersAvailable); + p2pServiceListeners.forEach(P2PServiceListener::onNoPeersAvailable); } @Override public void onDataReceived() { - p2pServiceListeners.stream().forEach(P2PServiceListener::onDataReceived); + p2pServiceListeners.forEach(P2PServiceListener::onDataReceived); } @@ -644,7 +644,7 @@ public void sendEncryptedMailboxMessage(NodeAddress peer, PubKeyRing peersPubKey log.debug("sendEncryptedMailboxMessage msg={}, peersNodeAddress={}", message, peer); SettableFuture future = networkNode.sendMessage(peer, prefixedSealedAndSignedMessage); - Futures.addCallback(future, new FutureCallback() { + Futures.addCallback(future, new FutureCallback<>() { @Override public void onSuccess(@Nullable Connection connection) { sendMailboxMessageListener.onArrived(); @@ -671,11 +671,6 @@ private boolean capabilityRequiredAndCapabilityNotSupported(NodeAddress peersNod if (!(message instanceof CapabilityRequiringPayload)) return false; - // We only expect AckMessage so far - if (!(message instanceof AckMessage)) - log.warn("We got a CapabilityRequiringPayload for the mailbox message which is not a AckMessage. " + - "peersNodeAddress={}", peersNodeAddress); - Set allPeers = peerManager.getPersistedPeers(); allPeers.addAll(peerManager.getReportedPeers()); allPeers.addAll(peerManager.getLivePeers(null)); @@ -878,8 +873,7 @@ public void addP2PServiceListener(P2PServiceListener listener) { } public void removeP2PServiceListener(P2PServiceListener listener) { - if (p2pServiceListeners.contains(listener)) - p2pServiceListeners.remove(listener); + p2pServiceListeners.remove(listener); } public void addHashSetChangedListener(HashMapChangedListener hashMapChangedListener) { From 186a9d670d33993a31bfba367eba5bb4261f3340 Mon Sep 17 00:00:00 2001 From: chimp1984 Date: Wed, 7 Oct 2020 14:46:55 -0500 Subject: [PATCH 12/59] Add findPeersCapabilities method --- .../main/java/bisq/network/p2p/network/NetworkNode.java | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/p2p/src/main/java/bisq/network/p2p/network/NetworkNode.java b/p2p/src/main/java/bisq/network/p2p/network/NetworkNode.java index 5b81cbb1c98..c2695833f7a 100644 --- a/p2p/src/main/java/bisq/network/p2p/network/NetworkNode.java +++ b/p2p/src/main/java/bisq/network/p2p/network/NetworkNode.java @@ -21,6 +21,7 @@ import bisq.common.Timer; import bisq.common.UserThread; +import bisq.common.app.Capabilities; import bisq.common.proto.network.NetworkEnvelope; import bisq.common.proto.network.NetworkProtoResolver; import bisq.common.util.Utilities; @@ -496,4 +497,12 @@ private void printInboundConnections() { public NodeAddress getNodeAddress() { return nodeAddressProperty.get(); } + + public Optional findPeersCapabilities(NodeAddress nodeAddress) { + return getConfirmedConnections().stream() + .filter(c -> c.getPeersNodeAddressProperty().get() != null) + .filter(c -> c.getPeersNodeAddressProperty().get().equals(nodeAddress)) + .map(Connection::getCapabilities) + .findAny(); + } } From 7af16d7af35a4a820e6de8c093dac8b472ff1281 Mon Sep 17 00:00:00 2001 From: chimp1984 Date: Wed, 7 Oct 2020 14:47:49 -0500 Subject: [PATCH 13/59] Add getDateAsLong method, add setter for capabilities --- .../java/bisq/network/p2p/peers/peerexchange/Peer.java | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/p2p/src/main/java/bisq/network/p2p/peers/peerexchange/Peer.java b/p2p/src/main/java/bisq/network/p2p/peers/peerexchange/Peer.java index 8c716c6787a..a7681f909ce 100644 --- a/p2p/src/main/java/bisq/network/p2p/peers/peerexchange/Peer.java +++ b/p2p/src/main/java/bisq/network/p2p/peers/peerexchange/Peer.java @@ -42,7 +42,8 @@ public final class Peer implements HasCapabilities, NetworkPayload, PersistableP private final long date; @Setter transient private int failedConnectionAttempts = 0; - private final Capabilities capabilities = new Capabilities(); + @Setter + private Capabilities capabilities = new Capabilities(); public Peer(NodeAddress nodeAddress, @Nullable Capabilities supportedCapabilities) { this(nodeAddress, new Date().getTime(), supportedCapabilities); @@ -91,6 +92,10 @@ public Date getDate() { return new Date(date); } + public long getDateAsLong() { + return date; + } + @Override public void onChanged(Capabilities supportedCapabilities) { if (!supportedCapabilities.isEmpty()) { From 25bfe2d6adf8e0b9e3ecfa180fa05359696f2b54 Mon Sep 17 00:00:00 2001 From: chimp1984 Date: Wed, 7 Oct 2020 15:17:07 -0500 Subject: [PATCH 14/59] Add findPeersCapabilities method Impl. applyCapabilities Cleanups --- .../RepublishGovernanceDataHandler.java | 2 +- .../java/bisq/network/p2p/P2PService.java | 14 +- .../bisq/network/p2p/peers/PeerManager.java | 163 ++++++++++++------ 3 files changed, 118 insertions(+), 61 deletions(-) diff --git a/core/src/main/java/bisq/core/dao/governance/blindvote/network/RepublishGovernanceDataHandler.java b/core/src/main/java/bisq/core/dao/governance/blindvote/network/RepublishGovernanceDataHandler.java index 19192e543d0..ea73a102c26 100644 --- a/core/src/main/java/bisq/core/dao/governance/blindvote/network/RepublishGovernanceDataHandler.java +++ b/core/src/main/java/bisq/core/dao/governance/blindvote/network/RepublishGovernanceDataHandler.java @@ -170,7 +170,7 @@ private void connectToNextNode() { private void connectToAnyFullNode() { Capabilities required = new Capabilities(Capability.DAO_FULL_NODE); - List list = peerManager.getLivePeers(null).stream() + List list = peerManager.getLivePeers().stream() .filter(peer -> peer.getCapabilities().containsAll(required)) .collect(Collectors.toList()); diff --git a/p2p/src/main/java/bisq/network/p2p/P2PService.java b/p2p/src/main/java/bisq/network/p2p/P2PService.java index b127833fd4d..2004cf5e1b6 100644 --- a/p2p/src/main/java/bisq/network/p2p/P2PService.java +++ b/p2p/src/main/java/bisq/network/p2p/P2PService.java @@ -31,7 +31,6 @@ import bisq.network.p2p.peers.PeerManager; import bisq.network.p2p.peers.getdata.RequestDataManager; import bisq.network.p2p.peers.keepalive.KeepAliveManager; -import bisq.network.p2p.peers.peerexchange.Peer; import bisq.network.p2p.peers.peerexchange.PeerExchangeManager; import bisq.network.p2p.seed.SeedNodeRepository; import bisq.network.p2p.storage.HashMapChangedListener; @@ -46,6 +45,7 @@ import bisq.network.p2p.storage.payload.ProtectedStoragePayload; import bisq.common.UserThread; +import bisq.common.app.Capabilities; import bisq.common.app.Capability; import bisq.common.crypto.CryptoException; import bisq.common.crypto.KeyRing; @@ -671,17 +671,11 @@ private boolean capabilityRequiredAndCapabilityNotSupported(NodeAddress peersNod if (!(message instanceof CapabilityRequiringPayload)) return false; - Set allPeers = peerManager.getPersistedPeers(); - allPeers.addAll(peerManager.getReportedPeers()); - allPeers.addAll(peerManager.getLivePeers(null)); // We might have multiple entries of the same peer without the supportedCapabilities field set if we received // it from old versions, so we filter those. - Optional optionalPeer = allPeers.stream() - .filter(peer -> peer.getNodeAddress().equals(peersNodeAddress)) - .filter(peer -> !peer.getCapabilities().isEmpty()) - .findAny(); - if (optionalPeer.isPresent()) { - boolean result = optionalPeer.get().getCapabilities().containsAll(((CapabilityRequiringPayload) message).getRequiredCapabilities()); + Optional optionalCapabilities = peerManager.findPeersCapabilities(peersNodeAddress); + if (optionalCapabilities.isPresent()) { + boolean result = optionalCapabilities.get().containsAll(((CapabilityRequiringPayload) message).getRequiredCapabilities()); if (!result) log.warn("We don't send the message because the peer does not support the required capability. " + diff --git a/p2p/src/main/java/bisq/network/p2p/peers/PeerManager.java b/p2p/src/main/java/bisq/network/p2p/peers/PeerManager.java index 9ec74dc2eb9..a45e7a975a2 100644 --- a/p2p/src/main/java/bisq/network/p2p/peers/PeerManager.java +++ b/p2p/src/main/java/bisq/network/p2p/peers/PeerManager.java @@ -60,6 +60,8 @@ import javax.annotation.Nullable; +import static com.google.common.base.Preconditions.checkArgument; + @Slf4j public class PeerManager implements ConnectionListener, PersistedDataHost { @@ -78,9 +80,6 @@ public class PeerManager implements ConnectionListener, PersistedDataHost { // Age of what we consider connected peers still as live peers private static final long MAX_AGE_LIVE_PEERS = TimeUnit.MINUTES.toMillis(30); private static final boolean PRINT_REPORTED_PEERS_DETAILS = true; - @Setter - private boolean allowDisconnectSeedNodes; - private Set latestLivePeers = new HashSet<>(); /////////////////////////////////////////////////////////////////////////////////////////// @@ -102,19 +101,26 @@ public interface Listener { private final NetworkNode networkNode; private final ClockWatcher clockWatcher; - - private int maxConnections; private final Set seedNodeAddresses; - private final PersistenceManager persistenceManager; - private final PeerList peerList = new PeerList(); - private final HashSet persistedPeers = new HashSet<>(); - private final Set reportedPeers = new HashSet<>(); private final ClockWatcher.Listener listener; private final List listeners = new CopyOnWriteArrayList<>(); + + // Persistable peerList + private final PeerList peerList = new PeerList(); + // Peers we had persisted TODO use peerList instead + @Getter + private final Set persistedPeers = new HashSet<>(); + // Peers we got reported from other peers + @Getter + private final Set reportedPeers = new HashSet<>(); + // Peer of last 30 min. + private final Set latestLivePeers = new HashSet<>(); + private Timer checkMaxConnectionsTimer; private boolean stopped; private boolean lostAllConnections; + private int maxConnections; @Getter private int minConnections; @@ -122,6 +128,8 @@ public interface Listener { private int maxConnectionsPeer; private int maxConnectionsNonDirect; private int maxConnectionsAbsolute; + @Setter + private boolean allowDisconnectSeedNodes; /////////////////////////////////////////////////////////////////////////////////////////// @@ -132,8 +140,8 @@ public interface Listener { public PeerManager(NetworkNode networkNode, SeedNodeRepository seedNodeRepository, ClockWatcher clockWatcher, - @Named(Config.MAX_CONNECTIONS) int maxConnections, - PersistenceManager persistenceManager) { + PersistenceManager persistenceManager, + @Named(Config.MAX_CONNECTIONS) int maxConnections) { this.networkNode = networkNode; this.seedNodeAddresses = new HashSet<>(seedNodeRepository.getSeedNodeAddresses()); this.clockWatcher = clockWatcher; @@ -156,7 +164,8 @@ public void onMinuteTick() { @Override public void onAwakeFromStandby(long missedMs) { - // TODO is "stopped = false;" correct? + // We got probably stopped set to true when we got a longer interruption (e.g. lost all connections), + // now we get awake again, so set stopped to false. stopped = false; listeners.forEach(Listener::onAwakeFromStandby); } @@ -213,15 +222,40 @@ public boolean peerHasCapability(NodeAddress peersNodeAddress, Capability capabi .orElse(false); } - // TODO get Capabilities from peers - public Optional findPeersCapabilities(NodeAddress peersNodeAddress) { - return networkNode.getConfirmedConnections().stream() - .filter(c -> c.getPeersNodeAddressProperty().get() != null) - .filter(c -> c.getPeersNodeAddressProperty().get().equals(peersNodeAddress)) - .map(Connection::getCapabilities) + public Optional findPeersCapabilities(NodeAddress nodeAddress) { + // We look up first our connections as that is our own data. If not found there we look up the peers which + // include reported peers. + Optional optionalCapabilities = networkNode.findPeersCapabilities(nodeAddress); + if (optionalCapabilities.isPresent()) { + return optionalCapabilities; + } + + // Reported peers are not trusted data. We could get capabilities which miss the + // peers real capability or we could get maliciously altered capabilities telling us the peer supports a + // capability which is in fact not supported. This could lead to connection loss as we might send data not + // recognized by the peer. As we register a listener on connection if we don't have set the capability from our + // own sources we would get it fixed as soon we have a connection with that peer, rendering such an attack + // inefficient. + // Also this risk is only for not updated peers, so in case that would be abused for an + // attack all users have a strong incentive to update ;-). + return getAllPeers().stream() + .filter(peer -> peer.getNodeAddress().equals(nodeAddress)) + .findAny().map(Peer::getCapabilities); + } + + public Optional findPeer(NodeAddress peersNodeAddress) { + return getAllPeers().stream() + .filter(peer -> peer.getNodeAddress().equals(peersNodeAddress)) .findAny(); } + public Set getAllPeers() { + Set allPeers = new HashSet<>(getLivePeers()); + allPeers.addAll(persistedPeers); + allPeers.addAll(reportedPeers); + return allPeers; + } + /////////////////////////////////////////////////////////////////////////////////////////// // ConnectionListener implementation /////////////////////////////////////////////////////////////////////////////////////////// @@ -460,15 +494,11 @@ private void removeTooOldReportedPeers() { reportedPeersToRemove.forEach(this::removeReportedPeer); } - public Set getReportedPeers() { - return reportedPeers; - } - public void addToReportedPeers(Set reportedPeersToAdd, Connection connection, - Capabilities supportedCapabilities) { + Capabilities capabilities) { - //TODO apply supportedCapabilities + applyCapabilities(connection, capabilities); printNewReportedPeers(reportedPeersToAdd); @@ -479,8 +509,7 @@ public void addToReportedPeers(Set reportedPeersToAdd, persistedPeers.addAll(reportedPeersToAdd); purgePersistedPeersIfExceeds(); - peerList.setAll(persistedPeers); - persistenceManager.requestPersistence(); + requestPersistence(); printReportedPeers(); } else { @@ -491,6 +520,17 @@ public void addToReportedPeers(Set reportedPeersToAdd, } } + private void applyCapabilities(Connection connection, Capabilities capabilities) { + if (capabilities != null && !capabilities.isEmpty()) { + connection.getPeersNodeAddressOptional().ifPresent(nodeAddress -> { + getAllPeers().stream() + .filter(peer -> peer.getNodeAddress().equals(nodeAddress)) + .forEach(peer -> peer.setCapabilities(capabilities)); + }); + requestPersistence(); + } + } + private void purgeReportedPeersIfExceeds() { int size = reportedPeers.size(); if (size > MAX_REPORTED_PEERS) { @@ -530,7 +570,7 @@ private void printNewReportedPeers(Set reportedPeers) { if (PRINT_REPORTED_PEERS_DETAILS) { StringBuilder result = new StringBuilder("We received new reportedPeers:"); List reportedPeersClone = new ArrayList<>(reportedPeers); - reportedPeersClone.stream().forEach(e -> result.append("\n\t").append(e)); + reportedPeersClone.forEach(e -> result.append("\n\t").append(e)); log.trace(result.toString()); } log.debug("Number of new arrived reported peers: {}", reportedPeers.size()); @@ -544,23 +584,28 @@ private void printNewReportedPeers(Set reportedPeers) { private boolean removePersistedPeer(Peer persistedPeer) { if (persistedPeers.contains(persistedPeer)) { persistedPeers.remove(persistedPeer); - peerList.setAll(persistedPeers); - persistenceManager.requestPersistence(); + requestPersistence(); return true; } else { return false; } } + private void requestPersistence() { + peerList.setAll(persistedPeers); + persistenceManager.requestPersistence(); + } + @SuppressWarnings("UnusedReturnValue") private boolean removePersistedPeer(NodeAddress nodeAddress) { - Optional persistedPeerOptional = getPersistedPeerOptional(nodeAddress); - return persistedPeerOptional.isPresent() && removePersistedPeer(persistedPeerOptional.get()); + Optional optionalPersistedPeer = getOptionalPersistedPeer(nodeAddress); + return optionalPersistedPeer.isPresent() && removePersistedPeer(optionalPersistedPeer.get()); } - private Optional getPersistedPeerOptional(NodeAddress nodeAddress) { + private Optional getOptionalPersistedPeer(NodeAddress nodeAddress) { return persistedPeers.stream() - .filter(e -> e.getNodeAddress().equals(nodeAddress)).findAny(); + .filter(e -> e.getNodeAddress().equals(nodeAddress)) + .findAny(); } private void removeTooOldPersistedPeers() { @@ -590,10 +635,6 @@ private void purgePersistedPeersIfExceeds() { } } - public Set getPersistedPeers() { - return persistedPeers; - } - /////////////////////////////////////////////////////////////////////////////////////////// // Misc @@ -644,7 +685,7 @@ public void handleConnectionFault(NodeAddress nodeAddress, @Nullable Connection log.debug("handleConnectionFault called: nodeAddress=" + nodeAddress); boolean doRemovePersistedPeer = false; removeReportedPeer(nodeAddress); - Optional persistedPeerOptional = getPersistedPeerOptional(nodeAddress); + Optional persistedPeerOptional = getOptionalPersistedPeer(nodeAddress); if (persistedPeerOptional.isPresent()) { Peer persistedPeer = persistedPeerOptional.get(); persistedPeer.increaseFailedConnectionAttempts(); @@ -674,17 +715,27 @@ public void shutDownConnection(NodeAddress peersNodeAddress, CloseConnectionReas // Delivers the live peers from the last 30 min (MAX_AGE_LIVE_PEERS) // We include older peers to avoid risks for network partitioning - public Set getLivePeers(NodeAddress excludedNodeAddress) { + public Set getLivePeers() { + return getLivePeers(null); + } + + public Set getLivePeers(@Nullable NodeAddress excludedNodeAddress) { int oldNumLatestLivePeers = latestLivePeers.size(); - Set currentLivePeers = new HashSet<>(getConnectedReportedPeers().stream() + + Set peers = new HashSet<>(latestLivePeers); + Set currentLivePeers = getConnectedReportedPeers().stream() .filter(e -> !isSeedNode(e)) .filter(e -> !e.getNodeAddress().equals(excludedNodeAddress)) - .collect(Collectors.toSet())); - latestLivePeers.addAll(currentLivePeers); + .collect(Collectors.toSet()); + peers.addAll(currentLivePeers); + long maxAge = new Date().getTime() - MAX_AGE_LIVE_PEERS; - latestLivePeers = latestLivePeers.stream() - .filter(peer -> peer.getDate().getTime() > maxAge) + latestLivePeers.clear(); + Set recentPeers = peers.stream() + .filter(peer -> peer.getDateAsLong() > maxAge) .collect(Collectors.toSet()); + latestLivePeers.addAll(recentPeers); + if (oldNumLatestLivePeers != latestLivePeers.size()) log.info("Num of latestLivePeers={}", latestLivePeers.size()); return latestLivePeers; @@ -703,17 +754,29 @@ private Set getConnectedReportedPeers() { // If we have a new connection the supportedCapabilities is empty. // We lookup if we have already stored the supportedCapabilities at the persisted or reported peers // and if so we use that. + Optional peersNodeAddressOptional = connection.getPeersNodeAddressOptional(); + checkArgument(peersNodeAddressOptional.isPresent()); // getConfirmedConnections delivers only connections where we know the address + boolean getCapabilitiesFromConnection = !supportedCapabilities.isEmpty(); if (supportedCapabilities.isEmpty()) { + // If not set we look up if we got the Capabilities set from any of the reported or persisted peers Set allPeers = new HashSet<>(getPersistedPeers()); allPeers.addAll(getReportedPeers()); - Optional ourPeer = allPeers.stream().filter(peer -> peer.getNodeAddress().equals(connection.getPeersNodeAddressOptional().get())) + Optional ourPeer = allPeers.stream() + .filter(peer -> peer.getNodeAddress().equals(peersNodeAddressOptional.get())) .filter(peer -> !peer.getCapabilities().isEmpty()) .findAny(); - if (ourPeer.isPresent()) + if (ourPeer.isPresent()) { supportedCapabilities = new Capabilities(ourPeer.get().getCapabilities()); + } + } + Peer peer = new Peer(peersNodeAddressOptional.get(), supportedCapabilities); + + // If we only got the capabilities from a reported peer of did not get any we add a listener, + // so once we get a connection with that peer and exchange a message containing the capabilities + // we get set the capabilities + if (!getCapabilitiesFromConnection) { + connection.addWeakCapabilitiesListener(peer); } - Peer peer = new Peer(connection.getPeersNodeAddressOptional().get(), supportedCapabilities); - connection.addWeakCapabilitiesListener(peer); return peer; }) .collect(Collectors.toSet()); @@ -730,8 +793,8 @@ private void printConnectedPeers() { if (!networkNode.getConfirmedConnections().isEmpty()) { StringBuilder result = new StringBuilder("\n\n------------------------------------------------------------\n" + "Connected peers for node " + networkNode.getNodeAddress() + ":"); - networkNode.getConfirmedConnections().stream().forEach(e -> result.append("\n") - .append(e.getPeersNodeAddressOptional().get()).append(" ").append(e.getPeerType())); + networkNode.getConfirmedConnections().forEach(e -> result.append("\n") + .append(e.getPeersNodeAddressOptional()).append(" ").append(e.getPeerType())); result.append("\n------------------------------------------------------------\n"); log.debug(result.toString()); } From 25526750a81dffd2b8c75fae4beb575a6f27b035 Mon Sep 17 00:00:00 2001 From: chimp1984 Date: Wed, 7 Oct 2020 15:22:50 -0500 Subject: [PATCH 15/59] Replace persistedPeers with peerList --- .../bisq/network/p2p/peers/PeerManager.java | 30 ++++++++++--------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/p2p/src/main/java/bisq/network/p2p/peers/PeerManager.java b/p2p/src/main/java/bisq/network/p2p/peers/PeerManager.java index a45e7a975a2..57641fd2d23 100644 --- a/p2p/src/main/java/bisq/network/p2p/peers/PeerManager.java +++ b/p2p/src/main/java/bisq/network/p2p/peers/PeerManager.java @@ -43,6 +43,7 @@ import com.google.common.annotations.VisibleForTesting; import java.util.ArrayList; +import java.util.Collection; import java.util.Comparator; import java.util.Date; import java.util.HashSet; @@ -108,9 +109,6 @@ public interface Listener { // Persistable peerList private final PeerList peerList = new PeerList(); - // Peers we had persisted TODO use peerList instead - @Getter - private final Set persistedPeers = new HashSet<>(); // Peers we got reported from other peers @Getter private final Set reportedPeers = new HashSet<>(); @@ -188,7 +186,7 @@ public void shutDown() { public void readPersisted() { PeerList persisted = persistenceManager.getPersisted(); if (persisted != null) { - this.persistedPeers.addAll(persisted.getList()); + peerList.setAll(persisted.getList()); } } @@ -251,11 +249,16 @@ public Optional findPeer(NodeAddress peersNodeAddress) { public Set getAllPeers() { Set allPeers = new HashSet<>(getLivePeers()); - allPeers.addAll(persistedPeers); + allPeers.addAll(peerList.getList()); allPeers.addAll(reportedPeers); return allPeers; } + public Collection getPersistedPeers() { + return peerList.getList(); + } + + /////////////////////////////////////////////////////////////////////////////////////////// // ConnectionListener implementation /////////////////////////////////////////////////////////////////////////////////////////// @@ -507,7 +510,7 @@ public void addToReportedPeers(Set reportedPeersToAdd, reportedPeers.addAll(reportedPeersToAdd); purgeReportedPeersIfExceeds(); - persistedPeers.addAll(reportedPeersToAdd); + peerList.getList().addAll(reportedPeersToAdd); purgePersistedPeersIfExceeds(); requestPersistence(); @@ -582,8 +585,8 @@ private void printNewReportedPeers(Set reportedPeers) { /////////////////////////////////////////////////////////////////////////////////////////// private boolean removePersistedPeer(Peer persistedPeer) { - if (persistedPeers.contains(persistedPeer)) { - persistedPeers.remove(persistedPeer); + if (peerList.getList().contains(persistedPeer)) { + peerList.getList().remove(persistedPeer); requestPersistence(); return true; } else { @@ -592,7 +595,6 @@ private boolean removePersistedPeer(Peer persistedPeer) { } private void requestPersistence() { - peerList.setAll(persistedPeers); persistenceManager.requestPersistence(); } @@ -603,27 +605,27 @@ private boolean removePersistedPeer(NodeAddress nodeAddress) { } private Optional getOptionalPersistedPeer(NodeAddress nodeAddress) { - return persistedPeers.stream() + return peerList.getList().stream() .filter(e -> e.getNodeAddress().equals(nodeAddress)) .findAny(); } private void removeTooOldPersistedPeers() { - Set persistedPeersToRemove = persistedPeers.stream() + Set persistedPeersToRemove = peerList.getList().stream() .filter(reportedPeer -> new Date().getTime() - reportedPeer.getDate().getTime() > MAX_AGE) .collect(Collectors.toSet()); persistedPeersToRemove.forEach(this::removePersistedPeer); } private void purgePersistedPeersIfExceeds() { - int size = persistedPeers.size(); + int size = peerList.getList().size(); int limit = MAX_PERSISTED_PEERS; if (size > limit) { log.trace("We have already {} persisted peers which exceeds our limit of {}." + "We remove random peers from the persisted peers list.", size, limit); int diff = size - limit; - List list = new ArrayList<>(persistedPeers); - // we dont use sorting by lastActivityDate to avoid attack vectors and keep it more random + List list = new ArrayList<>(peerList.getList()); + // we don't use sorting by lastActivityDate to avoid attack vectors and keep it more random for (int i = 0; i < diff; i++) { if (!list.isEmpty()) { Peer toRemove = list.remove(new Random().nextInt(list.size())); From bf674ea0cf3c67b4dcfd6a2ae437bdd30ed96066 Mon Sep 17 00:00:00 2001 From: chimp1984 Date: Wed, 7 Oct 2020 15:34:31 -0500 Subject: [PATCH 16/59] Use getPersistedPeers for peerList.getList() calls Rename getOptionalPersistedPeer to findPersistedPeer Improve getConnectedReportedPeers method --- .../bisq/network/p2p/peers/PeerManager.java | 52 ++++++++++--------- 1 file changed, 27 insertions(+), 25 deletions(-) diff --git a/p2p/src/main/java/bisq/network/p2p/peers/PeerManager.java b/p2p/src/main/java/bisq/network/p2p/peers/PeerManager.java index 57641fd2d23..8fa179a6f49 100644 --- a/p2p/src/main/java/bisq/network/p2p/peers/PeerManager.java +++ b/p2p/src/main/java/bisq/network/p2p/peers/PeerManager.java @@ -112,7 +112,7 @@ public interface Listener { // Peers we got reported from other peers @Getter private final Set reportedPeers = new HashSet<>(); - // Peer of last 30 min. + // Most recent peers with activity date of last 30 min. private final Set latestLivePeers = new HashSet<>(); private Timer checkMaxConnectionsTimer; @@ -249,7 +249,7 @@ public Optional findPeer(NodeAddress peersNodeAddress) { public Set getAllPeers() { Set allPeers = new HashSet<>(getLivePeers()); - allPeers.addAll(peerList.getList()); + allPeers.addAll(getPersistedPeers()); allPeers.addAll(reportedPeers); return allPeers; } @@ -510,7 +510,7 @@ public void addToReportedPeers(Set reportedPeersToAdd, reportedPeers.addAll(reportedPeersToAdd); purgeReportedPeersIfExceeds(); - peerList.getList().addAll(reportedPeersToAdd); + getPersistedPeers().addAll(reportedPeersToAdd); purgePersistedPeersIfExceeds(); requestPersistence(); @@ -585,8 +585,8 @@ private void printNewReportedPeers(Set reportedPeers) { /////////////////////////////////////////////////////////////////////////////////////////// private boolean removePersistedPeer(Peer persistedPeer) { - if (peerList.getList().contains(persistedPeer)) { - peerList.getList().remove(persistedPeer); + if (getPersistedPeers().contains(persistedPeer)) { + getPersistedPeers().remove(persistedPeer); requestPersistence(); return true; } else { @@ -600,31 +600,31 @@ private void requestPersistence() { @SuppressWarnings("UnusedReturnValue") private boolean removePersistedPeer(NodeAddress nodeAddress) { - Optional optionalPersistedPeer = getOptionalPersistedPeer(nodeAddress); + Optional optionalPersistedPeer = findPersistedPeer(nodeAddress); return optionalPersistedPeer.isPresent() && removePersistedPeer(optionalPersistedPeer.get()); } - private Optional getOptionalPersistedPeer(NodeAddress nodeAddress) { - return peerList.getList().stream() + private Optional findPersistedPeer(NodeAddress nodeAddress) { + return getPersistedPeers().stream() .filter(e -> e.getNodeAddress().equals(nodeAddress)) .findAny(); } private void removeTooOldPersistedPeers() { - Set persistedPeersToRemove = peerList.getList().stream() + Set persistedPeersToRemove = getPersistedPeers().stream() .filter(reportedPeer -> new Date().getTime() - reportedPeer.getDate().getTime() > MAX_AGE) .collect(Collectors.toSet()); persistedPeersToRemove.forEach(this::removePersistedPeer); } private void purgePersistedPeersIfExceeds() { - int size = peerList.getList().size(); + int size = getPersistedPeers().size(); int limit = MAX_PERSISTED_PEERS; if (size > limit) { log.trace("We have already {} persisted peers which exceeds our limit of {}." + "We remove random peers from the persisted peers list.", size, limit); int diff = size - limit; - List list = new ArrayList<>(peerList.getList()); + List list = new ArrayList<>(getPersistedPeers()); // we don't use sorting by lastActivityDate to avoid attack vectors and keep it more random for (int i = 0; i < diff; i++) { if (!list.isEmpty()) { @@ -687,7 +687,7 @@ public void handleConnectionFault(NodeAddress nodeAddress, @Nullable Connection log.debug("handleConnectionFault called: nodeAddress=" + nodeAddress); boolean doRemovePersistedPeer = false; removeReportedPeer(nodeAddress); - Optional persistedPeerOptional = getOptionalPersistedPeer(nodeAddress); + Optional persistedPeerOptional = findPersistedPeer(nodeAddress); if (persistedPeerOptional.isPresent()) { Peer persistedPeer = persistedPeerOptional.get(); persistedPeer.increaseFailedConnectionAttempts(); @@ -758,25 +758,27 @@ private Set getConnectedReportedPeers() { // and if so we use that. Optional peersNodeAddressOptional = connection.getPeersNodeAddressOptional(); checkArgument(peersNodeAddressOptional.isPresent()); // getConfirmedConnections delivers only connections where we know the address - boolean getCapabilitiesFromConnection = !supportedCapabilities.isEmpty(); - if (supportedCapabilities.isEmpty()) { - // If not set we look up if we got the Capabilities set from any of the reported or persisted peers - Set allPeers = new HashSet<>(getPersistedPeers()); - allPeers.addAll(getReportedPeers()); - Optional ourPeer = allPeers.stream() - .filter(peer -> peer.getNodeAddress().equals(peersNodeAddressOptional.get())) + NodeAddress peersNodeAddress = peersNodeAddressOptional.get(); + boolean capabilitiesNotFoundInConnection = supportedCapabilities.isEmpty(); + if (capabilitiesNotFoundInConnection) { + // If not found in connection we look up if we got the Capabilities set from any of the + // reported or persisted peers + Set persistedAndReported = new HashSet<>(getPersistedPeers()); + persistedAndReported.addAll(getReportedPeers()); + Optional candidate = persistedAndReported.stream() + .filter(peer -> peer.getNodeAddress().equals(peersNodeAddress)) .filter(peer -> !peer.getCapabilities().isEmpty()) .findAny(); - if (ourPeer.isPresent()) { - supportedCapabilities = new Capabilities(ourPeer.get().getCapabilities()); + if (candidate.isPresent()) { + supportedCapabilities = new Capabilities(candidate.get().getCapabilities()); } } - Peer peer = new Peer(peersNodeAddressOptional.get(), supportedCapabilities); + Peer peer = new Peer(peersNodeAddress, supportedCapabilities); - // If we only got the capabilities from a reported peer of did not get any we add a listener, + // If we did not found the capability from our own connection we add a listener, // so once we get a connection with that peer and exchange a message containing the capabilities - // we get set the capabilities - if (!getCapabilitiesFromConnection) { + // we get set the capabilities. + if (capabilitiesNotFoundInConnection) { connection.addWeakCapabilitiesListener(peer); } return peer; From cf0693098f5aee9d2a795ced5a241652882e0709 Mon Sep 17 00:00:00 2001 From: chimp1984 <54558767+chimp1984@users.noreply.github.com> Date: Wed, 7 Oct 2020 18:05:39 -0500 Subject: [PATCH 17/59] Update common/src/main/java/bisq/common/app/Capability.java Co-authored-by: sqrrm --- common/src/main/java/bisq/common/app/Capability.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/src/main/java/bisq/common/app/Capability.java b/common/src/main/java/bisq/common/app/Capability.java index 9379c9dd392..8ffb042844d 100644 --- a/common/src/main/java/bisq/common/app/Capability.java +++ b/common/src/main/java/bisq/common/app/Capability.java @@ -41,5 +41,5 @@ public enum Capability { MEDIATION, // Supports mediation feature REFUND_AGENT, // Supports refund agents TRADE_STATISTICS_HASH_UPDATE, // We changed the hash method in 1.2.0 and that requires update to 1.2.2 for handling it correctly, otherwise the seed nodes have to process too much data. - NO_ADDRESS_PRE_FIX // At 1.4.0 we removed the prefix filter for mailbox messages. If a peer has that capability we do not sent the prefix. + NO_ADDRESS_PRE_FIX // At 1.4.0 we removed the prefix filter for mailbox messages. If a peer has that capability we do not sent the prefix. } From 983f610e730c2417773220deae779b6bfe014cc0 Mon Sep 17 00:00:00 2001 From: chimp1984 <54558767+chimp1984@users.noreply.github.com> Date: Wed, 7 Oct 2020 18:14:49 -0500 Subject: [PATCH 18/59] Update p2p/src/main/java/bisq/network/p2p/network/Connection.java Co-authored-by: sqrrm --- p2p/src/main/java/bisq/network/p2p/network/Connection.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/p2p/src/main/java/bisq/network/p2p/network/Connection.java b/p2p/src/main/java/bisq/network/p2p/network/Connection.java index 6a29c7e0343..dca03c133de 100644 --- a/p2p/src/main/java/bisq/network/p2p/network/Connection.java +++ b/p2p/src/main/java/bisq/network/p2p/network/Connection.java @@ -900,11 +900,10 @@ protected boolean handleSupportedCapabilitiesMessage(NetworkEnvelope networkEnve @Nullable private NodeAddress getSenderNodeAddress(NetworkEnvelope networkEnvelope) { - return getPeersNodeAddressOptional().isPresent() ? - getPeersNodeAddressOptional().get() : + return getPeersNodeAddressOptional().orElse( networkEnvelope instanceof SendersNodeAddressMessage ? ((SendersNodeAddressMessage) networkEnvelope).getSenderNodeAddress() : - null; + null); } private String getSenderNodeAddressAsString(NetworkEnvelope networkEnvelope) { From 31e7e2655792b656ee6747919c7fd449a2d96850 Mon Sep 17 00:00:00 2001 From: chimp1984 Date: Wed, 7 Oct 2020 18:16:34 -0500 Subject: [PATCH 19/59] Fix incorrect handling of decryptedEntries size --- p2p/src/main/java/bisq/network/p2p/P2PService.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/p2p/src/main/java/bisq/network/p2p/P2PService.java b/p2p/src/main/java/bisq/network/p2p/P2PService.java index 2004cf5e1b6..0e60769ea66 100644 --- a/p2p/src/main/java/bisq/network/p2p/P2PService.java +++ b/p2p/src/main/java/bisq/network/p2p/P2PService.java @@ -450,9 +450,11 @@ public void onAdded(Collection protectedStorageEntries) { } private void processSingleMailboxEntry(Collection protectedMailboxStorageEntries) { + checkArgument(protectedMailboxStorageEntries.size() == 1); var decryptedEntries = new ArrayList<>(getDecryptedEntries(protectedMailboxStorageEntries)); - checkArgument(decryptedEntries.size() == 1); - storeMailboxDataAndNotifyListeners(decryptedEntries.get(0)); + if (protectedMailboxStorageEntries.size() == 1) { + storeMailboxDataAndNotifyListeners(decryptedEntries.get(0)); + } } // We run the batch processing of all mailbox messages we have received at startup in a thread to not block the UI. From 4575516d19293fe535eddbd54ef9146d69e6e349 Mon Sep 17 00:00:00 2001 From: chimp1984 Date: Wed, 7 Oct 2020 18:16:52 -0500 Subject: [PATCH 20/59] Refactor: Return early --- .../java/bisq/network/p2p/peers/PeerManager.java | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/p2p/src/main/java/bisq/network/p2p/peers/PeerManager.java b/p2p/src/main/java/bisq/network/p2p/peers/PeerManager.java index 8fa179a6f49..167d01a7da9 100644 --- a/p2p/src/main/java/bisq/network/p2p/peers/PeerManager.java +++ b/p2p/src/main/java/bisq/network/p2p/peers/PeerManager.java @@ -524,14 +524,16 @@ public void addToReportedPeers(Set reportedPeersToAdd, } private void applyCapabilities(Connection connection, Capabilities capabilities) { - if (capabilities != null && !capabilities.isEmpty()) { - connection.getPeersNodeAddressOptional().ifPresent(nodeAddress -> { - getAllPeers().stream() - .filter(peer -> peer.getNodeAddress().equals(nodeAddress)) - .forEach(peer -> peer.setCapabilities(capabilities)); - }); - requestPersistence(); + if (capabilities == null || capabilities.isEmpty()) { + return; } + + connection.getPeersNodeAddressOptional().ifPresent(nodeAddress -> { + getAllPeers().stream() + .filter(peer -> peer.getNodeAddress().equals(nodeAddress)) + .forEach(peer -> peer.setCapabilities(capabilities)); + }); + requestPersistence(); } private void purgeReportedPeersIfExceeds() { From ed960aba3fc7a60233c08402cff2ad4d73bc7b01 Mon Sep 17 00:00:00 2001 From: chimp1984 Date: Wed, 7 Oct 2020 18:44:04 -0500 Subject: [PATCH 21/59] Refactor: Rearrange code, remove unused methods, renamings (no functional change) --- .../bisq/network/p2p/peers/PeerManager.java | 454 +++++++++--------- 1 file changed, 216 insertions(+), 238 deletions(-) diff --git a/p2p/src/main/java/bisq/network/p2p/peers/PeerManager.java b/p2p/src/main/java/bisq/network/p2p/peers/PeerManager.java index 167d01a7da9..22712ffb3d9 100644 --- a/p2p/src/main/java/bisq/network/p2p/peers/PeerManager.java +++ b/p2p/src/main/java/bisq/network/p2p/peers/PeerManager.java @@ -64,7 +64,7 @@ import static com.google.common.base.Preconditions.checkArgument; @Slf4j -public class PeerManager implements ConnectionListener, PersistedDataHost { +public final class PeerManager implements ConnectionListener, PersistedDataHost { /////////////////////////////////////////////////////////////////////////////////////////// // Static @@ -104,7 +104,7 @@ public interface Listener { private final ClockWatcher clockWatcher; private final Set seedNodeAddresses; private final PersistenceManager persistenceManager; - private final ClockWatcher.Listener listener; + private final ClockWatcher.Listener clockWatcherListener; private final List listeners = new CopyOnWriteArrayList<>(); // Persistable peerList @@ -151,7 +151,7 @@ public PeerManager(NetworkNode networkNode, setConnectionLimits(maxConnections); // we check if app was idle for more then 5 sec. - listener = new ClockWatcher.Listener() { + clockWatcherListener = new ClockWatcher.Listener() { @Override public void onSecondTick() { } @@ -168,18 +168,18 @@ public void onAwakeFromStandby(long missedMs) { listeners.forEach(Listener::onAwakeFromStandby); } }; - clockWatcher.addListener(listener); + clockWatcher.addListener(clockWatcherListener); } public void shutDown() { networkNode.removeConnectionListener(this); - clockWatcher.removeListener(listener); + clockWatcher.removeListener(clockWatcherListener); stopCheckMaxConnectionsTimer(); } /////////////////////////////////////////////////////////////////////////////////////////// - // API + // PersistedDataHost implementation /////////////////////////////////////////////////////////////////////////////////////////// @Override @@ -190,57 +190,123 @@ public void readPersisted() { } } - public int getMaxConnections() { - return maxConnectionsAbsolute; + + /////////////////////////////////////////////////////////////////////////////////////////// + // ConnectionListener implementation + /////////////////////////////////////////////////////////////////////////////////////////// + + @Override + public void onConnection(Connection connection) { + if (isSeedNode(connection)) { + connection.setPeerType(Connection.PeerType.SEED_NODE); + } + + doHouseKeeping(); + + if (lostAllConnections) { + lostAllConnections = false; + stopped = false; + listeners.forEach(Listener::onNewConnectionAfterAllConnectionsLost); + } } - public void addListener(Listener listener) { - listeners.add(listener); + @Override + public void onDisconnect(CloseConnectionReason closeConnectionReason, Connection connection) { + log.info("onDisconnect called: nodeAddress={}, closeConnectionReason={}", + connection.getPeersNodeAddressOptional(), closeConnectionReason); + handleConnectionFault(connection); + lostAllConnections = networkNode.getAllConnections().isEmpty(); + if (lostAllConnections) { + stopped = true; + log.warn("\n------------------------------------------------------------\n" + + "All connections lost\n" + + "------------------------------------------------------------"); + listeners.forEach(Listener::onAllConnectionsLost); + } + maybeRemoveBannedPeer(closeConnectionReason, connection); } - public void removeListener(Listener listener) { - listeners.remove(listener); + @Override + public void onError(Throwable throwable) { } - // Modify this to change the relationships between connection limits. - // maxConnections default 12 - private void setConnectionLimits(int maxConnections) { - this.maxConnections = maxConnections; // app node 12; seedNode 30 - minConnections = Math.max(1, (int) Math.round(maxConnections * 0.7)); // app node 1-8; seedNode 21 - disconnectFromSeedNode = maxConnections; // app node 12; seedNode 30 - maxConnectionsPeer = Math.max(4, (int) Math.round(maxConnections * 1.3)); // app node 16; seedNode 39 - maxConnectionsNonDirect = Math.max(8, (int) Math.round(maxConnections * 1.7)); // app node 20; seedNode 51 - maxConnectionsAbsolute = Math.max(12, (int) Math.round(maxConnections * 2.5)); // app node 30; seedNode 66 + + /////////////////////////////////////////////////////////////////////////////////////////// + // Connection + /////////////////////////////////////////////////////////////////////////////////////////// + + public boolean hasSufficientConnections() { + return networkNode.getConfirmedConnections().size() >= minConnections; } + // Checks if that connection has the peers node address + public boolean isConfirmed(NodeAddress nodeAddress) { + return networkNode.getNodeAddressesOfConfirmedConnections().contains(nodeAddress); + } - public boolean peerHasCapability(NodeAddress peersNodeAddress, Capability capability) { - return findPeersCapabilities(peersNodeAddress) - .map(capabilities -> capabilities.contains(capability)) - .orElse(false); + public void handleConnectionFault(Connection connection) { + connection.getPeersNodeAddressOptional().ifPresent(nodeAddress -> handleConnectionFault(nodeAddress, connection)); } - public Optional findPeersCapabilities(NodeAddress nodeAddress) { - // We look up first our connections as that is our own data. If not found there we look up the peers which - // include reported peers. - Optional optionalCapabilities = networkNode.findPeersCapabilities(nodeAddress); - if (optionalCapabilities.isPresent()) { - return optionalCapabilities; + public void handleConnectionFault(NodeAddress nodeAddress) { + handleConnectionFault(nodeAddress, null); + } + + public void handleConnectionFault(NodeAddress nodeAddress, @Nullable Connection connection) { + log.debug("handleConnectionFault called: nodeAddress=" + nodeAddress); + boolean doRemovePersistedPeer = false; + removeReportedPeer(nodeAddress); + Optional persistedPeerOptional = findPersistedPeer(nodeAddress); + if (persistedPeerOptional.isPresent()) { + Peer persistedPeer = persistedPeerOptional.get(); + persistedPeer.increaseFailedConnectionAttempts(); + doRemovePersistedPeer = persistedPeer.tooManyFailedConnectionAttempts(); } + doRemovePersistedPeer = doRemovePersistedPeer || (connection != null && connection.getRuleViolation() != null); - // Reported peers are not trusted data. We could get capabilities which miss the - // peers real capability or we could get maliciously altered capabilities telling us the peer supports a - // capability which is in fact not supported. This could lead to connection loss as we might send data not - // recognized by the peer. As we register a listener on connection if we don't have set the capability from our - // own sources we would get it fixed as soon we have a connection with that peer, rendering such an attack - // inefficient. - // Also this risk is only for not updated peers, so in case that would be abused for an - // attack all users have a strong incentive to update ;-). - return getAllPeers().stream() - .filter(peer -> peer.getNodeAddress().equals(nodeAddress)) - .findAny().map(Peer::getCapabilities); + if (doRemovePersistedPeer) + removePersistedPeer(nodeAddress); + else + removeTooOldPersistedPeers(); } + public boolean isSeedNode(Connection connection) { + //TODO + return connection.hasPeersNodeAddress() && seedNodeAddresses.contains(connection.getPeersNodeAddressOptional().get()); + } + + public boolean isSelf(NodeAddress nodeAddress) { + return nodeAddress.equals(networkNode.getNodeAddress()); + } + + private boolean isSeedNode(Peer peer) { + return seedNodeAddresses.contains(peer.getNodeAddress()); + } + + public boolean isSeedNode(NodeAddress nodeAddress) { + return seedNodeAddresses.contains(nodeAddress); + } + + //TODO rename + public boolean isNodeBanned(CloseConnectionReason closeConnectionReason, Connection connection) { + return closeConnectionReason == CloseConnectionReason.PEER_BANNED && + connection.getPeersNodeAddressOptional().isPresent(); + } + + private void maybeRemoveBannedPeer(CloseConnectionReason closeConnectionReason, Connection connection) { + if (connection.getPeersNodeAddressOptional().isPresent() && isNodeBanned(closeConnectionReason, connection)) { + NodeAddress nodeAddress = connection.getPeersNodeAddressOptional().get(); + seedNodeAddresses.remove(nodeAddress); + removePersistedPeer(nodeAddress); + removeReportedPeer(nodeAddress); + } + } + + + /////////////////////////////////////////////////////////////////////////////////////////// + // Peer + /////////////////////////////////////////////////////////////////////////////////////////// + public Optional findPeer(NodeAddress peersNodeAddress) { return getAllPeers().stream() .filter(peer -> peer.getNodeAddress().equals(peersNodeAddress)) @@ -258,71 +324,102 @@ public Collection getPersistedPeers() { return peerList.getList(); } + public void addToReportedPeers(Set reportedPeersToAdd, + Connection connection, + Capabilities capabilities) { + applyCapabilities(connection, capabilities); - /////////////////////////////////////////////////////////////////////////////////////////// - // ConnectionListener implementation - /////////////////////////////////////////////////////////////////////////////////////////// + printNewReportedPeers(reportedPeersToAdd); - @Override - public void onConnection(Connection connection) { - boolean seedNode = isSeedNode(connection); - Optional addressOptional = connection.getPeersNodeAddressOptional(); - if (log.isDebugEnabled()) { - String peer = addressOptional.map(NodeAddress::getFullAddress).orElseGet(() -> - "not known yet (connection id=" + connection.getUid() + ")"); - log.debug("onConnection: peer = {}{}", - peer, - seedNode ? " (SeedNode)" : ""); + // We check if the reported msg is not violating our rules + if (reportedPeersToAdd.size() <= (MAX_REPORTED_PEERS + maxConnectionsAbsolute + 10)) { + reportedPeers.addAll(reportedPeersToAdd); + purgeReportedPeersIfExceeds(); + + getPersistedPeers().addAll(reportedPeersToAdd); + purgePersistedPeersIfExceeds(); + requestPersistence(); + + printReportedPeers(); + } else { + // If a node is trying to send too many list we treat it as rule violation. + // Reported list include the connected list. We use the max value and give some extra headroom. + // Will trigger a shutdown after 2nd time sending too much + connection.reportInvalidRequest(RuleViolation.TOO_MANY_REPORTED_PEERS_SENT); } + } - if (seedNode) - connection.setPeerType(Connection.PeerType.SEED_NODE); + // Delivers the live peers from the last 30 min (MAX_AGE_LIVE_PEERS) + // We include older peers to avoid risks for network partitioning + public Set getLivePeers() { + return getLivePeers(null); + } - doHouseKeeping(); + public Set getLivePeers(@Nullable NodeAddress excludedNodeAddress) { + int oldNumLatestLivePeers = latestLivePeers.size(); - if (lostAllConnections) { - lostAllConnections = false; - stopped = false; - listeners.stream().forEach(Listener::onNewConnectionAfterAllConnectionsLost); - } + Set peers = new HashSet<>(latestLivePeers); + Set currentLivePeers = getConnectedReportedPeers().stream() + .filter(e -> !isSeedNode(e)) + .filter(e -> !e.getNodeAddress().equals(excludedNodeAddress)) + .collect(Collectors.toSet()); + peers.addAll(currentLivePeers); + + long maxAge = new Date().getTime() - MAX_AGE_LIVE_PEERS; + latestLivePeers.clear(); + Set recentPeers = peers.stream() + .filter(peer -> peer.getDateAsLong() > maxAge) + .collect(Collectors.toSet()); + latestLivePeers.addAll(recentPeers); + + if (oldNumLatestLivePeers != latestLivePeers.size()) + log.info("Num of latestLivePeers={}", latestLivePeers.size()); + return latestLivePeers; } - @Override - public void onDisconnect(CloseConnectionReason closeConnectionReason, Connection connection) { - log.info("onDisconnect called: nodeAddress={}, closeConnectionReason={}", connection.getPeersNodeAddressOptional(), closeConnectionReason); - final Optional addressOptional = connection.getPeersNodeAddressOptional(); - log.debug("onDisconnect: peer = {}{} / closeConnectionReason: {}", - (addressOptional.isPresent() ? addressOptional.get().getFullAddress() : "not known yet (connection id=" + connection.getUid() + ")"), - isSeedNode(connection) ? " (SeedNode)" : "", - closeConnectionReason); + /////////////////////////////////////////////////////////////////////////////////////////// + // Capabilities + /////////////////////////////////////////////////////////////////////////////////////////// - handleConnectionFault(connection); + public boolean peerHasCapability(NodeAddress peersNodeAddress, Capability capability) { + return findPeersCapabilities(peersNodeAddress) + .map(capabilities -> capabilities.contains(capability)) + .orElse(false); + } - lostAllConnections = networkNode.getAllConnections().isEmpty(); - if (lostAllConnections) { - stopped = true; - log.warn("\n------------------------------------------------------------\n" + - "All connections lost\n" + - "------------------------------------------------------------"); - listeners.stream().forEach(Listener::onAllConnectionsLost); + public Optional findPeersCapabilities(NodeAddress nodeAddress) { + // We look up first our connections as that is our own data. If not found there we look up the peers which + // include reported peers. + Optional optionalCapabilities = networkNode.findPeersCapabilities(nodeAddress); + if (optionalCapabilities.isPresent()) { + return optionalCapabilities; } - if (connection.getPeersNodeAddressOptional().isPresent() && isNodeBanned(closeConnectionReason, connection)) { - final NodeAddress nodeAddress = connection.getPeersNodeAddressOptional().get(); - seedNodeAddresses.remove(nodeAddress); - removePersistedPeer(nodeAddress); - removeReportedPeer(nodeAddress); - } + // Reported peers are not trusted data. We could get capabilities which miss the + // peers real capability or we could get maliciously altered capabilities telling us the peer supports a + // capability which is in fact not supported. This could lead to connection loss as we might send data not + // recognized by the peer. As we register a listener on connection if we don't have set the capability from our + // own sources we would get it fixed as soon we have a connection with that peer, rendering such an attack + // inefficient. + // Also this risk is only for not updated peers, so in case that would be abused for an + // attack all users have a strong incentive to update ;-). + return getAllPeers().stream() + .filter(peer -> peer.getNodeAddress().equals(nodeAddress)) + .findAny().map(Peer::getCapabilities); } - public boolean isNodeBanned(CloseConnectionReason closeConnectionReason, Connection connection) { - return closeConnectionReason == CloseConnectionReason.PEER_BANNED && - connection.getPeersNodeAddressOptional().isPresent(); - } + private void applyCapabilities(Connection connection, Capabilities capabilities) { + if (capabilities == null || capabilities.isEmpty()) { + return; + } - @Override - public void onError(Throwable throwable) { + connection.getPeersNodeAddressOptional().ifPresent(nodeAddress -> { + getAllPeers().stream() + .filter(peer -> peer.getNodeAddress().equals(nodeAddress)) + .forEach(peer -> peer.setCapabilities(capabilities)); + }); + requestPersistence(); } @@ -463,29 +560,22 @@ private void removeSuperfluousSeedNodes() { } } - /////////////////////////////////////////////////////////////////////////////////////////// // Reported peers /////////////////////////////////////////////////////////////////////////////////////////// - private boolean removeReportedPeer(Peer reportedPeer) { - boolean contained = reportedPeers.remove(reportedPeer); + private void removeReportedPeer(Peer reportedPeer) { + reportedPeers.remove(reportedPeer); printReportedPeers(); - return contained; } - @SuppressWarnings("UnusedReturnValue") - @Nullable - private Peer removeReportedPeer(NodeAddress nodeAddress) { + private void removeReportedPeer(NodeAddress nodeAddress) { List reportedPeersClone = new ArrayList<>(reportedPeers); Optional reportedPeerOptional = reportedPeersClone.stream() .filter(e -> e.getNodeAddress().equals(nodeAddress)).findAny(); if (reportedPeerOptional.isPresent()) { Peer reportedPeer = reportedPeerOptional.get(); removeReportedPeer(reportedPeer); - return reportedPeer; - } else { - return null; } } @@ -497,44 +587,6 @@ private void removeTooOldReportedPeers() { reportedPeersToRemove.forEach(this::removeReportedPeer); } - public void addToReportedPeers(Set reportedPeersToAdd, - Connection connection, - Capabilities capabilities) { - - applyCapabilities(connection, capabilities); - - printNewReportedPeers(reportedPeersToAdd); - - // We check if the reported msg is not violating our rules - if (reportedPeersToAdd.size() <= (MAX_REPORTED_PEERS + maxConnectionsAbsolute + 10)) { - reportedPeers.addAll(reportedPeersToAdd); - purgeReportedPeersIfExceeds(); - - getPersistedPeers().addAll(reportedPeersToAdd); - purgePersistedPeersIfExceeds(); - requestPersistence(); - - printReportedPeers(); - } else { - // If a node is trying to send too many list we treat it as rule violation. - // Reported list include the connected list. We use the max value and give some extra headroom. - // Will trigger a shutdown after 2nd time sending too much - connection.reportInvalidRequest(RuleViolation.TOO_MANY_REPORTED_PEERS_SENT); - } - } - - private void applyCapabilities(Connection connection, Capabilities capabilities) { - if (capabilities == null || capabilities.isEmpty()) { - return; - } - - connection.getPeersNodeAddressOptional().ifPresent(nodeAddress -> { - getAllPeers().stream() - .filter(peer -> peer.getNodeAddress().equals(nodeAddress)) - .forEach(peer -> peer.setCapabilities(capabilities)); - }); - requestPersistence(); - } private void purgeReportedPeersIfExceeds() { int size = reportedPeers.size(); @@ -543,7 +595,7 @@ private void purgeReportedPeersIfExceeds() { "We remove random peers from the reported peers list.", size, MAX_REPORTED_PEERS); int diff = size - MAX_REPORTED_PEERS; List list = new ArrayList<>(reportedPeers); - // we dont use sorting by lastActivityDate to keep it more random + // we don't use sorting by lastActivityDate to keep it more random for (int i = 0; i < diff; i++) { if (!list.isEmpty()) { Peer toRemove = list.remove(new Random().nextInt(list.size())); @@ -557,12 +609,11 @@ private void purgeReportedPeersIfExceeds() { private void printReportedPeers() { if (!reportedPeers.isEmpty()) { - //noinspection ConstantConditions if (PRINT_REPORTED_PEERS_DETAILS) { StringBuilder result = new StringBuilder("\n\n------------------------------------------------------------\n" + "Collected reported peers:"); List reportedPeersClone = new ArrayList<>(reportedPeers); - reportedPeersClone.stream().forEach(e -> result.append("\n").append(e)); + reportedPeersClone.forEach(e -> result.append("\n").append(e)); result.append("\n------------------------------------------------------------\n"); log.trace(result.toString()); } @@ -571,7 +622,6 @@ private void printReportedPeers() { } private void printNewReportedPeers(Set reportedPeers) { - //noinspection ConstantConditions if (PRINT_REPORTED_PEERS_DETAILS) { StringBuilder result = new StringBuilder("We received new reportedPeers:"); List reportedPeersClone = new ArrayList<>(reportedPeers); @@ -583,7 +633,7 @@ private void printNewReportedPeers(Set reportedPeers) { /////////////////////////////////////////////////////////////////////////////////////////// - // Persisted list + // Persisted peers /////////////////////////////////////////////////////////////////////////////////////////// private boolean removePersistedPeer(Peer persistedPeer) { @@ -641,114 +691,42 @@ private void purgePersistedPeersIfExceeds() { /////////////////////////////////////////////////////////////////////////////////////////// - // Misc + // Getters /////////////////////////////////////////////////////////////////////////////////////////// - public boolean hasSufficientConnections() { - return networkNode.getNodeAddressesOfConfirmedConnections().size() >= minConnections; - } - - private boolean isSeedNode(Peer reportedPeer) { - return seedNodeAddresses.contains(reportedPeer.getNodeAddress()); - } - - public boolean isSeedNode(NodeAddress nodeAddress) { - return seedNodeAddresses.contains(nodeAddress); - } - - public boolean isSeedNode(Connection connection) { - return connection.hasPeersNodeAddress() && seedNodeAddresses.contains(connection.getPeersNodeAddressOptional().get()); - } - - public boolean isSelf(Peer reportedPeer) { - return isSelf(reportedPeer.getNodeAddress()); - } - - public boolean isSelf(NodeAddress nodeAddress) { - return nodeAddress.equals(networkNode.getNodeAddress()); - } - - public boolean isConfirmed(Peer reportedPeer) { - return isConfirmed(reportedPeer.getNodeAddress()); - } - - // Checks if that connection has the peers node address - public boolean isConfirmed(NodeAddress nodeAddress) { - return networkNode.getNodeAddressesOfConfirmedConnections().contains(nodeAddress); - } - - public void handleConnectionFault(Connection connection) { - connection.getPeersNodeAddressOptional().ifPresent(nodeAddress -> handleConnectionFault(nodeAddress, connection)); - } - - public void handleConnectionFault(NodeAddress nodeAddress) { - handleConnectionFault(nodeAddress, null); + public int getMaxConnections() { + return maxConnectionsAbsolute; } - public void handleConnectionFault(NodeAddress nodeAddress, @Nullable Connection connection) { - log.debug("handleConnectionFault called: nodeAddress=" + nodeAddress); - boolean doRemovePersistedPeer = false; - removeReportedPeer(nodeAddress); - Optional persistedPeerOptional = findPersistedPeer(nodeAddress); - if (persistedPeerOptional.isPresent()) { - Peer persistedPeer = persistedPeerOptional.get(); - persistedPeer.increaseFailedConnectionAttempts(); - doRemovePersistedPeer = persistedPeer.tooManyFailedConnectionAttempts(); - } - doRemovePersistedPeer = doRemovePersistedPeer || (connection != null && connection.getRuleViolation() != null); - - if (doRemovePersistedPeer) - removePersistedPeer(nodeAddress); - else - removeTooOldPersistedPeers(); - } - public void shutDownConnection(Connection connection, CloseConnectionReason closeConnectionReason) { - if (connection.getPeerType() != Connection.PeerType.DIRECT_MSG_PEER) - connection.shutDown(closeConnectionReason); - } + /////////////////////////////////////////////////////////////////////////////////////////// + // Listeners + /////////////////////////////////////////////////////////////////////////////////////////// - public void shutDownConnection(NodeAddress peersNodeAddress, CloseConnectionReason closeConnectionReason) { - networkNode.getAllConnections().stream() - .filter(connection -> connection.getPeersNodeAddressOptional().isPresent() && - connection.getPeersNodeAddressOptional().get().equals(peersNodeAddress) && - connection.getPeerType() != Connection.PeerType.DIRECT_MSG_PEER) - .findAny() - .ifPresent(connection -> connection.shutDown(closeConnectionReason)); + public void addListener(Listener listener) { + listeners.add(listener); } - // Delivers the live peers from the last 30 min (MAX_AGE_LIVE_PEERS) - // We include older peers to avoid risks for network partitioning - public Set getLivePeers() { - return getLivePeers(null); + public void removeListener(Listener listener) { + listeners.remove(listener); } - public Set getLivePeers(@Nullable NodeAddress excludedNodeAddress) { - int oldNumLatestLivePeers = latestLivePeers.size(); - - Set peers = new HashSet<>(latestLivePeers); - Set currentLivePeers = getConnectedReportedPeers().stream() - .filter(e -> !isSeedNode(e)) - .filter(e -> !e.getNodeAddress().equals(excludedNodeAddress)) - .collect(Collectors.toSet()); - peers.addAll(currentLivePeers); - - long maxAge = new Date().getTime() - MAX_AGE_LIVE_PEERS; - latestLivePeers.clear(); - Set recentPeers = peers.stream() - .filter(peer -> peer.getDateAsLong() > maxAge) - .collect(Collectors.toSet()); - latestLivePeers.addAll(recentPeers); - - if (oldNumLatestLivePeers != latestLivePeers.size()) - log.info("Num of latestLivePeers={}", latestLivePeers.size()); - return latestLivePeers; - } /////////////////////////////////////////////////////////////////////////////////////////// - // Private + // Private misc /////////////////////////////////////////////////////////////////////////////////////////// + // Modify this to change the relationships between connection limits. + // maxConnections default 12 + private void setConnectionLimits(int maxConnections) { + this.maxConnections = maxConnections; // app node 12; seedNode 30 + minConnections = Math.max(1, (int) Math.round(maxConnections * 0.7)); // app node 1-8; seedNode 21 + disconnectFromSeedNode = maxConnections; // app node 12; seedNode 30 + maxConnectionsPeer = Math.max(4, (int) Math.round(maxConnections * 1.3)); // app node 16; seedNode 39 + maxConnectionsNonDirect = Math.max(8, (int) Math.round(maxConnections * 1.7)); // app node 20; seedNode 51 + maxConnectionsAbsolute = Math.max(12, (int) Math.round(maxConnections * 2.5)); // app node 30; seedNode 66 + } + private Set getConnectedReportedPeers() { // networkNode.getConfirmedConnections includes: // filter(connection -> connection.getPeersNodeAddressOptional().isPresent()) From 0686079946fe35f15f1ba8d6f8892142c9069550 Mon Sep 17 00:00:00 2001 From: chimp1984 Date: Wed, 7 Oct 2020 18:46:08 -0500 Subject: [PATCH 22/59] Refactor: Rename method --- .../core/dao/node/lite/network/LiteNodeNetworkService.java | 2 +- p2p/src/main/java/bisq/network/p2p/peers/PeerManager.java | 5 ++--- .../bisq/network/p2p/peers/getdata/RequestDataManager.java | 4 ++-- .../network/p2p/peers/peerexchange/PeerExchangeManager.java | 2 +- 4 files changed, 6 insertions(+), 7 deletions(-) diff --git a/core/src/main/java/bisq/core/dao/node/lite/network/LiteNodeNetworkService.java b/core/src/main/java/bisq/core/dao/node/lite/network/LiteNodeNetworkService.java index 5c7a861a7c4..0b40e80b96c 100644 --- a/core/src/main/java/bisq/core/dao/node/lite/network/LiteNodeNetworkService.java +++ b/core/src/main/java/bisq/core/dao/node/lite/network/LiteNodeNetworkService.java @@ -180,7 +180,7 @@ public void onConnection(Connection connection) { public void onDisconnect(CloseConnectionReason closeConnectionReason, Connection connection) { closeHandler(connection); - if (peerManager.isNodeBanned(closeConnectionReason, connection)) { + if (peerManager.isPeerBanned(closeConnectionReason, connection)) { connection.getPeersNodeAddressOptional().ifPresent(nodeAddress -> { seedNodeAddresses.remove(nodeAddress); removeFromRequestBlocksHandlerMap(nodeAddress); diff --git a/p2p/src/main/java/bisq/network/p2p/peers/PeerManager.java b/p2p/src/main/java/bisq/network/p2p/peers/PeerManager.java index 22712ffb3d9..85b9783bf92 100644 --- a/p2p/src/main/java/bisq/network/p2p/peers/PeerManager.java +++ b/p2p/src/main/java/bisq/network/p2p/peers/PeerManager.java @@ -287,14 +287,13 @@ public boolean isSeedNode(NodeAddress nodeAddress) { return seedNodeAddresses.contains(nodeAddress); } - //TODO rename - public boolean isNodeBanned(CloseConnectionReason closeConnectionReason, Connection connection) { + public boolean isPeerBanned(CloseConnectionReason closeConnectionReason, Connection connection) { return closeConnectionReason == CloseConnectionReason.PEER_BANNED && connection.getPeersNodeAddressOptional().isPresent(); } private void maybeRemoveBannedPeer(CloseConnectionReason closeConnectionReason, Connection connection) { - if (connection.getPeersNodeAddressOptional().isPresent() && isNodeBanned(closeConnectionReason, connection)) { + if (connection.getPeersNodeAddressOptional().isPresent() && isPeerBanned(closeConnectionReason, connection)) { NodeAddress nodeAddress = connection.getPeersNodeAddressOptional().get(); seedNodeAddresses.remove(nodeAddress); removePersistedPeer(nodeAddress); diff --git a/p2p/src/main/java/bisq/network/p2p/peers/getdata/RequestDataManager.java b/p2p/src/main/java/bisq/network/p2p/peers/getdata/RequestDataManager.java index 48433f3e10d..38d2bd3ca4b 100644 --- a/p2p/src/main/java/bisq/network/p2p/peers/getdata/RequestDataManager.java +++ b/p2p/src/main/java/bisq/network/p2p/peers/getdata/RequestDataManager.java @@ -210,8 +210,8 @@ public void onConnection(Connection connection) { public void onDisconnect(CloseConnectionReason closeConnectionReason, Connection connection) { closeHandler(connection); - if (peerManager.isNodeBanned(closeConnectionReason, connection) && connection.getPeersNodeAddressOptional().isPresent()) { - final NodeAddress nodeAddress = connection.getPeersNodeAddressOptional().get(); + if (peerManager.isPeerBanned(closeConnectionReason, connection) && connection.getPeersNodeAddressOptional().isPresent()) { + NodeAddress nodeAddress = connection.getPeersNodeAddressOptional().get(); seedNodeAddresses.remove(nodeAddress); handlerMap.remove(nodeAddress); } diff --git a/p2p/src/main/java/bisq/network/p2p/peers/peerexchange/PeerExchangeManager.java b/p2p/src/main/java/bisq/network/p2p/peers/peerexchange/PeerExchangeManager.java index 8d62ead5a59..2d85f79982f 100644 --- a/p2p/src/main/java/bisq/network/p2p/peers/peerexchange/PeerExchangeManager.java +++ b/p2p/src/main/java/bisq/network/p2p/peers/peerexchange/PeerExchangeManager.java @@ -147,7 +147,7 @@ public void onDisconnect(CloseConnectionReason closeConnectionReason, Connection }, RETRY_DELAY_SEC); } - if (peerManager.isNodeBanned(closeConnectionReason, connection)) + if (peerManager.isPeerBanned(closeConnectionReason, connection)) seedNodeAddresses.remove(connection.getPeersNodeAddressOptional().get()); } From 31ce5ccf671c99adc222c564098a95ee745b53b0 Mon Sep 17 00:00:00 2001 From: chimp1984 Date: Wed, 7 Oct 2020 18:47:43 -0500 Subject: [PATCH 23/59] Use connection.getPeersNodeAddressOptional().isPresent() instead of connection.hasPeersNodeAddress() which does the same internally --- p2p/src/main/java/bisq/network/p2p/peers/PeerManager.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/p2p/src/main/java/bisq/network/p2p/peers/PeerManager.java b/p2p/src/main/java/bisq/network/p2p/peers/PeerManager.java index 85b9783bf92..30c5157768c 100644 --- a/p2p/src/main/java/bisq/network/p2p/peers/PeerManager.java +++ b/p2p/src/main/java/bisq/network/p2p/peers/PeerManager.java @@ -271,8 +271,8 @@ public void handleConnectionFault(NodeAddress nodeAddress, @Nullable Connection } public boolean isSeedNode(Connection connection) { - //TODO - return connection.hasPeersNodeAddress() && seedNodeAddresses.contains(connection.getPeersNodeAddressOptional().get()); + return connection.getPeersNodeAddressOptional().isPresent() && + seedNodeAddresses.contains(connection.getPeersNodeAddressOptional().get()); } public boolean isSelf(NodeAddress nodeAddress) { @@ -306,6 +306,7 @@ private void maybeRemoveBannedPeer(CloseConnectionReason closeConnectionReason, // Peer /////////////////////////////////////////////////////////////////////////////////////////// + @SuppressWarnings("unused") public Optional findPeer(NodeAddress peersNodeAddress) { return getAllPeers().stream() .filter(peer -> peer.getNodeAddress().equals(peersNodeAddress)) From cfda0aff68f314b14d6d5eb9a59c7c10dfbc4d3c Mon Sep 17 00:00:00 2001 From: chimp1984 Date: Wed, 7 Oct 2020 19:17:40 -0500 Subject: [PATCH 24/59] Fix tests --- monitor/src/main/java/bisq/monitor/metric/P2PNetworkLoad.java | 2 +- p2p/src/test/java/bisq/network/p2p/MockNode.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/monitor/src/main/java/bisq/monitor/metric/P2PNetworkLoad.java b/monitor/src/main/java/bisq/monitor/metric/P2PNetworkLoad.java index b9fbefb5551..4a231d8da61 100644 --- a/monitor/src/main/java/bisq/monitor/metric/P2PNetworkLoad.java +++ b/monitor/src/main/java/bisq/monitor/metric/P2PNetworkLoad.java @@ -133,7 +133,7 @@ protected void execute() { networkProtoResolver); DefaultSeedNodeRepository seedNodeRepository = new DefaultSeedNodeRepository(config); PeerManager peerManager = new PeerManager(networkNode, seedNodeRepository, new ClockWatcher(), - maxConnections, new PersistenceManager<>(storageDir, persistenceProtoResolver, corruptedStorageFileHandler)); + new PersistenceManager<>(storageDir, persistenceProtoResolver, corruptedStorageFileHandler), maxConnections); // init file storage peerManager.readPersisted(); diff --git a/p2p/src/test/java/bisq/network/p2p/MockNode.java b/p2p/src/test/java/bisq/network/p2p/MockNode.java index 9520cae156a..2112bcc39b9 100644 --- a/p2p/src/test/java/bisq/network/p2p/MockNode.java +++ b/p2p/src/test/java/bisq/network/p2p/MockNode.java @@ -60,7 +60,7 @@ public MockNode(int maxConnections) throws IOException { networkNode = mock(NetworkNode.class); File storageDir = Files.createTempDirectory("storage").toFile(); PersistenceManager persistenceManager = new PersistenceManager<>(storageDir, mock(PersistenceProtoResolver.class), mock(CorruptedStorageFileHandler.class)); - peerManager = new PeerManager(networkNode, mock(SeedNodeRepository.class), new ClockWatcher(), maxConnections, persistenceManager); + peerManager = new PeerManager(networkNode, mock(SeedNodeRepository.class), new ClockWatcher(), persistenceManager, maxConnections); connections = new HashSet<>(); when(networkNode.getAllConnections()).thenReturn(connections); } From 351db88992e65eaccb5a545b35c7f1b274e49156 Mon Sep 17 00:00:00 2001 From: chimp1984 Date: Wed, 7 Oct 2020 21:02:44 -0500 Subject: [PATCH 25/59] Use a hashset instead of list to avoid duplicates. Filter out my own node from persisted peers. Remove log in DisputeAgentManager which gets called repeatedly --- .../dispute/agent/DisputeAgentManager.java | 2 -- .../bisq/network/p2p/peers/PeerManager.java | 8 +++-- .../p2p/peers/peerexchange/PeerList.java | 31 ++++++++++++------- 3 files changed, 26 insertions(+), 15 deletions(-) diff --git a/core/src/main/java/bisq/core/support/dispute/agent/DisputeAgentManager.java b/core/src/main/java/bisq/core/support/dispute/agent/DisputeAgentManager.java index ac6480bb1b2..73a73110d4c 100644 --- a/core/src/main/java/bisq/core/support/dispute/agent/DisputeAgentManager.java +++ b/core/src/main/java/bisq/core/support/dispute/agent/DisputeAgentManager.java @@ -214,8 +214,6 @@ public void updateMap() { observableMap.putAll(filtered); observableMap.values().forEach(this::addAcceptedDisputeAgentToUser); - - log.info("Available disputeAgents: {}", observableMap.keySet()); } diff --git a/p2p/src/main/java/bisq/network/p2p/peers/PeerManager.java b/p2p/src/main/java/bisq/network/p2p/peers/PeerManager.java index 30c5157768c..3df99fac298 100644 --- a/p2p/src/main/java/bisq/network/p2p/peers/PeerManager.java +++ b/p2p/src/main/java/bisq/network/p2p/peers/PeerManager.java @@ -186,7 +186,7 @@ public void shutDown() { public void readPersisted() { PeerList persisted = persistenceManager.getPersisted(); if (persisted != null) { - peerList.setAll(persisted.getList()); + peerList.setAll(persisted.getSet()); } } @@ -321,7 +321,7 @@ public Set getAllPeers() { } public Collection getPersistedPeers() { - return peerList.getList(); + return peerList.getSet(); } public void addToReportedPeers(Set reportedPeersToAdd, @@ -329,6 +329,10 @@ public void addToReportedPeers(Set reportedPeersToAdd, Capabilities capabilities) { applyCapabilities(connection, capabilities); + reportedPeersToAdd = reportedPeersToAdd.stream() + .filter(peer -> !isSelf(peer.getNodeAddress())) + .collect(Collectors.toSet()); + printNewReportedPeers(reportedPeersToAdd); // We check if the reported msg is not violating our rules diff --git a/p2p/src/main/java/bisq/network/p2p/peers/peerexchange/PeerList.java b/p2p/src/main/java/bisq/network/p2p/peers/peerexchange/PeerList.java index d32a4952c03..37112ec96ea 100644 --- a/p2p/src/main/java/bisq/network/p2p/peers/peerexchange/PeerList.java +++ b/p2p/src/main/java/bisq/network/p2p/peers/peerexchange/PeerList.java @@ -21,46 +21,55 @@ import com.google.protobuf.Message; -import java.util.ArrayList; import java.util.Collection; -import java.util.List; +import java.util.HashSet; +import java.util.Set; import java.util.stream.Collectors; import lombok.EqualsAndHashCode; import lombok.Getter; +import lombok.extern.slf4j.Slf4j; +@Slf4j @EqualsAndHashCode public class PeerList implements PersistableEnvelope { @Getter - private final List list = new ArrayList<>(); + private final Set set = new HashSet<>(); public PeerList() { } - public PeerList(List list) { - setAll(list); + public PeerList(Set set) { + setAll(set); } public int size() { - return list.size(); + return set.size(); } @Override public Message toProtoMessage() { return protobuf.PersistableEnvelope.newBuilder() .setPeerList(protobuf.PeerList.newBuilder() - .addAllPeer(list.stream().map(Peer::toProtoMessage).collect(Collectors.toList()))) + .addAllPeer(set.stream().map(Peer::toProtoMessage).collect(Collectors.toList()))) .build(); } public static PeerList fromProto(protobuf.PeerList proto) { - return new PeerList(new ArrayList<>(proto.getPeerList().stream() + return new PeerList(proto.getPeerList().stream() .map(Peer::fromProto) - .collect(Collectors.toList()))); + .collect(Collectors.toSet())); } public void setAll(Collection collection) { - this.list.clear(); - this.list.addAll(collection); + this.set.clear(); + this.set.addAll(collection); + } + + @Override + public String toString() { + return "PeerList{" + + "\n set=" + set + + "\n}"; } } From 2fd010498d379eee40796196e06811a8c6844148 Mon Sep 17 00:00:00 2001 From: chimp1984 Date: Wed, 7 Oct 2020 21:18:28 -0500 Subject: [PATCH 26/59] Decrease failedConnectionAttempts onConnection Rename method --- .../main/java/bisq/network/p2p/peers/PeerManager.java | 11 ++++++++--- .../bisq/network/p2p/peers/peerexchange/Peer.java | 6 +++++- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/p2p/src/main/java/bisq/network/p2p/peers/PeerManager.java b/p2p/src/main/java/bisq/network/p2p/peers/PeerManager.java index 3df99fac298..9b82beaf29f 100644 --- a/p2p/src/main/java/bisq/network/p2p/peers/PeerManager.java +++ b/p2p/src/main/java/bisq/network/p2p/peers/PeerManager.java @@ -208,6 +208,11 @@ public void onConnection(Connection connection) { stopped = false; listeners.forEach(Listener::onNewConnectionAfterAllConnectionsLost); } + + if (connection.getPeersNodeAddressOptional().isPresent()) { + findPeer(connection.getPeersNodeAddressOptional().get()) + .ifPresent(Peer::onConnection); + } } @Override @@ -253,16 +258,16 @@ public void handleConnectionFault(NodeAddress nodeAddress) { } public void handleConnectionFault(NodeAddress nodeAddress, @Nullable Connection connection) { - log.debug("handleConnectionFault called: nodeAddress=" + nodeAddress); boolean doRemovePersistedPeer = false; removeReportedPeer(nodeAddress); Optional persistedPeerOptional = findPersistedPeer(nodeAddress); if (persistedPeerOptional.isPresent()) { Peer persistedPeer = persistedPeerOptional.get(); - persistedPeer.increaseFailedConnectionAttempts(); + persistedPeer.onDisconnect(); doRemovePersistedPeer = persistedPeer.tooManyFailedConnectionAttempts(); } - doRemovePersistedPeer = doRemovePersistedPeer || (connection != null && connection.getRuleViolation() != null); + boolean ruleViolation = connection != null && connection.getRuleViolation() != null; + doRemovePersistedPeer = doRemovePersistedPeer || ruleViolation; if (doRemovePersistedPeer) removePersistedPeer(nodeAddress); diff --git a/p2p/src/main/java/bisq/network/p2p/peers/peerexchange/Peer.java b/p2p/src/main/java/bisq/network/p2p/peers/peerexchange/Peer.java index a7681f909ce..99678ad6053 100644 --- a/p2p/src/main/java/bisq/network/p2p/peers/peerexchange/Peer.java +++ b/p2p/src/main/java/bisq/network/p2p/peers/peerexchange/Peer.java @@ -80,10 +80,14 @@ public static Peer fromProto(protobuf.Peer proto) { // API /////////////////////////////////////////////////////////////////////////////////////////// - public void increaseFailedConnectionAttempts() { + public void onDisconnect() { this.failedConnectionAttempts++; } + public void onConnection() { + this.failedConnectionAttempts--; + } + public boolean tooManyFailedConnectionAttempts() { return failedConnectionAttempts >= MAX_FAILED_CONNECTION_ATTEMPTS; } From f36a17389e735ec4b80e75f5287d7e5d38c6c112 Mon Sep 17 00:00:00 2001 From: chimp1984 Date: Wed, 7 Oct 2020 21:30:07 -0500 Subject: [PATCH 27/59] Fix incorrect collection used in == 1 check --- p2p/src/main/java/bisq/network/p2p/P2PService.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/p2p/src/main/java/bisq/network/p2p/P2PService.java b/p2p/src/main/java/bisq/network/p2p/P2PService.java index 0e60769ea66..1940f8b4479 100644 --- a/p2p/src/main/java/bisq/network/p2p/P2PService.java +++ b/p2p/src/main/java/bisq/network/p2p/P2PService.java @@ -452,7 +452,7 @@ public void onAdded(Collection protectedStorageEntries) { private void processSingleMailboxEntry(Collection protectedMailboxStorageEntries) { checkArgument(protectedMailboxStorageEntries.size() == 1); var decryptedEntries = new ArrayList<>(getDecryptedEntries(protectedMailboxStorageEntries)); - if (protectedMailboxStorageEntries.size() == 1) { + if (decryptedEntries.size() == 1) { storeMailboxDataAndNotifyListeners(decryptedEntries.get(0)); } } From b748bffbfe31aa81a524affe997124aad9044dbc Mon Sep 17 00:00:00 2001 From: chimp1984 Date: Wed, 7 Oct 2020 21:30:23 -0500 Subject: [PATCH 28/59] Add isPresent check --- .../network/p2p/peers/peerexchange/PeerExchangeManager.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/p2p/src/main/java/bisq/network/p2p/peers/peerexchange/PeerExchangeManager.java b/p2p/src/main/java/bisq/network/p2p/peers/peerexchange/PeerExchangeManager.java index 2d85f79982f..9cd7d8686b7 100644 --- a/p2p/src/main/java/bisq/network/p2p/peers/peerexchange/PeerExchangeManager.java +++ b/p2p/src/main/java/bisq/network/p2p/peers/peerexchange/PeerExchangeManager.java @@ -147,7 +147,8 @@ public void onDisconnect(CloseConnectionReason closeConnectionReason, Connection }, RETRY_DELAY_SEC); } - if (peerManager.isPeerBanned(closeConnectionReason, connection)) + if (peerManager.isPeerBanned(closeConnectionReason, connection) && + connection.getPeersNodeAddressOptional().isPresent()) seedNodeAddresses.remove(connection.getPeersNodeAddressOptional().get()); } From c7f23e8deb3266502607b682a75afbccba727a1a Mon Sep 17 00:00:00 2001 From: chimp1984 Date: Wed, 7 Oct 2020 22:29:22 -0500 Subject: [PATCH 29/59] Do not log size as we don't want to call potentially expensive toProtoMessage method --- .../bisq/network/p2p/storage/persistence/StoreService.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/p2p/src/main/java/bisq/network/p2p/storage/persistence/StoreService.java b/p2p/src/main/java/bisq/network/p2p/storage/persistence/StoreService.java index adc1688593e..5d1b57d9120 100644 --- a/p2p/src/main/java/bisq/network/p2p/storage/persistence/StoreService.java +++ b/p2p/src/main/java/bisq/network/p2p/storage/persistence/StoreService.java @@ -117,12 +117,11 @@ protected T getStore(String fileName) { T persisted = persistenceManager.getPersisted(fileName); if (persisted != null) { store = persisted; - - int length = store.toProtoMessage().toByteArray().length; + /* int length = store.toProtoMessage().getSerializedSize(); double size = length > 1_000_000D ? length / 1_000_000D : length / 1_000D; String unit = length > 1_000_000D ? "MB" : "KB"; log.info("{}: size of {}: {} {}", this.getClass().getSimpleName(), - persisted.getClass().getSimpleName(), size, unit); + persisted.getClass().getSimpleName(), size, unit);*/ } else { store = createStore(); } From f53290b8174614efbd031a7e6db69f4e18077bcf Mon Sep 17 00:00:00 2001 From: chimp1984 Date: Wed, 7 Oct 2020 22:31:03 -0500 Subject: [PATCH 30/59] Copy peers in a new hashset to avoid concurrent modification exc at serialisation Remove final Cleanups --- .../p2p/network/SynchronizedProtoOutputStream.java | 6 +++--- .../main/java/bisq/network/p2p/peers/PeerManager.java | 7 ++++--- .../p2p/peers/peerexchange/GetPeersRequestHandler.java | 5 +++-- .../p2p/peers/peerexchange/PeerExchangeHandler.java | 5 ++++- .../peers/peerexchange/messages/GetPeersRequest.java | 10 ++++++++-- .../peers/peerexchange/messages/GetPeersResponse.java | 8 ++++++-- 6 files changed, 28 insertions(+), 13 deletions(-) diff --git a/p2p/src/main/java/bisq/network/p2p/network/SynchronizedProtoOutputStream.java b/p2p/src/main/java/bisq/network/p2p/network/SynchronizedProtoOutputStream.java index 4614e20f79b..96d944c75fa 100644 --- a/p2p/src/main/java/bisq/network/p2p/network/SynchronizedProtoOutputStream.java +++ b/p2p/src/main/java/bisq/network/p2p/network/SynchronizedProtoOutputStream.java @@ -50,11 +50,11 @@ void writeEnvelope(NetworkEnvelope envelope) { } catch (InterruptedException e) { Thread currentThread = Thread.currentThread(); currentThread.interrupt(); - final String msg = "Thread " + currentThread + " was interrupted. InterruptedException=" + e; + String msg = "Thread " + currentThread + " was interrupted. InterruptedException=" + e; log.error(msg); throw new BisqRuntimeException(msg, e); } catch (ExecutionException e) { - final String msg = "Failed to write envelope. ExecutionException " + e; + String msg = "Failed to write envelope. ExecutionException " + e; log.error(msg); throw new BisqRuntimeException(msg, e); } @@ -65,7 +65,7 @@ void onConnectionShutdown() { executorService.shutdownNow(); super.onConnectionShutdown(); } catch (Throwable t) { - log.error("Failed to handle connection shutdown. Throwable={}", t); + log.error("Failed to handle connection shutdown. Throwable={}", t.toString()); } } } diff --git a/p2p/src/main/java/bisq/network/p2p/peers/PeerManager.java b/p2p/src/main/java/bisq/network/p2p/peers/PeerManager.java index 9b82beaf29f..737f7d0d80d 100644 --- a/p2p/src/main/java/bisq/network/p2p/peers/PeerManager.java +++ b/p2p/src/main/java/bisq/network/p2p/peers/PeerManager.java @@ -401,7 +401,7 @@ public Optional findPeersCapabilities(NodeAddress nodeAddress) { // We look up first our connections as that is our own data. If not found there we look up the peers which // include reported peers. Optional optionalCapabilities = networkNode.findPeersCapabilities(nodeAddress); - if (optionalCapabilities.isPresent()) { + if (optionalCapabilities.isPresent() && !optionalCapabilities.get().isEmpty()) { return optionalCapabilities; } @@ -415,7 +415,8 @@ public Optional findPeersCapabilities(NodeAddress nodeAddress) { // attack all users have a strong incentive to update ;-). return getAllPeers().stream() .filter(peer -> peer.getNodeAddress().equals(nodeAddress)) - .findAny().map(Peer::getCapabilities); + .findAny() + .map(Peer::getCapabilities); } private void applyCapabilities(Connection connection, Capabilities capabilities) { @@ -647,7 +648,7 @@ private void printNewReportedPeers(Set reportedPeers) { private boolean removePersistedPeer(Peer persistedPeer) { if (getPersistedPeers().contains(persistedPeer)) { - getPersistedPeers().remove(persistedPeer); + //getPersistedPeers().remove(persistedPeer); requestPersistence(); return true; } else { diff --git a/p2p/src/main/java/bisq/network/p2p/peers/peerexchange/GetPeersRequestHandler.java b/p2p/src/main/java/bisq/network/p2p/peers/peerexchange/GetPeersRequestHandler.java index f280587c86b..2b11b0c64a9 100644 --- a/p2p/src/main/java/bisq/network/p2p/peers/peerexchange/GetPeersRequestHandler.java +++ b/p2p/src/main/java/bisq/network/p2p/peers/peerexchange/GetPeersRequestHandler.java @@ -32,6 +32,7 @@ import com.google.common.util.concurrent.MoreExecutors; import com.google.common.util.concurrent.SettableFuture; +import java.util.HashSet; import java.util.concurrent.TimeUnit; import lombok.extern.slf4j.Slf4j; @@ -83,11 +84,11 @@ public GetPeersRequestHandler(NetworkNode networkNode, PeerManager peerManager, // API /////////////////////////////////////////////////////////////////////////////////////////// - public void handle(GetPeersRequest getPeersRequest, final Connection connection) { + public void handle(GetPeersRequest getPeersRequest, Connection connection) { checkArgument(connection.getPeersNodeAddressOptional().isPresent(), "The peers address must have been already set at the moment"); GetPeersResponse getPeersResponse = new GetPeersResponse(getPeersRequest.getNonce(), - peerManager.getLivePeers(connection.getPeersNodeAddressOptional().get())); + new HashSet<>(peerManager.getLivePeers(connection.getPeersNodeAddressOptional().get()))); checkArgument(timeoutTimer == null, "onGetPeersRequest must not be called twice."); timeoutTimer = UserThread.runAfter(() -> { // setup before sending to avoid race conditions diff --git a/p2p/src/main/java/bisq/network/p2p/peers/peerexchange/PeerExchangeHandler.java b/p2p/src/main/java/bisq/network/p2p/peers/peerexchange/PeerExchangeHandler.java index df6b685dec7..384f05c34c5 100644 --- a/p2p/src/main/java/bisq/network/p2p/peers/peerexchange/PeerExchangeHandler.java +++ b/p2p/src/main/java/bisq/network/p2p/peers/peerexchange/PeerExchangeHandler.java @@ -35,6 +35,7 @@ import com.google.common.util.concurrent.MoreExecutors; import com.google.common.util.concurrent.SettableFuture; +import java.util.HashSet; import java.util.Random; import java.util.concurrent.TimeUnit; @@ -104,7 +105,9 @@ private void sendGetPeersRequest(NodeAddress nodeAddress) { log.debug("sendGetPeersRequest to nodeAddress={}", nodeAddress); if (!stopped) { if (networkNode.getNodeAddress() != null) { - GetPeersRequest getPeersRequest = new GetPeersRequest(networkNode.getNodeAddress(), nonce, peerManager.getLivePeers(nodeAddress)); + GetPeersRequest getPeersRequest = new GetPeersRequest(networkNode.getNodeAddress(), + nonce, + new HashSet<>(peerManager.getLivePeers(nodeAddress))); if (timeoutTimer == null) { timeoutTimer = UserThread.runAfter(() -> { // setup before sending to avoid race conditions if (!stopped) { diff --git a/p2p/src/main/java/bisq/network/p2p/peers/peerexchange/messages/GetPeersRequest.java b/p2p/src/main/java/bisq/network/p2p/peers/peerexchange/messages/GetPeersRequest.java index 2f23ef4308b..90f31b97dae 100644 --- a/p2p/src/main/java/bisq/network/p2p/peers/peerexchange/messages/GetPeersRequest.java +++ b/p2p/src/main/java/bisq/network/p2p/peers/peerexchange/messages/GetPeersRequest.java @@ -47,8 +47,14 @@ public final class GetPeersRequest extends NetworkEnvelope implements PeerExchan @Nullable private final Capabilities supportedCapabilities; - public GetPeersRequest(NodeAddress senderNodeAddress, int nonce, Set reportedPeers) { - this(senderNodeAddress, nonce, reportedPeers, Capabilities.app, Version.getP2PMessageVersion()); + public GetPeersRequest(NodeAddress senderNodeAddress, + int nonce, + Set reportedPeers) { + this(senderNodeAddress, + nonce, + reportedPeers, + Capabilities.app, + Version.getP2PMessageVersion()); } diff --git a/p2p/src/main/java/bisq/network/p2p/peers/peerexchange/messages/GetPeersResponse.java b/p2p/src/main/java/bisq/network/p2p/peers/peerexchange/messages/GetPeersResponse.java index 5a41fe91f40..149bcb4946e 100644 --- a/p2p/src/main/java/bisq/network/p2p/peers/peerexchange/messages/GetPeersResponse.java +++ b/p2p/src/main/java/bisq/network/p2p/peers/peerexchange/messages/GetPeersResponse.java @@ -43,8 +43,12 @@ public final class GetPeersResponse extends NetworkEnvelope implements PeerExcha @Nullable private final Capabilities supportedCapabilities; - public GetPeersResponse(int requestNonce, Set reportedPeers) { - this(requestNonce, reportedPeers, Capabilities.app, Version.getP2PMessageVersion()); + public GetPeersResponse(int requestNonce, + Set reportedPeers) { + this(requestNonce, + reportedPeers, + Capabilities.app, + Version.getP2PMessageVersion()); } From 447235c2afe2ed1d50bd68ca941e6c41df548ffc Mon Sep 17 00:00:00 2001 From: chimp1984 Date: Wed, 7 Oct 2020 22:48:10 -0500 Subject: [PATCH 31/59] Dont reassign param --- .../main/java/bisq/network/p2p/peers/PeerManager.java | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/p2p/src/main/java/bisq/network/p2p/peers/PeerManager.java b/p2p/src/main/java/bisq/network/p2p/peers/PeerManager.java index 737f7d0d80d..8f81fef5e0b 100644 --- a/p2p/src/main/java/bisq/network/p2p/peers/PeerManager.java +++ b/p2p/src/main/java/bisq/network/p2p/peers/PeerManager.java @@ -334,18 +334,18 @@ public void addToReportedPeers(Set reportedPeersToAdd, Capabilities capabilities) { applyCapabilities(connection, capabilities); - reportedPeersToAdd = reportedPeersToAdd.stream() + Set peers = reportedPeersToAdd.stream() .filter(peer -> !isSelf(peer.getNodeAddress())) .collect(Collectors.toSet()); - printNewReportedPeers(reportedPeersToAdd); + printNewReportedPeers(peers); // We check if the reported msg is not violating our rules - if (reportedPeersToAdd.size() <= (MAX_REPORTED_PEERS + maxConnectionsAbsolute + 10)) { - reportedPeers.addAll(reportedPeersToAdd); + if (peers.size() <= (MAX_REPORTED_PEERS + maxConnectionsAbsolute + 10)) { + reportedPeers.addAll(peers); purgeReportedPeersIfExceeds(); - getPersistedPeers().addAll(reportedPeersToAdd); + getPersistedPeers().addAll(peers); purgePersistedPeersIfExceeds(); requestPersistence(); From c88bc1c99c918997b7b2f1ad18330c171a5b3f1c Mon Sep 17 00:00:00 2001 From: chimp1984 Date: Wed, 7 Oct 2020 23:24:33 -0500 Subject: [PATCH 32/59] Use custom class MailboxItem instead of Tuple Rename mailboxMap to mailboxItemsByUid --- .../core/trade/protocol/TradeProtocol.java | 5 +- .../java/bisq/network/p2p/P2PService.java | 48 ++++++++++++------- 2 files changed, 33 insertions(+), 20 deletions(-) diff --git a/core/src/main/java/bisq/core/trade/protocol/TradeProtocol.java b/core/src/main/java/bisq/core/trade/protocol/TradeProtocol.java index f27fa0804af..af259e25c74 100644 --- a/core/src/main/java/bisq/core/trade/protocol/TradeProtocol.java +++ b/core/src/main/java/bisq/core/trade/protocol/TradeProtocol.java @@ -30,6 +30,7 @@ import bisq.network.p2p.DecryptedMessageWithPubKey; import bisq.network.p2p.MailboxMessage; import bisq.network.p2p.NodeAddress; +import bisq.network.p2p.P2PService; import bisq.network.p2p.SendMailboxMessageListener; import bisq.network.p2p.messaging.DecryptedMailboxListener; @@ -77,8 +78,8 @@ protected void onInitialized() { processModel.getP2PService().addDecryptedDirectMessageListener(this); } processModel.getP2PService().addDecryptedMailboxListener(this); - processModel.getP2PService().getMailboxMap().values() - .stream().map(e -> e.second) + processModel.getP2PService().getMailboxItemsByUid().values() + .stream().map(P2PService.MailboxItem::getDecryptedMessageWithPubKey) .forEach(this::handleDecryptedMessageWithPubKey); } diff --git a/p2p/src/main/java/bisq/network/p2p/P2PService.java b/p2p/src/main/java/bisq/network/p2p/P2PService.java index 1940f8b4479..95b68f0c611 100644 --- a/p2p/src/main/java/bisq/network/p2p/P2PService.java +++ b/p2p/src/main/java/bisq/network/p2p/P2PService.java @@ -53,7 +53,6 @@ import bisq.common.proto.ProtobufferException; import bisq.common.proto.network.NetworkEnvelope; import bisq.common.proto.persistable.PersistedDataHost; -import bisq.common.util.Tuple2; import bisq.common.util.Utilities; import com.google.inject.Inject; @@ -97,6 +96,7 @@ import org.slf4j.LoggerFactory; import lombok.Getter; +import lombok.Value; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; @@ -126,7 +126,7 @@ public class P2PService implements SetupListener, MessageListener, ConnectionLis private final Set decryptedMailboxListeners = new CopyOnWriteArraySet<>(); private final Set p2pServiceListeners = new CopyOnWriteArraySet<>(); @Getter - private final Map> mailboxMap = new HashMap<>(); + private final Map mailboxItemsByUid = new HashMap<>(); private final Set shutDownResultHandlers = new CopyOnWriteArraySet<>(); private final BooleanProperty hiddenServicePublished = new SimpleBooleanProperty(); private final BooleanProperty preliminaryDataReceived = new SimpleBooleanProperty(); @@ -462,7 +462,7 @@ private void processSingleMailboxEntry(Collection private void threadedBatchProcessMailboxEntries(Collection protectedMailboxStorageEntries) { ListeningExecutorService executor = Utilities.getSingleThreadListeningExecutor("processMailboxEntry-" + new Random().nextInt(1000)); long ts = System.currentTimeMillis(); - ListenableFuture>> future = executor.submit(() -> { + ListenableFuture> future = executor.submit(() -> { var decryptedEntries = getDecryptedEntries(protectedMailboxStorageEntries); log.info("Batch processing of {} mailbox entries took {} ms", protectedMailboxStorageEntries.size(), @@ -471,8 +471,8 @@ private void threadedBatchProcessMailboxEntries(Collection() { - public void onSuccess(Set> decryptedEntries) { - UserThread.execute(() -> decryptedEntries.forEach(e -> storeMailboxDataAndNotifyListeners(e))); + public void onSuccess(Set decryptedMailboxMessageWithEntries) { + UserThread.execute(() -> decryptedMailboxMessageWithEntries.forEach(e -> storeMailboxDataAndNotifyListeners(e))); } public void onFailure(@NotNull Throwable throwable) { @@ -481,25 +481,24 @@ public void onFailure(@NotNull Throwable throwable) { }, MoreExecutors.directExecutor()); } - private Set> getDecryptedEntries(Collection protectedMailboxStorageEntries) { - Set> decryptedEntries = new HashSet<>(); + private Set getDecryptedEntries(Collection protectedMailboxStorageEntries) { + Set decryptedMailboxMessageWithEntries = new HashSet<>(); protectedMailboxStorageEntries.stream() .map(this::decryptProtectedMailboxStorageEntry) .filter(Objects::nonNull) - .forEach(decryptedEntries::add); - return decryptedEntries; + .forEach(decryptedMailboxMessageWithEntries::add); + return decryptedMailboxMessageWithEntries; } @Nullable - private Tuple2 decryptProtectedMailboxStorageEntry( - ProtectedMailboxStorageEntry protectedMailboxStorageEntry) { + private MailboxItem decryptProtectedMailboxStorageEntry(ProtectedMailboxStorageEntry protectedMailboxStorageEntry) { try { DecryptedMessageWithPubKey decryptedMessageWithPubKey = encryptionService.decryptAndVerify(protectedMailboxStorageEntry .getMailboxStoragePayload() .getPrefixedSealedAndSignedMessage() .getSealedAndSigned()); checkArgument(decryptedMessageWithPubKey.getNetworkEnvelope() instanceof MailboxMessage); - return new Tuple2<>(protectedMailboxStorageEntry, decryptedMessageWithPubKey); + return new MailboxItem(protectedMailboxStorageEntry, decryptedMessageWithPubKey); } catch (CryptoException ignore) { // Expected if message was not intended for us } catch (ProtobufferException e) { @@ -509,11 +508,11 @@ private Tuple2 decrypt return null; } - private void storeMailboxDataAndNotifyListeners(Tuple2 tuple2) { - DecryptedMessageWithPubKey decryptedMessageWithPubKey = tuple2.second; + private void storeMailboxDataAndNotifyListeners(MailboxItem mailboxItem) { + DecryptedMessageWithPubKey decryptedMessageWithPubKey = mailboxItem.getDecryptedMessageWithPubKey(); MailboxMessage mailboxMessage = (MailboxMessage) decryptedMessageWithPubKey.getNetworkEnvelope(); NodeAddress sender = mailboxMessage.getSenderNodeAddress(); - mailboxMap.put(mailboxMessage.getUid(), tuple2); + mailboxItemsByUid.put(mailboxMessage.getUid(), mailboxItem); log.info("Received a {} mailbox message with uid {} and senderAddress {}", mailboxMessage.getClass().getSimpleName(), mailboxMessage.getUid(), sender); decryptedMailboxListeners.forEach(e -> e.onMailboxMessageAdded(decryptedMessageWithPubKey, sender)); @@ -771,8 +770,8 @@ private void delayedRemoveEntryFromMailbox(DecryptedMessageWithPubKey decryptedM MailboxMessage mailboxMessage = (MailboxMessage) decryptedMessageWithPubKey.getNetworkEnvelope(); String uid = mailboxMessage.getUid(); - if (mailboxMap.containsKey(uid)) { - ProtectedMailboxStorageEntry mailboxData = mailboxMap.get(uid).first; + if (mailboxItemsByUid.containsKey(uid)) { + ProtectedMailboxStorageEntry mailboxData = mailboxItemsByUid.get(uid).getProtectedMailboxStorageEntry(); if (mailboxData != null && mailboxData.getProtectedStoragePayload() instanceof MailboxStoragePayload) { MailboxStoragePayload expirableMailboxStoragePayload = (MailboxStoragePayload) mailboxData.getProtectedStoragePayload(); PublicKey receiversPubKey = mailboxData.getReceiversPubKey(); @@ -788,7 +787,7 @@ private void delayedRemoveEntryFromMailbox(DecryptedMessageWithPubKey decryptedM log.error("Signing at getDataWithSignedSeqNr failed. That should never happen."); } - mailboxMap.remove(uid); + mailboxItemsByUid.remove(uid); log.info("Removed successfully decryptedMsgWithPubKey. uid={}", uid); } } else { @@ -920,4 +919,17 @@ public PeerManager getPeerManager() { public KeyRing getKeyRing() { return keyRing; } + + + @Value + public class MailboxItem { + private final ProtectedMailboxStorageEntry protectedMailboxStorageEntry; + private final DecryptedMessageWithPubKey decryptedMessageWithPubKey; + + public MailboxItem(ProtectedMailboxStorageEntry protectedMailboxStorageEntry, + DecryptedMessageWithPubKey decryptedMessageWithPubKey) { + this.protectedMailboxStorageEntry = protectedMailboxStorageEntry; + this.decryptedMessageWithPubKey = decryptedMessageWithPubKey; + } + } } From c8feef150e002803799b95b9e121f7854ad699de Mon Sep 17 00:00:00 2001 From: chimp1984 Date: Thu, 8 Oct 2020 11:16:54 -0500 Subject: [PATCH 33/59] Apply code review suggestions Fix incorrectly commented out code (was for dev testing commented out) --- .../bisq/network/p2p/peers/PeerManager.java | 20 ++++++++----------- .../peerexchange/PeerExchangeManager.java | 6 +++--- 2 files changed, 11 insertions(+), 15 deletions(-) diff --git a/p2p/src/main/java/bisq/network/p2p/peers/PeerManager.java b/p2p/src/main/java/bisq/network/p2p/peers/PeerManager.java index 8f81fef5e0b..07b16336e41 100644 --- a/p2p/src/main/java/bisq/network/p2p/peers/PeerManager.java +++ b/p2p/src/main/java/bisq/network/p2p/peers/PeerManager.java @@ -208,11 +208,9 @@ public void onConnection(Connection connection) { stopped = false; listeners.forEach(Listener::onNewConnectionAfterAllConnectionsLost); } - - if (connection.getPeersNodeAddressOptional().isPresent()) { - findPeer(connection.getPeersNodeAddressOptional().get()) - .ifPresent(Peer::onConnection); - } + connection.getPeersNodeAddressOptional() + .flatMap(this::findPeer) + .ifPresent(Peer::onConnection); } @Override @@ -581,12 +579,10 @@ private void removeReportedPeer(Peer reportedPeer) { private void removeReportedPeer(NodeAddress nodeAddress) { List reportedPeersClone = new ArrayList<>(reportedPeers); - Optional reportedPeerOptional = reportedPeersClone.stream() - .filter(e -> e.getNodeAddress().equals(nodeAddress)).findAny(); - if (reportedPeerOptional.isPresent()) { - Peer reportedPeer = reportedPeerOptional.get(); - removeReportedPeer(reportedPeer); - } + reportedPeersClone.stream() + .filter(e -> e.getNodeAddress().equals(nodeAddress)) + .findAny() + .ifPresent(this::removeReportedPeer); } private void removeTooOldReportedPeers() { @@ -648,7 +644,7 @@ private void printNewReportedPeers(Set reportedPeers) { private boolean removePersistedPeer(Peer persistedPeer) { if (getPersistedPeers().contains(persistedPeer)) { - //getPersistedPeers().remove(persistedPeer); + getPersistedPeers().remove(persistedPeer); requestPersistence(); return true; } else { diff --git a/p2p/src/main/java/bisq/network/p2p/peers/peerexchange/PeerExchangeManager.java b/p2p/src/main/java/bisq/network/p2p/peers/peerexchange/PeerExchangeManager.java index 9cd7d8686b7..a843abf92d8 100644 --- a/p2p/src/main/java/bisq/network/p2p/peers/peerexchange/PeerExchangeManager.java +++ b/p2p/src/main/java/bisq/network/p2p/peers/peerexchange/PeerExchangeManager.java @@ -147,9 +147,9 @@ public void onDisconnect(CloseConnectionReason closeConnectionReason, Connection }, RETRY_DELAY_SEC); } - if (peerManager.isPeerBanned(closeConnectionReason, connection) && - connection.getPeersNodeAddressOptional().isPresent()) - seedNodeAddresses.remove(connection.getPeersNodeAddressOptional().get()); + if (peerManager.isPeerBanned(closeConnectionReason, connection)) { + connection.getPeersNodeAddressOptional().ifPresent(seedNodeAddresses::remove); + } } @Override From 4f685f8f4becfc77e2e3be61645c2cff857668ee Mon Sep 17 00:00:00 2001 From: chimp1984 Date: Thu, 8 Oct 2020 13:24:51 -0500 Subject: [PATCH 34/59] When updating the capability from a reported peer we check if the reported one has higher capabilities, otherwise we ignore it. --- .../java/bisq/common/app/Capabilities.java | 14 +++++++ .../bisq/common/app/CapabilitiesTest.java | 42 +++++++++++++++++++ .../bisq/network/p2p/peers/PeerManager.java | 7 ++-- 3 files changed, 60 insertions(+), 3 deletions(-) diff --git a/common/src/main/java/bisq/common/app/Capabilities.java b/common/src/main/java/bisq/common/app/Capabilities.java index bf95c15c48d..d0b3e50a3f8 100644 --- a/common/src/main/java/bisq/common/app/Capabilities.java +++ b/common/src/main/java/bisq/common/app/Capabilities.java @@ -184,4 +184,18 @@ public String prettyPrint() { public int size() { return capabilities.size(); } + + // We return true if our capabilities have less capabilities than the parameter value + public boolean hasLess(Capabilities other) { + return findHighestCapability(this) < findHighestCapability(other); + } + + // We use the sum of all capabilities. Alternatively we could use the highest entry. + // Neither would support removal of past capabilities, a use case we never had so far and which might have + // backward compatibility issues, so we should treat capabilities as an append-only data structure. + public int findHighestCapability(Capabilities capabilities) { + return (int) capabilities.capabilities.stream() + .mapToLong(e -> (long) e.ordinal()) + .sum(); + } } diff --git a/common/src/test/java/bisq/common/app/CapabilitiesTest.java b/common/src/test/java/bisq/common/app/CapabilitiesTest.java index 735fc899eaf..2595ea8b180 100644 --- a/common/src/test/java/bisq/common/app/CapabilitiesTest.java +++ b/common/src/test/java/bisq/common/app/CapabilitiesTest.java @@ -23,6 +23,7 @@ import org.junit.Test; +import static bisq.common.app.Capability.DAO_FULL_NODE; import static bisq.common.app.Capability.SEED_NODE; import static bisq.common.app.Capability.TRADE_STATISTICS; import static bisq.common.app.Capability.TRADE_STATISTICS_2; @@ -40,6 +41,47 @@ public void testNoCapabilitiesAvailable() { assertFalse(DUT.containsAll(new Capabilities(SEED_NODE))); } + @Test + public void testHasLess() { + assertTrue(new Capabilities().hasLess(new Capabilities(SEED_NODE))); + assertFalse(new Capabilities().hasLess(new Capabilities())); + assertFalse(new Capabilities(SEED_NODE).hasLess(new Capabilities())); + assertTrue(new Capabilities(SEED_NODE).hasLess(new Capabilities(DAO_FULL_NODE))); + assertFalse(new Capabilities(DAO_FULL_NODE).hasLess(new Capabilities(SEED_NODE))); + + Capabilities all = new Capabilities( + Capability.TRADE_STATISTICS, + Capability.TRADE_STATISTICS_2, + Capability.ACCOUNT_AGE_WITNESS, + Capability.ACK_MSG, + Capability.PROPOSAL, + Capability.BLIND_VOTE, + Capability.DAO_STATE, + Capability.BUNDLE_OF_ENVELOPES, + Capability.MEDIATION, + Capability.SIGNED_ACCOUNT_AGE_WITNESS, + Capability.REFUND_AGENT, + Capability.TRADE_STATISTICS_HASH_UPDATE + ); + Capabilities other = new Capabilities( + Capability.TRADE_STATISTICS, + Capability.TRADE_STATISTICS_2, + Capability.ACCOUNT_AGE_WITNESS, + Capability.ACK_MSG, + Capability.PROPOSAL, + Capability.BLIND_VOTE, + Capability.DAO_STATE, + Capability.BUNDLE_OF_ENVELOPES, + Capability.MEDIATION, + Capability.SIGNED_ACCOUNT_AGE_WITNESS, + Capability.REFUND_AGENT, + Capability.TRADE_STATISTICS_HASH_UPDATE, + Capability.NO_ADDRESS_PRE_FIX + ); + + assertTrue(all.hasLess(other)); + } + @Test public void testO() { Capabilities DUT = new Capabilities(TRADE_STATISTICS); diff --git a/p2p/src/main/java/bisq/network/p2p/peers/PeerManager.java b/p2p/src/main/java/bisq/network/p2p/peers/PeerManager.java index 07b16336e41..20af0453194 100644 --- a/p2p/src/main/java/bisq/network/p2p/peers/PeerManager.java +++ b/p2p/src/main/java/bisq/network/p2p/peers/PeerManager.java @@ -417,15 +417,16 @@ public Optional findPeersCapabilities(NodeAddress nodeAddress) { .map(Peer::getCapabilities); } - private void applyCapabilities(Connection connection, Capabilities capabilities) { - if (capabilities == null || capabilities.isEmpty()) { + private void applyCapabilities(Connection connection, Capabilities newCapabilities) { + if (newCapabilities == null || newCapabilities.isEmpty()) { return; } connection.getPeersNodeAddressOptional().ifPresent(nodeAddress -> { getAllPeers().stream() .filter(peer -> peer.getNodeAddress().equals(nodeAddress)) - .forEach(peer -> peer.setCapabilities(capabilities)); + .filter(peer -> peer.getCapabilities().hasLess(newCapabilities)) + .forEach(peer -> peer.setCapabilities(newCapabilities)); }); requestPersistence(); } From 6e3fdbc96ab5c2ab2c58a452b5e42031dca262d6 Mon Sep 17 00:00:00 2001 From: chimp1984 Date: Thu, 8 Oct 2020 13:36:32 -0500 Subject: [PATCH 35/59] Apply codacy suggestions --- .../src/test/java/bisq/common/app/CapabilitiesTest.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/common/src/test/java/bisq/common/app/CapabilitiesTest.java b/common/src/test/java/bisq/common/app/CapabilitiesTest.java index 2595ea8b180..f939ff9cd83 100644 --- a/common/src/test/java/bisq/common/app/CapabilitiesTest.java +++ b/common/src/test/java/bisq/common/app/CapabilitiesTest.java @@ -50,8 +50,8 @@ public void testHasLess() { assertFalse(new Capabilities(DAO_FULL_NODE).hasLess(new Capabilities(SEED_NODE))); Capabilities all = new Capabilities( - Capability.TRADE_STATISTICS, - Capability.TRADE_STATISTICS_2, + TRADE_STATISTICS, + TRADE_STATISTICS_2, Capability.ACCOUNT_AGE_WITNESS, Capability.ACK_MSG, Capability.PROPOSAL, @@ -64,8 +64,8 @@ public void testHasLess() { Capability.TRADE_STATISTICS_HASH_UPDATE ); Capabilities other = new Capabilities( - Capability.TRADE_STATISTICS, - Capability.TRADE_STATISTICS_2, + TRADE_STATISTICS, + TRADE_STATISTICS_2, Capability.ACCOUNT_AGE_WITNESS, Capability.ACK_MSG, Capability.PROPOSAL, From 00bed02839a0be741dad3eef578c7ad530cda093 Mon Sep 17 00:00:00 2001 From: chimp1984 Date: Mon, 5 Oct 2020 15:21:53 -0500 Subject: [PATCH 36/59] Add TradeStatistics3 and related classes Add TRADE_STATISTICS_3 Capability Add TradeStatistics3 to proto resolvers Make message TradeStatistics2 deprecated --- .../main/java/bisq/common/app/Capability.java | 3 +- .../bisq/core/proto/CoreProtoResolver.java | 3 + .../CorePersistenceProtoResolver.java | 3 + .../core/setup/CoreNetworkCapabilities.java | 3 +- .../trade/statistics/TradeStatistics3.java | 320 ++++++++++++++++++ .../TradeStatistics3StorageService.java | 84 +++++ .../statistics/TradeStatistics3Store.java | 73 ++++ .../statistics/TradeStatisticsConverter.java | 147 ++++++++ .../src/main/java/bisq/monitor/Monitor.java | 3 +- proto/src/main/proto/pb.proto | 57 ++-- 10 files changed, 674 insertions(+), 22 deletions(-) create mode 100644 core/src/main/java/bisq/core/trade/statistics/TradeStatistics3.java create mode 100644 core/src/main/java/bisq/core/trade/statistics/TradeStatistics3StorageService.java create mode 100644 core/src/main/java/bisq/core/trade/statistics/TradeStatistics3Store.java create mode 100644 core/src/main/java/bisq/core/trade/statistics/TradeStatisticsConverter.java diff --git a/common/src/main/java/bisq/common/app/Capability.java b/common/src/main/java/bisq/common/app/Capability.java index 8ffb042844d..1c9aebd8898 100644 --- a/common/src/main/java/bisq/common/app/Capability.java +++ b/common/src/main/java/bisq/common/app/Capability.java @@ -41,5 +41,6 @@ public enum Capability { MEDIATION, // Supports mediation feature REFUND_AGENT, // Supports refund agents TRADE_STATISTICS_HASH_UPDATE, // We changed the hash method in 1.2.0 and that requires update to 1.2.2 for handling it correctly, otherwise the seed nodes have to process too much data. - NO_ADDRESS_PRE_FIX // At 1.4.0 we removed the prefix filter for mailbox messages. If a peer has that capability we do not sent the prefix. + NO_ADDRESS_PRE_FIX, // At 1.4.0 we removed the prefix filter for mailbox messages. If a peer has that capability we do not sent the prefix. + TRADE_STATISTICS_3 // We used a new reduced trade statistics model from v1.4.0 on } diff --git a/core/src/main/java/bisq/core/proto/CoreProtoResolver.java b/core/src/main/java/bisq/core/proto/CoreProtoResolver.java index 999f4e8e5f1..921a1fa5a4a 100644 --- a/core/src/main/java/bisq/core/proto/CoreProtoResolver.java +++ b/core/src/main/java/bisq/core/proto/CoreProtoResolver.java @@ -54,6 +54,7 @@ import bisq.core.payment.payload.WeChatPayAccountPayload; import bisq.core.payment.payload.WesternUnionAccountPayload; import bisq.core.trade.statistics.TradeStatistics2; +import bisq.core.trade.statistics.TradeStatistics3; import bisq.common.proto.ProtoResolver; import bisq.common.proto.ProtobufferRuntimeException; @@ -178,6 +179,8 @@ public PersistablePayload fromProto(protobuf.PersistableNetworkPayload proto) { return BlindVotePayload.fromProto(proto.getBlindVotePayload()); case SIGNED_WITNESS: return SignedWitness.fromProto(proto.getSignedWitness()); + case TRADE_STATISTICS3: + return TradeStatistics3.fromProto(proto.getTradeStatistics3()); default: throw new ProtobufferRuntimeException("Unknown proto message case (PB.PersistableNetworkPayload). messageCase=" + proto.getMessageCase()); } diff --git a/core/src/main/java/bisq/core/proto/persistable/CorePersistenceProtoResolver.java b/core/src/main/java/bisq/core/proto/persistable/CorePersistenceProtoResolver.java index efefffe881a..26addc24b0e 100644 --- a/core/src/main/java/bisq/core/proto/persistable/CorePersistenceProtoResolver.java +++ b/core/src/main/java/bisq/core/proto/persistable/CorePersistenceProtoResolver.java @@ -39,6 +39,7 @@ import bisq.core.support.dispute.refund.RefundDisputeList; import bisq.core.trade.TradableList; import bisq.core.trade.statistics.TradeStatistics2Store; +import bisq.core.trade.statistics.TradeStatistics3Store; import bisq.core.user.PreferencesPayload; import bisq.core.user.UserPayload; @@ -126,6 +127,8 @@ public PersistableEnvelope fromProto(protobuf.PersistableEnvelope proto) { return UnconfirmedBsqChangeOutputList.fromProto(proto.getUnconfirmedBsqChangeOutputList()); case SIGNED_WITNESS_STORE: return SignedWitnessStore.fromProto(proto.getSignedWitnessStore()); + case TRADE_STATISTICS3_STORE: + return TradeStatistics3Store.fromProto(proto.getTradeStatistics3Store()); default: throw new ProtobufferRuntimeException("Unknown proto message case(PB.PersistableEnvelope). " + diff --git a/core/src/main/java/bisq/core/setup/CoreNetworkCapabilities.java b/core/src/main/java/bisq/core/setup/CoreNetworkCapabilities.java index fd951d98592..c5c70e8877d 100644 --- a/core/src/main/java/bisq/core/setup/CoreNetworkCapabilities.java +++ b/core/src/main/java/bisq/core/setup/CoreNetworkCapabilities.java @@ -40,7 +40,8 @@ static void setSupportedCapabilities(Config config) { Capability.SIGNED_ACCOUNT_AGE_WITNESS, Capability.REFUND_AGENT, Capability.TRADE_STATISTICS_HASH_UPDATE, - Capability.NO_ADDRESS_PRE_FIX + Capability.NO_ADDRESS_PRE_FIX, + Capability.TRADE_STATISTICS_3 ); if (config.daoActivated) { diff --git a/core/src/main/java/bisq/core/trade/statistics/TradeStatistics3.java b/core/src/main/java/bisq/core/trade/statistics/TradeStatistics3.java new file mode 100644 index 00000000000..94d267a7a77 --- /dev/null +++ b/core/src/main/java/bisq/core/trade/statistics/TradeStatistics3.java @@ -0,0 +1,320 @@ +/* + * This file is part of Bisq. + * + * Bisq is free software: you can redistribute it and/or modify it + * under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or (at + * your option) any later version. + * + * Bisq is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public + * License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with Bisq. If not, see . + */ + +package bisq.core.trade.statistics; + +import bisq.core.monetary.Altcoin; +import bisq.core.monetary.AltcoinExchangeRate; +import bisq.core.monetary.Price; +import bisq.core.monetary.Volume; +import bisq.core.offer.OfferUtil; + +import bisq.network.p2p.storage.payload.CapabilityRequiringPayload; +import bisq.network.p2p.storage.payload.PersistableNetworkPayload; +import bisq.network.p2p.storage.payload.ProcessOncePersistableNetworkPayload; + +import bisq.common.app.Capabilities; +import bisq.common.app.Capability; +import bisq.common.crypto.Hash; +import bisq.common.proto.ProtoUtil; +import bisq.common.util.CollectionUtils; +import bisq.common.util.ExtraDataMapValidator; +import bisq.common.util.JsonExclude; +import bisq.common.util.Utilities; + +import com.google.protobuf.ByteString; + +import org.bitcoinj.core.Coin; +import org.bitcoinj.utils.ExchangeRate; +import org.bitcoinj.utils.Fiat; + +import com.google.common.base.Charsets; + +import java.util.Date; +import java.util.Map; +import java.util.Optional; + +import lombok.Value; +import lombok.extern.slf4j.Slf4j; + +import javax.annotation.Nullable; + +import static com.google.common.base.Preconditions.checkNotNull; + +/** + * This new trade statistics class uses only the bare minimum of data. + * Data size is about 50 bytes in average + */ +@Slf4j +@Value +public final class TradeStatistics3 implements ProcessOncePersistableNetworkPayload, PersistableNetworkPayload, + CapabilityRequiringPayload { + + // This enum must not change the order as we use the ordinal for storage to reduce data size. + // The payment method string can be quite long and would consume 15% more space. + // When we get a new payment method we can add it to the enum at the end. Old users would add it as string if not + // recognized. + private enum PaymentMethodMapper { + OK_PAY, + CASH_APP, + VENMO, + AUSTRALIA_PAYID, // seems there is a dev trade + UPHOLD, + MONEY_BEAM, + POPMONEY, + REVOLUT, + PERFECT_MONEY, + SEPA, + SEPA_INSTANT, + FASTER_PAYMENTS, + NATIONAL_BANK, + JAPAN_BANK, + SAME_BANK, + SPECIFIC_BANKS, + SWISH, + ALI_PAY, + WECHAT_PAY, + CLEAR_X_CHANGE, + CHASE_QUICK_PAY, + INTERAC_E_TRANSFER, + US_POSTAL_MONEY_ORDER, + CASH_DEPOSIT, + MONEY_GRAM, + WESTERN_UNION, + HAL_CASH, + F2F, + BLOCK_CHAINS, + PROMPT_PAY, + ADVANCED_CASH, + BLOCK_CHAINS_INSTANT + } + + private final String currency; + private final long price; + private final long amount; + private final String paymentMethod; + // As only seller is publishing it is the sellers trade date + private final long date; + + // Old converted trade stat objects might not have it set + @Nullable + @JsonExclude + private final String mediator; // todo entries from old data could be pruned + @Nullable + @JsonExclude + private final String refundAgent; + + // todo should we add referrerId as well? get added to extra map atm but not used so far + + // Hash get set in constructor from json of all the other data fields (with hash = null). + @JsonExclude + private final byte[] hash; + // Should be only used in emergency case if we need to add data but do not want to break backward compatibility + // at the P2P network storage checks. The hash of the object will be used to verify if the data is valid. Any new + // field in a class would break that hash and therefore break the storage mechanism. + @Nullable + @JsonExclude + private Map extraDataMap; + + public TradeStatistics3(String currency, + long price, + long amount, + String paymentMethod, + long date, + String mediator, + String refundAgent, + @Nullable Map extraDataMap) { + this(currency, + price, + amount, + paymentMethod, + date, + mediator, + refundAgent, + extraDataMap, + null); + } + + // Used from conversion method where we use the hash of the TradeStatistics2 objects to avoid duplicate entries + public TradeStatistics3(String currency, + long price, + long amount, + String paymentMethod, + long date, + String mediator, + String refundAgent, + @Nullable byte[] hash) { + this(currency, + price, + amount, + paymentMethod, + date, + mediator, + refundAgent, + null, + hash); + } + + /////////////////////////////////////////////////////////////////////////////////////////// + // PROTO BUFFER + /////////////////////////////////////////////////////////////////////////////////////////// + + private TradeStatistics3(String currency, + long price, + long amount, + String paymentMethod, + long date, + @Nullable String mediator, + @Nullable String refundAgent, + @Nullable Map extraDataMap, + @Nullable byte[] hash) { + this.currency = currency; + this.price = price; + this.amount = amount; + String tempPaymentMethod; + try { + tempPaymentMethod = String.valueOf(PaymentMethodMapper.valueOf(paymentMethod).ordinal()); + } catch (Throwable t) { + tempPaymentMethod = paymentMethod; + } + this.paymentMethod = tempPaymentMethod; + this.date = date; + this.mediator = mediator; + this.refundAgent = refundAgent; + this.extraDataMap = ExtraDataMapValidator.getValidatedExtraDataMap(extraDataMap); + + this.hash = hash == null ? createHash() : hash; + } + + public byte[] createHash() { + // We create hash from all fields excluding hash itself. We use json as simple data serialisation. + // TradeDate is different for both peers so we ignore it for hash. ExtraDataMap is ignored as well as at + // software updates we might have different entries which would cause a different hash. + return Hash.getSha256Ripemd160hash(Utilities.objectToJson(this).getBytes(Charsets.UTF_8)); + } + + private protobuf.TradeStatistics3.Builder getBuilder() { + protobuf.TradeStatistics3.Builder builder = protobuf.TradeStatistics3.newBuilder() + .setCurrency(currency) + .setPrice(price) + .setAmount(amount) + .setPaymentMethod(paymentMethod) + .setDate(date) + .setHash(ByteString.copyFrom(hash)); + Optional.ofNullable(mediator).ifPresent(builder::setMediator); + Optional.ofNullable(refundAgent).ifPresent(builder::setRefundAgent); + Optional.ofNullable(extraDataMap).ifPresent(builder::putAllExtraData); + return builder; + } + + public protobuf.TradeStatistics3 toProtoTradeStatistics3() { + return getBuilder().build(); + } + + @Override + public protobuf.PersistableNetworkPayload toProtoMessage() { + return protobuf.PersistableNetworkPayload.newBuilder().setTradeStatistics3(getBuilder()).build(); + } + + public static TradeStatistics3 fromProto(protobuf.TradeStatistics3 proto) { + return new TradeStatistics3( + proto.getCurrency(), + proto.getPrice(), + proto.getAmount(), + proto.getPaymentMethod(), + proto.getDate(), + ProtoUtil.stringOrNullFromProto(proto.getMediator()), + ProtoUtil.stringOrNullFromProto(proto.getRefundAgent()), + CollectionUtils.isEmpty(proto.getExtraDataMap()) ? null : proto.getExtraDataMap(), + proto.getHash().toByteArray()); + } + + + /////////////////////////////////////////////////////////////////////////////////////////// + // API + /////////////////////////////////////////////////////////////////////////////////////////// + + @Override + public byte[] getHash() { + return hash; + } + + @Override + public boolean verifyHashSize() { + checkNotNull(hash, "hash must not be null"); + return hash.length == 20; + } + + @Override + public Capabilities getRequiredCapabilities() { + return new Capabilities(Capability.TRADE_STATISTICS_3); + } + + public String getPaymentMethod() { + try { + return PaymentMethodMapper.values()[Integer.parseInt(paymentMethod)].name(); + } catch (Throwable ignore) { + return paymentMethod; + } + } + + public Date getTradeDate() { + return new Date(date); + } + + public Price getTradePrice() { + return Price.valueOf(currency, price); + } + + public Coin getTradeAmount() { + return Coin.valueOf(amount); + } + + public Volume getTradeVolume() { + if (getTradePrice().getMonetary() instanceof Altcoin) { + return new Volume(new AltcoinExchangeRate((Altcoin) getTradePrice().getMonetary()).coinToAltcoin(getTradeAmount())); + } else { + Volume volume = new Volume(new ExchangeRate((Fiat) getTradePrice().getMonetary()).coinToFiat(getTradeAmount())); + return OfferUtil.getRoundedFiatVolume(volume); + } + } + + public boolean isValid() { + return amount > 0 && + price > 0 && + date > 0 && + paymentMethod != null && + !paymentMethod.isEmpty() && + currency != null && + !currency.isEmpty(); + } + + @Override + public String toString() { + return "TradeStatistics3{" + + "\n currency='" + currency + '\'' + + ",\n price=" + price + + ",\n amount=" + amount + + ",\n paymentMethod='" + paymentMethod + '\'' + + ",\n date=" + date + + ",\n mediator='" + mediator + '\'' + + ",\n refundAgent='" + refundAgent + '\'' + + ",\n hash=" + Utilities.bytesAsHexString(hash) + + ",\n extraDataMap=" + extraDataMap + + "\n}"; + } +} diff --git a/core/src/main/java/bisq/core/trade/statistics/TradeStatistics3StorageService.java b/core/src/main/java/bisq/core/trade/statistics/TradeStatistics3StorageService.java new file mode 100644 index 00000000000..16398412319 --- /dev/null +++ b/core/src/main/java/bisq/core/trade/statistics/TradeStatistics3StorageService.java @@ -0,0 +1,84 @@ +/* + * This file is part of Bisq. + * + * bisq is free software: you can redistribute it and/or modify it + * under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or (at + * your option) any later version. + * + * bisq is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public + * License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with bisq. If not, see . + */ + +package bisq.core.trade.statistics; + +import bisq.network.p2p.storage.payload.PersistableNetworkPayload; +import bisq.network.p2p.storage.persistence.HistoricalDataStoreService; + +import bisq.common.config.Config; +import bisq.common.persistence.PersistenceManager; + +import javax.inject.Inject; +import javax.inject.Named; +import javax.inject.Singleton; + +import java.io.File; + +import lombok.extern.slf4j.Slf4j; + +@Singleton +@Slf4j +public class TradeStatistics3StorageService extends HistoricalDataStoreService { + private static final String FILE_NAME = "TradeStatistics3Store"; + + + /////////////////////////////////////////////////////////////////////////////////////////// + // Constructor + /////////////////////////////////////////////////////////////////////////////////////////// + + @Inject + public TradeStatistics3StorageService(@Named(Config.STORAGE_DIR) File storageDir, + PersistenceManager persistenceManager) { + super(storageDir, persistenceManager); + } + + + /////////////////////////////////////////////////////////////////////////////////////////// + // API + /////////////////////////////////////////////////////////////////////////////////////////// + + @Override + public String getFileName() { + return FILE_NAME; + } + + @Override + protected void initializePersistenceManager() { + persistenceManager.initialize(store, PersistenceManager.Source.NETWORK); + } + + @Override + public boolean canHandle(PersistableNetworkPayload payload) { + return payload instanceof TradeStatistics3; + } + + + /////////////////////////////////////////////////////////////////////////////////////////// + // Protected + /////////////////////////////////////////////////////////////////////////////////////////// + + @Override + protected TradeStatistics3Store createStore() { + return new TradeStatistics3Store(); + } + + public void persistNow() { + persistenceManager.persistNow(() -> { + }); + } +} diff --git a/core/src/main/java/bisq/core/trade/statistics/TradeStatistics3Store.java b/core/src/main/java/bisq/core/trade/statistics/TradeStatistics3Store.java new file mode 100644 index 00000000000..7c79778d4e1 --- /dev/null +++ b/core/src/main/java/bisq/core/trade/statistics/TradeStatistics3Store.java @@ -0,0 +1,73 @@ +/* + * This file is part of Bisq. + * + * Bisq is free software: you can redistribute it and/or modify it + * under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or (at + * your option) any later version. + * + * Bisq is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public + * License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with Bisq. If not, see . + */ + +package bisq.core.trade.statistics; + +import bisq.network.p2p.storage.P2PDataStorage; +import bisq.network.p2p.storage.persistence.PersistableNetworkPayloadStore; + +import com.google.protobuf.Message; + +import java.util.List; +import java.util.stream.Collectors; + +import lombok.extern.slf4j.Slf4j; + +/** + * We store only the payload in the PB file to save disc space. The hash of the payload can be created anyway and + * is only used as key in the map. So we have a hybrid data structure which is represented as list in the protobuffer + * definition and provide a hashMap for the domain access. + */ +@Slf4j +public class TradeStatistics3Store extends PersistableNetworkPayloadStore { + + TradeStatistics3Store() { + } + + + /////////////////////////////////////////////////////////////////////////////////////////// + // PROTO BUFFER + /////////////////////////////////////////////////////////////////////////////////////////// + + private TradeStatistics3Store(List list) { + list.forEach(item -> map.put(new P2PDataStorage.ByteArray(item.getHash()), item)); + } + + public Message toProtoMessage() { + return protobuf.PersistableEnvelope.newBuilder() + .setTradeStatistics3Store(getBuilder()) + .build(); + } + + private protobuf.TradeStatistics3Store.Builder getBuilder() { + List protoList = map.values().stream() + .map(payload -> (TradeStatistics3) payload) + .map(TradeStatistics3::toProtoTradeStatistics3) + .collect(Collectors.toList()); + return protobuf.TradeStatistics3Store.newBuilder().addAllItems(protoList); + } + + public static TradeStatistics3Store fromProto(protobuf.TradeStatistics3Store proto) { + List list = proto.getItemsList().stream() + .map(TradeStatistics3::fromProto).collect(Collectors.toList()); + return new TradeStatistics3Store(list); + } + + public boolean containsKey(P2PDataStorage.ByteArray hash) { + return map.containsKey(hash); + } +} diff --git a/core/src/main/java/bisq/core/trade/statistics/TradeStatisticsConverter.java b/core/src/main/java/bisq/core/trade/statistics/TradeStatisticsConverter.java new file mode 100644 index 00000000000..fc9e0bc4c5f --- /dev/null +++ b/core/src/main/java/bisq/core/trade/statistics/TradeStatisticsConverter.java @@ -0,0 +1,147 @@ +/* + * This file is part of Bisq. + * + * Bisq is free software: you can redistribute it and/or modify it + * under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or (at + * your option) any later version. + * + * Bisq is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public + * License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with Bisq. If not, see . + */ + +package bisq.core.trade.statistics; + +import bisq.network.p2p.BootstrapListener; +import bisq.network.p2p.P2PService; +import bisq.network.p2p.storage.P2PDataStorage; +import bisq.network.p2p.storage.payload.PersistableNetworkPayload; +import bisq.network.p2p.storage.persistence.AppendOnlyDataStoreService; + +import bisq.common.config.Config; +import bisq.common.file.FileUtil; + +import com.google.inject.Inject; + +import javax.inject.Named; + +import java.io.File; +import java.io.IOException; + +import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; + +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public class TradeStatisticsConverter { + + @Inject + public TradeStatisticsConverter(P2PService p2PService, + P2PDataStorage p2PDataStorage, + TradeStatistics2StorageService tradeStatistics2StorageService, + TradeStatistics3StorageService tradeStatistics3StorageService, + AppendOnlyDataStoreService appendOnlyDataStoreService, + @Named(Config.STORAGE_DIR) File storageDir) { + File tradeStatistics2Store = new File(storageDir, "TradeStatistics2Store"); + appendOnlyDataStoreService.addService(tradeStatistics2StorageService); + + p2PService.addP2PServiceListener(new BootstrapListener() { + @Override + public void onTorNodeReady() { + if (!tradeStatistics2Store.exists()) { + return; + } + + // We convert early once tor is initialized but still not ready to receive data + var mapOfLiveData = tradeStatistics3StorageService.getMapOfLiveData(); + convertToTradeStatistics3(tradeStatistics2StorageService.getMapOfAllData().values()) + .forEach(e -> mapOfLiveData.put(new P2PDataStorage.ByteArray(e.getHash()), e)); + tradeStatistics3StorageService.persistNow(); + try { + log.info("We delete now the old trade statistics file as it was converted to the new format."); + FileUtil.deleteFileIfExists(tradeStatistics2Store); + } catch (IOException e) { + e.printStackTrace(); + log.error(e.toString()); + } + } + + @Override + public void onUpdatedDataReceived() { + } + }); + + // We listen to old TradeStatistics2 objects, convert and store them and rebroadcast. + p2PDataStorage.addAppendOnlyDataStoreListener(payload -> { + if (payload instanceof TradeStatistics2) { + TradeStatistics3 tradeStatistics3 = convertToTradeStatistics3((TradeStatistics2) payload, true); + // We add it to the p2PDataStorage, which handles to get the data stored in the maps and maybe + // re-broadcast as tradeStatistics3 object if not already received. + p2PDataStorage.addPersistableNetworkPayload(tradeStatistics3, null, true); + } + }); + } + + private static Set convertToTradeStatistics3(Collection persistableNetworkPayloads) { + Set result = new HashSet<>(); + long ts = System.currentTimeMillis(); + + // We might have duplicate entries from both traders as the trade date was different from old clients. + // This should not be the case with converting old persisted data as we did filter those out but it is the case + // when we receive old trade stat objects from the network of 2 not updated traders. + // The hash was ignoring the trade date so we use that to get a unique list + Map mapWithoutDuplicates = new HashMap<>(); + persistableNetworkPayloads.stream() + .filter(e -> e instanceof TradeStatistics2) + .map(e -> (TradeStatistics2) e) + .filter(TradeStatistics2::isValid) + .forEach(e -> mapWithoutDuplicates.putIfAbsent(new P2PDataStorage.ByteArray(e.getHash()), e)); + + log.info("We convert the existing {} trade statistics objects to the new format. " + + "This might take a bit but is only done once.", mapWithoutDuplicates.size()); + + mapWithoutDuplicates.values().stream() + .map(e -> convertToTradeStatistics3(e, false)) + .filter(TradeStatistics3::isValid) + .forEach(result::add); + + log.info("Conversion to {} new trade statistic objects has been completed after {} ms", + result.size(), System.currentTimeMillis() - ts); + + return result; + } + + private static TradeStatistics3 convertToTradeStatistics3(TradeStatistics2 tradeStatistics2, boolean fromNetwork) { + Map extraDataMap = tradeStatistics2.getExtraDataMap(); + String mediator = extraDataMap != null ? extraDataMap.get(TradeStatistics2.MEDIATOR_ADDRESS) : null; + String refundAgent = extraDataMap != null ? extraDataMap.get(TradeStatistics2.REFUND_AGENT_ADDRESS) : null; + long time = tradeStatistics2.getTradeDate().getTime(); + byte[] hash = null; + if (fromNetwork) { + // We need to avoid that we duplicate tradeStatistics2 objects in case both traders have not udpated yet. + // Before v1.4.0 both traders published the trade statistics. If one trader has updated he will check + // the capabilities of the peer and if the peer has not updated he will leave publishing to the peer, so we + // do not have the problem of duplicated objects. + // To ensure we add only one object we will use the hash of the tradeStatistics2 object which is the same + // for both traders as it excluded the trade date which is different for both. + hash = tradeStatistics2.getHash(); + } + return new TradeStatistics3(tradeStatistics2.getCurrencyCode(), + tradeStatistics2.getTradePrice().getValue(), + tradeStatistics2.getTradeAmount().getValue(), + tradeStatistics2.getOfferPaymentMethod(), + time, + mediator, + refundAgent, + hash); + } +} diff --git a/monitor/src/main/java/bisq/monitor/Monitor.java b/monitor/src/main/java/bisq/monitor/Monitor.java index 19691c3d66f..c382b82cf24 100644 --- a/monitor/src/main/java/bisq/monitor/Monitor.java +++ b/monitor/src/main/java/bisq/monitor/Monitor.java @@ -90,7 +90,8 @@ private void start() throws Throwable { Capability.DAO_STATE, Capability.BUNDLE_OF_ENVELOPES, Capability.REFUND_AGENT, - Capability.MEDIATION); + Capability.MEDIATION, + Capability.TRADE_STATISTICS_3); // assemble Metrics // - create reporters diff --git a/proto/src/main/proto/pb.proto b/proto/src/main/proto/pb.proto index 11c32109811..9e701ae5f25 100644 --- a/proto/src/main/proto/pb.proto +++ b/proto/src/main/proto/pb.proto @@ -524,10 +524,11 @@ message StoragePayload { message PersistableNetworkPayload { oneof message { AccountAgeWitness account_age_witness = 1; - TradeStatistics2 trade_statistics2 = 2; + TradeStatistics2 trade_statistics2 = 2 [deprecated = true]; ProposalPayload proposal_payload = 3; BlindVotePayload blind_vote_payload = 4; SignedWitness signed_witness = 5; + TradeStatistics3 trade_statistics3 = 6; } } @@ -672,22 +673,34 @@ message TradeStatistics { } message TradeStatistics2 { - string base_currency = 1; - string counter_currency = 2; - OfferPayload.Direction direction = 3; - int64 trade_price = 4; - int64 trade_amount = 5; - int64 trade_date = 6; - string payment_method_id = 7; - int64 offer_date = 8; - bool offer_use_market_based_price = 9; - double offer_market_price_margin = 10; - int64 offer_amount = 11; - int64 offer_min_amount = 12; - string offer_id = 13; - string deposit_tx_id = 14; - bytes hash = 15; - map extra_data = 16; + string base_currency = 1 [deprecated = true]; + string counter_currency = 2 [deprecated = true]; + OfferPayload.Direction direction = 3 [deprecated = true]; + int64 trade_price = 4 [deprecated = true]; + int64 trade_amount = 5 [deprecated = true]; + int64 trade_date = 6 [deprecated = true]; + string payment_method_id = 7 [deprecated = true]; + int64 offer_date = 8 [deprecated = true]; + bool offer_use_market_based_price = 9 [deprecated = true]; + double offer_market_price_margin = 10 [deprecated = true]; + int64 offer_amount = 11 [deprecated = true]; + int64 offer_min_amount = 12 [deprecated = true]; + string offer_id = 13 [deprecated = true]; + string deposit_tx_id = 14 [deprecated = true]; + bytes hash = 15 [deprecated = true]; + map extra_data = 16 [deprecated = true]; +} + +message TradeStatistics3 { + string currency = 1; + int64 price = 2; + int64 amount = 3; + string payment_method = 4; + int64 date = 5; + string mediator = 6; + string refund_agent = 7; + bytes hash = 8; + map extra_data = 9; } message MailboxStoragePayload { @@ -1152,7 +1165,7 @@ message PersistableEnvelope { // BsqState bsq_state = 12; // not used but as other non-dao data have a higher index number we leave it to make clear that we cannot change following indexes AccountAgeWitnessStore account_age_witness_store = 13; - TradeStatistics2Store trade_statistics2_store = 14; + TradeStatistics2Store trade_statistics2_store = 14 [deprecated = true]; // PersistableNetworkPayloadList persistable_network_payload_list = 15; // long deprecated & migration away from it is already done @@ -1171,6 +1184,7 @@ message PersistableEnvelope { SignedWitnessStore signed_witness_store = 28; MediationDisputeList mediation_dispute_list = 29; RefundDisputeList refund_dispute_list = 30; + TradeStatistics3Store trade_statistics3_store = 31; } } @@ -1211,8 +1225,13 @@ message SignedWitnessStore { } // We use a list not a hash map to save disc space. The hash can be calculated from the payload anyway +// Deprecated message TradeStatistics2Store { - repeated TradeStatistics2 items = 1; + repeated TradeStatistics2 items = 1 [deprecated = true]; +} + +message TradeStatistics3Store { + repeated TradeStatistics3 items = 1; } message PeerList { From 406bcfb06474831638c975da836a4ca066262df9 Mon Sep 17 00:00:00 2001 From: chimp1984 Date: Mon, 5 Oct 2020 15:26:16 -0500 Subject: [PATCH 37/59] Remove PublishTradeStatistics from buyer protocol We let seller publish trade stats to avoid those issues with duplicated entries as trade date is different. We could fix that to use the same trade date, but it seems to be not needed that both traders are publishing and the risk if a trade stat does not get successfully published does not cause real problems. There is guarantee anyway that the data is broadcast even if both do it. In case we still want to do it from both sides we need to use the sellers trade date which is exchanged early in the trade protocol but yet not further used beside for account age check. --- .../src/main/java/bisq/core/trade/protocol/BuyerProtocol.java | 4 +--- desktop/src/main/java/bisq/desktop/main/debug/DebugView.java | 2 -- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/core/src/main/java/bisq/core/trade/protocol/BuyerProtocol.java b/core/src/main/java/bisq/core/trade/protocol/BuyerProtocol.java index 64c718ce412..ffff35a79bb 100644 --- a/core/src/main/java/bisq/core/trade/protocol/BuyerProtocol.java +++ b/core/src/main/java/bisq/core/trade/protocol/BuyerProtocol.java @@ -24,7 +24,6 @@ import bisq.core.trade.messages.PayoutTxPublishedMessage; import bisq.core.trade.messages.TradeMessage; import bisq.core.trade.protocol.tasks.ApplyFilter; -import bisq.core.trade.protocol.tasks.PublishTradeStatistics; import bisq.core.trade.protocol.tasks.TradeTask; import bisq.core.trade.protocol.tasks.buyer.BuyerProcessDepositTxAndDelayedPayoutTxMessage; import bisq.core.trade.protocol.tasks.buyer.BuyerProcessPayoutTxPublishedMessage; @@ -114,8 +113,7 @@ protected void handle(DepositTxAndDelayedPayoutTxMessage message, NodeAddress pe removeMailboxMessageAfterProcessing(message); })) .setup(tasks(BuyerProcessDepositTxAndDelayedPayoutTxMessage.class, - BuyerVerifiesFinalDelayedPayoutTx.class, - PublishTradeStatistics.class) + BuyerVerifiesFinalDelayedPayoutTx.class) .using(new TradeTaskRunner(trade, () -> { stopTimeout(); diff --git a/desktop/src/main/java/bisq/desktop/main/debug/DebugView.java b/desktop/src/main/java/bisq/desktop/main/debug/DebugView.java index 86b0c4d0d05..cbf2ca618d9 100644 --- a/desktop/src/main/java/bisq/desktop/main/debug/DebugView.java +++ b/desktop/src/main/java/bisq/desktop/main/debug/DebugView.java @@ -179,7 +179,6 @@ public void initialize() { BuyerProcessDepositTxAndDelayedPayoutTxMessage.class, BuyerVerifiesFinalDelayedPayoutTx.class, - PublishTradeStatistics.class, ApplyFilter.class, MakerVerifyTakerFeePayment.class, @@ -216,7 +215,6 @@ public void initialize() { BuyerProcessDepositTxAndDelayedPayoutTxMessage.class, BuyerVerifiesFinalDelayedPayoutTx.class, - PublishTradeStatistics.class, ApplyFilter.class, TakerVerifyMakerFeePayment.class, From b75aa6771f286d09a227acbcb125bc84d905ae94 Mon Sep 17 00:00:00 2001 From: chimp1984 Date: Mon, 5 Oct 2020 15:26:41 -0500 Subject: [PATCH 38/59] Refactor: Move class --- .../src/main/java/bisq/core/trade/protocol/SellerProtocol.java | 2 +- .../protocol/tasks/{ => seller}/PublishTradeStatistics.java | 3 ++- desktop/src/main/java/bisq/desktop/main/debug/DebugView.java | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) rename core/src/main/java/bisq/core/trade/protocol/tasks/{ => seller}/PublishTradeStatistics.java (96%) diff --git a/core/src/main/java/bisq/core/trade/protocol/SellerProtocol.java b/core/src/main/java/bisq/core/trade/protocol/SellerProtocol.java index 35eea700b6c..e5c559b1517 100644 --- a/core/src/main/java/bisq/core/trade/protocol/SellerProtocol.java +++ b/core/src/main/java/bisq/core/trade/protocol/SellerProtocol.java @@ -23,8 +23,8 @@ import bisq.core.trade.messages.DelayedPayoutTxSignatureResponse; import bisq.core.trade.messages.TradeMessage; import bisq.core.trade.protocol.tasks.ApplyFilter; -import bisq.core.trade.protocol.tasks.PublishTradeStatistics; import bisq.core.trade.protocol.tasks.TradeTask; +import bisq.core.trade.protocol.tasks.seller.PublishTradeStatistics; import bisq.core.trade.protocol.tasks.seller.SellerBroadcastPayoutTx; import bisq.core.trade.protocol.tasks.seller.SellerFinalizesDelayedPayoutTx; import bisq.core.trade.protocol.tasks.seller.SellerProcessCounterCurrencyTransferStartedMessage; diff --git a/core/src/main/java/bisq/core/trade/protocol/tasks/PublishTradeStatistics.java b/core/src/main/java/bisq/core/trade/protocol/tasks/seller/PublishTradeStatistics.java similarity index 96% rename from core/src/main/java/bisq/core/trade/protocol/tasks/PublishTradeStatistics.java rename to core/src/main/java/bisq/core/trade/protocol/tasks/seller/PublishTradeStatistics.java index 8eb71ffdd87..c894c5413b3 100644 --- a/core/src/main/java/bisq/core/trade/protocol/tasks/PublishTradeStatistics.java +++ b/core/src/main/java/bisq/core/trade/protocol/tasks/seller/PublishTradeStatistics.java @@ -15,11 +15,12 @@ * along with Bisq. If not, see . */ -package bisq.core.trade.protocol.tasks; +package bisq.core.trade.protocol.tasks.seller; import bisq.core.offer.Offer; import bisq.core.offer.OfferPayload; import bisq.core.trade.Trade; +import bisq.core.trade.protocol.tasks.TradeTask; import bisq.core.trade.statistics.TradeStatistics2; import bisq.network.p2p.NodeAddress; diff --git a/desktop/src/main/java/bisq/desktop/main/debug/DebugView.java b/desktop/src/main/java/bisq/desktop/main/debug/DebugView.java index cbf2ca618d9..16c767db612 100644 --- a/desktop/src/main/java/bisq/desktop/main/debug/DebugView.java +++ b/desktop/src/main/java/bisq/desktop/main/debug/DebugView.java @@ -27,7 +27,6 @@ import bisq.core.offer.placeoffer.tasks.CreateMakerFeeTx; import bisq.core.offer.placeoffer.tasks.ValidateOffer; import bisq.core.trade.protocol.tasks.ApplyFilter; -import bisq.core.trade.protocol.tasks.PublishTradeStatistics; import bisq.core.trade.protocol.tasks.VerifyPeersAccountAgeWitness; import bisq.core.trade.protocol.tasks.buyer.BuyerProcessDelayedPayoutTxSignatureRequest; import bisq.core.trade.protocol.tasks.buyer.BuyerProcessDepositTxAndDelayedPayoutTxMessage; @@ -50,6 +49,7 @@ import bisq.core.trade.protocol.tasks.maker.MakerRemovesOpenOffer; import bisq.core.trade.protocol.tasks.maker.MakerSetsLockTime; import bisq.core.trade.protocol.tasks.maker.MakerVerifyTakerFeePayment; +import bisq.core.trade.protocol.tasks.seller.PublishTradeStatistics; import bisq.core.trade.protocol.tasks.seller.SellerBroadcastPayoutTx; import bisq.core.trade.protocol.tasks.seller.SellerCreatesDelayedPayoutTx; import bisq.core.trade.protocol.tasks.seller.SellerFinalizesDelayedPayoutTx; From a522d0ade9870af3c4a51dc4d6ce59a123e1e34a Mon Sep 17 00:00:00 2001 From: chimp1984 Date: Mon, 5 Oct 2020 15:26:56 -0500 Subject: [PATCH 39/59] Refactor: Rename class --- .../main/java/bisq/core/trade/protocol/SellerProtocol.java | 4 ++-- ...eStatistics.java => SellerPublishesTradeStatistics.java} | 4 ++-- .../src/main/java/bisq/desktop/main/debug/DebugView.java | 6 +++--- 3 files changed, 7 insertions(+), 7 deletions(-) rename core/src/main/java/bisq/core/trade/protocol/tasks/seller/{PublishTradeStatistics.java => SellerPublishesTradeStatistics.java} (95%) diff --git a/core/src/main/java/bisq/core/trade/protocol/SellerProtocol.java b/core/src/main/java/bisq/core/trade/protocol/SellerProtocol.java index e5c559b1517..1b4815f4678 100644 --- a/core/src/main/java/bisq/core/trade/protocol/SellerProtocol.java +++ b/core/src/main/java/bisq/core/trade/protocol/SellerProtocol.java @@ -24,12 +24,12 @@ import bisq.core.trade.messages.TradeMessage; import bisq.core.trade.protocol.tasks.ApplyFilter; import bisq.core.trade.protocol.tasks.TradeTask; -import bisq.core.trade.protocol.tasks.seller.PublishTradeStatistics; import bisq.core.trade.protocol.tasks.seller.SellerBroadcastPayoutTx; import bisq.core.trade.protocol.tasks.seller.SellerFinalizesDelayedPayoutTx; import bisq.core.trade.protocol.tasks.seller.SellerProcessCounterCurrencyTransferStartedMessage; import bisq.core.trade.protocol.tasks.seller.SellerProcessDelayedPayoutTxSignatureResponse; import bisq.core.trade.protocol.tasks.seller.SellerPublishesDepositTx; +import bisq.core.trade.protocol.tasks.seller.SellerPublishesTradeStatistics; import bisq.core.trade.protocol.tasks.seller.SellerSendPayoutTxPublishedMessage; import bisq.core.trade.protocol.tasks.seller.SellerSendsDepositTxAndDelayedPayoutTxMessage; import bisq.core.trade.protocol.tasks.seller.SellerSignAndFinalizePayoutTx; @@ -79,7 +79,7 @@ protected void handle(DelayedPayoutTxSignatureResponse message, NodeAddress peer SellerFinalizesDelayedPayoutTx.class, SellerSendsDepositTxAndDelayedPayoutTxMessage.class, SellerPublishesDepositTx.class, - PublishTradeStatistics.class)) + SellerPublishesTradeStatistics.class)) .run(() -> { // We stop timeout here and don't start a new one as the // SellerSendsDepositTxAndDelayedPayoutTxMessage repeats the send the message and has it's own diff --git a/core/src/main/java/bisq/core/trade/protocol/tasks/seller/PublishTradeStatistics.java b/core/src/main/java/bisq/core/trade/protocol/tasks/seller/SellerPublishesTradeStatistics.java similarity index 95% rename from core/src/main/java/bisq/core/trade/protocol/tasks/seller/PublishTradeStatistics.java rename to core/src/main/java/bisq/core/trade/protocol/tasks/seller/SellerPublishesTradeStatistics.java index c894c5413b3..2dfb52e44f5 100644 --- a/core/src/main/java/bisq/core/trade/protocol/tasks/seller/PublishTradeStatistics.java +++ b/core/src/main/java/bisq/core/trade/protocol/tasks/seller/SellerPublishesTradeStatistics.java @@ -37,8 +37,8 @@ import static com.google.common.base.Preconditions.checkNotNull; @Slf4j -public class PublishTradeStatistics extends TradeTask { - public PublishTradeStatistics(TaskRunner taskHandler, Trade trade) { +public class SellerPublishesTradeStatistics extends TradeTask { + public SellerPublishesTradeStatistics(TaskRunner taskHandler, Trade trade) { super(taskHandler, trade); } diff --git a/desktop/src/main/java/bisq/desktop/main/debug/DebugView.java b/desktop/src/main/java/bisq/desktop/main/debug/DebugView.java index 16c767db612..89d0acbf090 100644 --- a/desktop/src/main/java/bisq/desktop/main/debug/DebugView.java +++ b/desktop/src/main/java/bisq/desktop/main/debug/DebugView.java @@ -49,13 +49,13 @@ import bisq.core.trade.protocol.tasks.maker.MakerRemovesOpenOffer; import bisq.core.trade.protocol.tasks.maker.MakerSetsLockTime; import bisq.core.trade.protocol.tasks.maker.MakerVerifyTakerFeePayment; -import bisq.core.trade.protocol.tasks.seller.PublishTradeStatistics; import bisq.core.trade.protocol.tasks.seller.SellerBroadcastPayoutTx; import bisq.core.trade.protocol.tasks.seller.SellerCreatesDelayedPayoutTx; import bisq.core.trade.protocol.tasks.seller.SellerFinalizesDelayedPayoutTx; import bisq.core.trade.protocol.tasks.seller.SellerProcessCounterCurrencyTransferStartedMessage; import bisq.core.trade.protocol.tasks.seller.SellerProcessDelayedPayoutTxSignatureResponse; import bisq.core.trade.protocol.tasks.seller.SellerPublishesDepositTx; +import bisq.core.trade.protocol.tasks.seller.SellerPublishesTradeStatistics; import bisq.core.trade.protocol.tasks.seller.SellerSendDelayedPayoutTxSignatureRequest; import bisq.core.trade.protocol.tasks.seller.SellerSendPayoutTxPublishedMessage; import bisq.core.trade.protocol.tasks.seller.SellerSendsDepositTxAndDelayedPayoutTxMessage; @@ -145,7 +145,7 @@ public void initialize() { SellerFinalizesDelayedPayoutTx.class, SellerSendsDepositTxAndDelayedPayoutTxMessage.class, SellerPublishesDepositTx.class, - PublishTradeStatistics.class, + SellerPublishesTradeStatistics.class, SellerProcessCounterCurrencyTransferStartedMessage.class, ApplyFilter.class, @@ -246,7 +246,7 @@ public void initialize() { SellerFinalizesDelayedPayoutTx.class, SellerSendsDepositTxAndDelayedPayoutTxMessage.class, SellerPublishesDepositTx.class, - PublishTradeStatistics.class, + SellerPublishesTradeStatistics.class, SellerProcessCounterCurrencyTransferStartedMessage.class, ApplyFilter.class, From b2665bdd2cf73659e41fa763f05abb19654560d8 Mon Sep 17 00:00:00 2001 From: chimp1984 Date: Mon, 5 Oct 2020 17:43:35 -0500 Subject: [PATCH 40/59] let seller publish trade statistics only if peer is updated user. If not the peer will publish only. --- .../SellerPublishesTradeStatistics.java | 76 ++++++++++++------- .../java/bisq/network/p2p/P2PService.java | 12 +++ 2 files changed, 62 insertions(+), 26 deletions(-) diff --git a/core/src/main/java/bisq/core/trade/protocol/tasks/seller/SellerPublishesTradeStatistics.java b/core/src/main/java/bisq/core/trade/protocol/tasks/seller/SellerPublishesTradeStatistics.java index 2dfb52e44f5..0489ac0e325 100644 --- a/core/src/main/java/bisq/core/trade/protocol/tasks/seller/SellerPublishesTradeStatistics.java +++ b/core/src/main/java/bisq/core/trade/protocol/tasks/seller/SellerPublishesTradeStatistics.java @@ -21,12 +21,13 @@ import bisq.core.offer.OfferPayload; import bisq.core.trade.Trade; import bisq.core.trade.protocol.tasks.TradeTask; -import bisq.core.trade.statistics.TradeStatistics2; +import bisq.core.trade.statistics.TradeStatistics3; import bisq.network.p2p.NodeAddress; import bisq.network.p2p.network.NetworkNode; import bisq.network.p2p.network.TorNetworkNode; +import bisq.common.app.Capability; import bisq.common.taskrunner.TaskRunner; import java.util.HashMap; @@ -49,31 +50,54 @@ protected void run() { checkNotNull(trade.getDepositTx()); - Map extraDataMap = new HashMap<>(); - if (processModel.getReferralIdService().getOptionalReferralId().isPresent()) { - extraDataMap.put(OfferPayload.REFERRAL_ID, processModel.getReferralIdService().getOptionalReferralId().get()); - } - - NodeAddress mediatorNodeAddress = checkNotNull(trade.getMediatorNodeAddress()); - // The first 4 chars are sufficient to identify a mediator. - // For testing with regtest/localhost we use the full address as its localhost and would result in - // same values for multiple mediators. - NetworkNode networkNode = model.getProcessModel().getP2PService().getNetworkNode(); - String address = networkNode instanceof TorNetworkNode ? - mediatorNodeAddress.getFullAddress().substring(0, 4) : - mediatorNodeAddress.getFullAddress(); - extraDataMap.put(TradeStatistics2.MEDIATOR_ADDRESS, address); - - Offer offer = checkNotNull(trade.getOffer()); - TradeStatistics2 tradeStatistics = new TradeStatistics2(offer.getOfferPayload(), - trade.getTradePrice(), - checkNotNull(trade.getTradeAmount()), - trade.getDate(), - trade.getDepositTxId(), - extraDataMap); - processModel.getP2PService().addPersistableNetworkPayload(tradeStatistics, true); - - complete(); + processModel.getP2PService().findPeersCapabilities(trade.getTradingPeerNodeAddress()) + .filter(capabilities -> capabilities.containsAll(Capability.TRADE_STATISTICS_3)) + .ifPresentOrElse(capabilities -> { + // Our peer has updated, so as we are the seller we will publish the trade statistics. + // The peer as buyer does not publish anymore with v.1.4.0 (where Capability.TRADE_STATISTICS_3 was added) + + Map extraDataMap = new HashMap<>(); + if (processModel.getReferralIdService().getOptionalReferralId().isPresent()) { + extraDataMap.put(OfferPayload.REFERRAL_ID, processModel.getReferralIdService().getOptionalReferralId().get()); + } + + NodeAddress mediatorNodeAddress = checkNotNull(trade.getMediatorNodeAddress()); + // The first 4 chars are sufficient to identify a mediator. + // For testing with regtest/localhost we use the full address as its localhost and would result in + // same values for multiple mediators. + NetworkNode networkNode = model.getProcessModel().getP2PService().getNetworkNode(); + String truncatedMediatorNodeAddress = networkNode instanceof TorNetworkNode ? + mediatorNodeAddress.getFullAddress().substring(0, 4) : + mediatorNodeAddress.getFullAddress(); + + NodeAddress refundAgentNodeAddress = checkNotNull(trade.getRefundAgentNodeAddress()); + String truncatedRefundAgentNodeAddress = networkNode instanceof TorNetworkNode ? + refundAgentNodeAddress.getFullAddress().substring(0, 4) : + refundAgentNodeAddress.getFullAddress(); + + Offer offer = checkNotNull(trade.getOffer()); + TradeStatistics3 tradeStatistics = new TradeStatistics3(offer.getCurrencyCode(), + trade.getTradePrice().getValue(), + trade.getTradeAmountAsLong(), + offer.getPaymentMethod().getId(), + trade.getTakeOfferDate().getTime(), + truncatedMediatorNodeAddress, + truncatedRefundAgentNodeAddress, + extraDataMap); + if (tradeStatistics.isValid()) { + log.info("Publishing trade statistics"); + processModel.getP2PService().addPersistableNetworkPayload(tradeStatistics, true); + } else { + log.warn("Trade statistics are invalid. We do not publish. {}", tradeStatistics); + } + + complete(); + }, + () -> { + log.info("Our peer does not has updated yet, so they will publish the trade statistics. " + + "To avoid duplicates we do not publish from our side."); + complete(); + }); } catch (Throwable t) { failed(t); } diff --git a/p2p/src/main/java/bisq/network/p2p/P2PService.java b/p2p/src/main/java/bisq/network/p2p/P2PService.java index 95b68f0c611..bf594ecf6fc 100644 --- a/p2p/src/main/java/bisq/network/p2p/P2PService.java +++ b/p2p/src/main/java/bisq/network/p2p/P2PService.java @@ -920,6 +920,18 @@ public KeyRing getKeyRing() { return keyRing; } + public Optional findPeersCapabilities(NodeAddress peer) { + return networkNode.getConfirmedConnections().stream() + .filter(e -> e.getPeersNodeAddressOptional().isPresent()) + .filter(e -> e.getPeersNodeAddressOptional().get().equals(peer)) + .map(Connection::getCapabilities) + .findAny(); + } + + + /////////////////////////////////////////////////////////////////////////////////////////// + // Private + /////////////////////////////////////////////////////////////////////////////////////////// @Value public class MailboxItem { From 6d43c09b33e4a5de6fb0cae17d3430c13d89b9f1 Mon Sep 17 00:00:00 2001 From: chimp1984 Date: Mon, 5 Oct 2020 17:53:13 -0500 Subject: [PATCH 41/59] Delete TradeStatistics (version 1) --- .../network/CoreNetworkProtoResolver.java | 4 - .../trade/statistics/TradeStatistics.java | 236 ------------------ proto/src/main/proto/pb.proto | 26 +- 3 files changed, 2 insertions(+), 264 deletions(-) delete mode 100644 core/src/main/java/bisq/core/trade/statistics/TradeStatistics.java diff --git a/core/src/main/java/bisq/core/proto/network/CoreNetworkProtoResolver.java b/core/src/main/java/bisq/core/proto/network/CoreNetworkProtoResolver.java index 8d57a75c14b..675011cec3d 100644 --- a/core/src/main/java/bisq/core/proto/network/CoreNetworkProtoResolver.java +++ b/core/src/main/java/bisq/core/proto/network/CoreNetworkProtoResolver.java @@ -59,7 +59,6 @@ import bisq.core.trade.messages.PeerPublishedDelayedPayoutTxMessage; import bisq.core.trade.messages.RefreshTradeStateRequest; import bisq.core.trade.messages.TraderSignedWitnessMessage; -import bisq.core.trade.statistics.TradeStatistics; import bisq.network.p2p.AckMessage; import bisq.network.p2p.BundleOfEnvelopes; @@ -265,9 +264,6 @@ public NetworkPayload fromProto(protobuf.StoragePayload proto) { return RefundAgent.fromProto(proto.getRefundAgent()); case FILTER: return Filter.fromProto(proto.getFilter()); - case TRADE_STATISTICS: - // Still used to convert TradeStatistics data from pre v0.6 versions - return TradeStatistics.fromProto(proto.getTradeStatistics()); case MAILBOX_STORAGE_PAYLOAD: return MailboxStoragePayload.fromProto(proto.getMailboxStoragePayload()); case OFFER_PAYLOAD: diff --git a/core/src/main/java/bisq/core/trade/statistics/TradeStatistics.java b/core/src/main/java/bisq/core/trade/statistics/TradeStatistics.java deleted file mode 100644 index ad543213191..00000000000 --- a/core/src/main/java/bisq/core/trade/statistics/TradeStatistics.java +++ /dev/null @@ -1,236 +0,0 @@ -/* - * This file is part of Bisq. - * - * Bisq is free software: you can redistribute it and/or modify it - * under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or (at - * your option) any later version. - * - * Bisq is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public - * License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with Bisq. If not, see . - */ - -package bisq.core.trade.statistics; - -import bisq.core.monetary.Altcoin; -import bisq.core.monetary.AltcoinExchangeRate; -import bisq.core.monetary.Price; -import bisq.core.monetary.Volume; -import bisq.core.offer.OfferPayload; - -import bisq.network.p2p.storage.payload.ExpirablePayload; -import bisq.network.p2p.storage.payload.ProcessOncePersistableNetworkPayload; -import bisq.network.p2p.storage.payload.ProtectedStoragePayload; - -import bisq.common.crypto.Sig; -import bisq.common.proto.persistable.PersistablePayload; -import bisq.common.util.CollectionUtils; -import bisq.common.util.ExtraDataMapValidator; -import bisq.common.util.JsonExclude; - -import com.google.protobuf.ByteString; - -import org.bitcoinj.core.Coin; -import org.bitcoinj.utils.ExchangeRate; -import org.bitcoinj.utils.Fiat; - -import java.security.PublicKey; - -import java.util.Date; -import java.util.Map; -import java.util.Optional; -import java.util.concurrent.TimeUnit; - -import lombok.EqualsAndHashCode; -import lombok.Value; -import lombok.extern.slf4j.Slf4j; - -import javax.annotation.Nullable; - -/** - * @deprecated Was used in pre v0.6.0 version - */ -@Deprecated -@Slf4j -@EqualsAndHashCode(exclude = {"signaturePubKeyBytes"}) -@Value -public final class TradeStatistics implements ProcessOncePersistableNetworkPayload, ProtectedStoragePayload, ExpirablePayload, PersistablePayload { - private final OfferPayload.Direction direction; - private final String baseCurrency; - private final String counterCurrency; - private final String offerPaymentMethod; - private final long offerDate; - private final boolean offerUseMarketBasedPrice; - private final double offerMarketPriceMargin; - private final long offerAmount; - private final long offerMinAmount; - private final String offerId; - private final long tradePrice; - private final long tradeAmount; - private final long tradeDate; - private final String depositTxId; - @JsonExclude - private final byte[] signaturePubKeyBytes; - @JsonExclude - transient private final PublicKey signaturePubKey; - - // Should be only used in emergency case if we need to add data but do not want to break backward compatibility - // at the P2P network storage checks. The hash of the object will be used to verify if the data is valid. Any new - // field in a class would break that hash and therefore break the storage mechanism. - @Nullable - private Map extraDataMap; - - public TradeStatistics(OfferPayload offerPayload, - Price tradePrice, - Coin tradeAmount, - Date tradeDate, - String depositTxId, - byte[] signaturePubKeyBytes) { - this(offerPayload.getDirection(), - offerPayload.getBaseCurrencyCode(), - offerPayload.getCounterCurrencyCode(), - offerPayload.getPaymentMethodId(), - offerPayload.getDate(), - offerPayload.isUseMarketBasedPrice(), - offerPayload.getMarketPriceMargin(), - offerPayload.getAmount(), - offerPayload.getMinAmount(), - offerPayload.getId(), - tradePrice.getValue(), - tradeAmount.value, - tradeDate.getTime(), - depositTxId, - signaturePubKeyBytes, - null); - } - - - /////////////////////////////////////////////////////////////////////////////////////////// - // PROTO BUFFER - /////////////////////////////////////////////////////////////////////////////////////////// - - TradeStatistics(OfferPayload.Direction direction, - String baseCurrency, - String counterCurrency, - String offerPaymentMethod, - long offerDate, - boolean offerUseMarketBasedPrice, - double offerMarketPriceMargin, - long offerAmount, - long offerMinAmount, - String offerId, - long tradePrice, - long tradeAmount, - long tradeDate, - String depositTxId, - byte[] signaturePubKeyBytes, - @Nullable Map extraDataMap) { - this.direction = direction; - this.baseCurrency = baseCurrency; - this.counterCurrency = counterCurrency; - this.offerPaymentMethod = offerPaymentMethod; - this.offerDate = offerDate; - this.offerUseMarketBasedPrice = offerUseMarketBasedPrice; - this.offerMarketPriceMargin = offerMarketPriceMargin; - this.offerAmount = offerAmount; - this.offerMinAmount = offerMinAmount; - this.offerId = offerId; - this.tradePrice = tradePrice; - this.tradeAmount = tradeAmount; - this.tradeDate = tradeDate; - this.depositTxId = depositTxId; - this.signaturePubKeyBytes = signaturePubKeyBytes; - this.extraDataMap = ExtraDataMapValidator.getValidatedExtraDataMap(extraDataMap); - - signaturePubKey = Sig.getPublicKeyFromBytes(signaturePubKeyBytes); - } - - @Override - public protobuf.StoragePayload toProtoMessage() { - final protobuf.TradeStatistics.Builder builder = protobuf.TradeStatistics.newBuilder() - .setDirection(OfferPayload.Direction.toProtoMessage(direction)) - .setBaseCurrency(baseCurrency) - .setCounterCurrency(counterCurrency) - .setPaymentMethodId(offerPaymentMethod) - .setOfferDate(offerDate) - .setOfferUseMarketBasedPrice(offerUseMarketBasedPrice) - .setOfferMarketPriceMargin(offerMarketPriceMargin) - .setOfferAmount(offerAmount) - .setOfferMinAmount(offerMinAmount) - .setOfferId(offerId) - .setTradePrice(tradePrice) - .setTradeAmount(tradeAmount) - .setTradeDate(tradeDate) - .setDepositTxId(depositTxId) - .setSignaturePubKeyBytes(ByteString.copyFrom(signaturePubKeyBytes)); - Optional.ofNullable(extraDataMap).ifPresent(builder::putAllExtraData); - return protobuf.StoragePayload.newBuilder().setTradeStatistics(builder).build(); - } - - public protobuf.TradeStatistics toProtoTradeStatistics() { - return toProtoMessage().getTradeStatistics(); - } - - public static TradeStatistics fromProto(protobuf.TradeStatistics proto) { - return new TradeStatistics( - OfferPayload.Direction.fromProto(proto.getDirection()), - proto.getBaseCurrency(), - proto.getCounterCurrency(), - proto.getPaymentMethodId(), - proto.getOfferDate(), - proto.getOfferUseMarketBasedPrice(), - proto.getOfferMarketPriceMargin(), - proto.getOfferAmount(), - proto.getOfferMinAmount(), - proto.getOfferId(), - proto.getTradePrice(), - proto.getTradeAmount(), - proto.getTradeDate(), - proto.getDepositTxId(), - proto.getSignaturePubKeyBytes().toByteArray(), - CollectionUtils.isEmpty(proto.getExtraDataMap()) ? null : proto.getExtraDataMap()); - } - - - /////////////////////////////////////////////////////////////////////////////////////////// - // Getters - /////////////////////////////////////////////////////////////////////////////////////////// - - @Override - public long getTTL() { - return TimeUnit.DAYS.toMillis(30); - } - - @Override - public PublicKey getOwnerPubKey() { - return signaturePubKey; - } - - public Date getTradeDate() { - return new Date(tradeDate); - } - - public Price getTradePrice() { - return Price.valueOf(getCurrencyCode(), tradePrice); - } - - public String getCurrencyCode() { - return baseCurrency.equals("BTC") ? counterCurrency : baseCurrency; - } - - public Coin getTradeAmount() { - return Coin.valueOf(tradeAmount); - } - - public Volume getTradeVolume() { - if (getTradePrice().getMonetary() instanceof Altcoin) - return new Volume(new AltcoinExchangeRate((Altcoin) getTradePrice().getMonetary()).coinToAltcoin(getTradeAmount())); - else - return new Volume(new ExchangeRate((Fiat) getTradePrice().getMonetary()).coinToFiat(getTradeAmount())); - } -} diff --git a/proto/src/main/proto/pb.proto b/proto/src/main/proto/pb.proto index 9e701ae5f25..8d7b152bd27 100644 --- a/proto/src/main/proto/pb.proto +++ b/proto/src/main/proto/pb.proto @@ -510,9 +510,7 @@ message StoragePayload { Mediator mediator = 3; Filter filter = 4; - // not used anymore from v0.6 on. But leave it for receiving TradeStatistics objects from older - // versions and convert it to TradeStatistics2 objects. - TradeStatistics trade_statistics = 5 [deprecated = true]; + // TradeStatistics trade_statistics = 5 [deprecated = true]; Removed in v.1.4.0 MailboxStoragePayload mailbox_storage_payload = 6; OfferPayload offer_payload = 7; @@ -651,27 +649,7 @@ message Filter { bool disable_auto_conf = 24; } -// not used anymore from v0.6 on. But leave it for receiving TradeStatistics objects from older -// versions and convert it to TradeStatistics2 objects. -message TradeStatistics { - string base_currency = 1; - string counter_currency = 2; - OfferPayload.Direction direction = 3; - int64 trade_price = 4; - int64 trade_amount = 5; - int64 trade_date = 6; - string payment_method_id = 7; - int64 offer_date = 8; - bool offer_use_market_based_price = 9; - double offer_market_price_margin = 10; - int64 offer_amount = 11; - int64 offer_min_amount = 12; - string offer_id = 13; - string deposit_tx_id = 14; - bytes signature_pub_key_bytes = 15; - map extra_data = 16; -} - +// Deprecated message TradeStatistics2 { string base_currency = 1 [deprecated = true]; string counter_currency = 2 [deprecated = true]; From 98207518d61bdabfc3c26a285c096aa6b8df41cc Mon Sep 17 00:00:00 2001 From: chimp1984 Date: Mon, 5 Oct 2020 17:59:24 -0500 Subject: [PATCH 42/59] We "hack" TradeStatistics2StorageService to fulfill our needs: 1. We do not want that initial data request/response use old trades statistics for excluded key hashes. Thats why we return an empty map in getMap. 2. We do not read resource file as we have removed that. 3. We do not persist as we convert the existing data and re-publish as new data, or at startup we convert the old data to the new one and then delete the file. --- .../TradeStatistics2StorageService.java | 23 +++++++++++++++++-- .../storage/persistence/MapStoreService.java | 2 +- 2 files changed, 22 insertions(+), 3 deletions(-) diff --git a/core/src/main/java/bisq/core/trade/statistics/TradeStatistics2StorageService.java b/core/src/main/java/bisq/core/trade/statistics/TradeStatistics2StorageService.java index 52822e2f2e0..27024943903 100644 --- a/core/src/main/java/bisq/core/trade/statistics/TradeStatistics2StorageService.java +++ b/core/src/main/java/bisq/core/trade/statistics/TradeStatistics2StorageService.java @@ -29,6 +29,7 @@ import java.io.File; +import java.util.HashMap; import java.util.Map; import lombok.extern.slf4j.Slf4j; @@ -65,11 +66,29 @@ public String getFileName() { @Override public Map getMap() { - return store.getMap(); + // As it is used for data request and response and we do not want to send any old trade stat data anymore. + return new HashMap<>(); + } + + // We overwrite that method to receive old trade stats from the network. As we deactivated getMap to not deliver + // hashes we needed to use the getMapOfAllData method to actually store the data. + // That's a bit of a hack but it's just for transition and can be removed after a few months anyway. + // Alternatively we could create a new interface to handle it differently on the other client classes but that + // seems to be not justified as it is needed only temporarily. + @Override + protected PersistableNetworkPayload putIfAbsent(P2PDataStorage.ByteArray hash, PersistableNetworkPayload payload) { + PersistableNetworkPayload previous = getMapOfAllData().putIfAbsent(hash, payload); + return previous; + } + + @Override + protected void readFromResources(String postFix) { + // We do not attempt to read from resources as that file is not provided anymore + readStore(); } public Map getMapOfAllData() { - return getMap(); + return store.getMap(); } @Override diff --git a/p2p/src/main/java/bisq/network/p2p/storage/persistence/MapStoreService.java b/p2p/src/main/java/bisq/network/p2p/storage/persistence/MapStoreService.java index 52f37bf44e5..7be3e4d1906 100644 --- a/p2p/src/main/java/bisq/network/p2p/storage/persistence/MapStoreService.java +++ b/p2p/src/main/java/bisq/network/p2p/storage/persistence/MapStoreService.java @@ -61,7 +61,7 @@ void put(P2PDataStorage.ByteArray hash, R payload) { requestPersistence(); } - R putIfAbsent(P2PDataStorage.ByteArray hash, R payload) { + protected R putIfAbsent(P2PDataStorage.ByteArray hash, R payload) { R previous = getMap().putIfAbsent(hash, payload); requestPersistence(); return previous; From c4a4c878b86830adaf8fb20899123034a508a4a2 Mon Sep 17 00:00:00 2001 From: chimp1984 Date: Mon, 5 Oct 2020 18:12:49 -0500 Subject: [PATCH 43/59] Apply TradeStatistics3 to TradeStatisticsManager and some related classes --- .../core/provider/price/PriceFeedService.java | 17 +-- .../java/bisq/core/trade/TradeModule.java | 4 - .../statistics/TradeStatisticsForJson.java | 52 +++---- .../statistics/TradeStatisticsManager.java | 129 ++++++------------ 4 files changed, 67 insertions(+), 135 deletions(-) diff --git a/core/src/main/java/bisq/core/provider/price/PriceFeedService.java b/core/src/main/java/bisq/core/provider/price/PriceFeedService.java index 3db0e855e09..168812b4f43 100644 --- a/core/src/main/java/bisq/core/provider/price/PriceFeedService.java +++ b/core/src/main/java/bisq/core/provider/price/PriceFeedService.java @@ -23,7 +23,7 @@ import bisq.core.monetary.Price; import bisq.core.provider.PriceNodeHttpClient; import bisq.core.provider.ProvidersRepository; -import bisq.core.trade.statistics.TradeStatistics2; +import bisq.core.trade.statistics.TradeStatistics3; import bisq.core.user.Preferences; import bisq.network.http.HttpClient; @@ -50,6 +50,7 @@ import java.time.Instant; import java.util.ArrayList; +import java.util.Comparator; import java.util.Date; import java.util.HashMap; import java.util.List; @@ -288,12 +289,12 @@ public Date getLastRequestTimeStamp() { return new Date(epochInMillisAtLastRequest); } - public void applyLatestBisqMarketPrice(Set tradeStatisticsSet) { + public void applyLatestBisqMarketPrice(Set tradeStatisticsSet) { // takes about 10 ms for 5000 items - Map> mapByCurrencyCode = new HashMap<>(); + Map> mapByCurrencyCode = new HashMap<>(); tradeStatisticsSet.forEach(e -> { - final List list; - final String currencyCode = e.getCurrencyCode(); + List list; + String currencyCode = e.getCurrency(); if (mapByCurrencyCode.containsKey(currencyCode)) { list = mapByCurrencyCode.get(currencyCode); } else { @@ -306,9 +307,9 @@ public void applyLatestBisqMarketPrice(Set tradeStatisticsSet) mapByCurrencyCode.values().stream() .filter(list -> !list.isEmpty()) .forEach(list -> { - list.sort((o1, o2) -> o1.getTradeDate().compareTo(o2.getTradeDate())); - TradeStatistics2 tradeStatistics = list.get(list.size() - 1); - setBisqMarketPrice(tradeStatistics.getCurrencyCode(), tradeStatistics.getTradePrice()); + list.sort(Comparator.comparing(o -> o.getTradeDate())); + TradeStatistics3 tradeStatistics = list.get(list.size() - 1); + setBisqMarketPrice(tradeStatistics.getCurrency(), tradeStatistics.getTradePrice()); }); } diff --git a/core/src/main/java/bisq/core/trade/TradeModule.java b/core/src/main/java/bisq/core/trade/TradeModule.java index 779cf6e7600..751bd5a0eb1 100644 --- a/core/src/main/java/bisq/core/trade/TradeModule.java +++ b/core/src/main/java/bisq/core/trade/TradeModule.java @@ -24,8 +24,6 @@ import bisq.core.trade.closed.ClosedTradableManager; import bisq.core.trade.failed.FailedTradesManager; import bisq.core.trade.statistics.ReferralIdService; -import bisq.core.trade.statistics.TradeStatistics2StorageService; -import bisq.core.trade.statistics.TradeStatisticsManager; import bisq.common.app.AppModule; import bisq.common.config.Config; @@ -46,8 +44,6 @@ public TradeModule(Config config) { @Override protected void configure() { bind(TradeManager.class).in(Singleton.class); - bind(TradeStatisticsManager.class).in(Singleton.class); - bind(TradeStatistics2StorageService.class).in(Singleton.class); bind(ClosedTradableManager.class).in(Singleton.class); bind(FailedTradesManager.class).in(Singleton.class); bind(AccountAgeWitnessService.class).in(Singleton.class); diff --git a/core/src/main/java/bisq/core/trade/statistics/TradeStatisticsForJson.java b/core/src/main/java/bisq/core/trade/statistics/TradeStatisticsForJson.java index fbf675dd8e5..ebff70616be 100644 --- a/core/src/main/java/bisq/core/trade/statistics/TradeStatisticsForJson.java +++ b/core/src/main/java/bisq/core/trade/statistics/TradeStatisticsForJson.java @@ -21,7 +21,6 @@ import bisq.core.locale.Res; import bisq.core.monetary.Price; import bisq.core.monetary.Volume; -import bisq.core.offer.OfferPayload; import bisq.common.util.MathUtils; @@ -38,65 +37,44 @@ @ToString @Slf4j public final class TradeStatisticsForJson { - public final String currency; - public final OfferPayload.Direction direction; public final long tradePrice; public final long tradeAmount; public final long tradeDate; public final String paymentMethod; - public final long offerDate; - public final boolean useMarketBasedPrice; - public final double marketPriceMargin; - public final long offerAmount; - public final long offerMinAmount; - public final String offerId; - public final String depositTxId; // primaryMarket fields are based on industry standard where primaryMarket is always in the focus (in the app BTC is always in the focus - will be changed in a larger refactoring once) public String currencyPair; - public OfferPayload.Direction primaryMarketDirection; public long primaryMarketTradePrice; public long primaryMarketTradeAmount; public long primaryMarketTradeVolume; - public TradeStatisticsForJson(TradeStatistics2 tradeStatistics) { - this.direction = OfferPayload.Direction.valueOf(tradeStatistics.getDirection().name()); - this.currency = tradeStatistics.getCurrencyCode(); - this.paymentMethod = tradeStatistics.getOfferPaymentMethod(); - this.offerDate = tradeStatistics.getOfferDate(); - this.useMarketBasedPrice = tradeStatistics.isOfferUseMarketBasedPrice(); - this.marketPriceMargin = tradeStatistics.getOfferMarketPriceMargin(); - this.offerAmount = tradeStatistics.getOfferAmount(); - this.offerMinAmount = tradeStatistics.getOfferMinAmount(); - this.offerId = tradeStatistics.getOfferId(); - this.tradePrice = tradeStatistics.getTradePrice().getValue(); - this.tradeAmount = tradeStatistics.getTradeAmount().getValue(); - this.tradeDate = tradeStatistics.getTradeDate().getTime(); - this.depositTxId = tradeStatistics.getDepositTxId(); + public TradeStatisticsForJson(TradeStatistics3 tradeStatistics) { + this.currency = tradeStatistics.getCurrency(); + this.paymentMethod = tradeStatistics.getPaymentMethod(); + this.tradePrice = tradeStatistics.getPrice(); + this.tradeAmount = tradeStatistics.getAmount(); + this.tradeDate = tradeStatistics.getDate(); try { - final Price tradePrice = getTradePrice(); + Price tradePrice = getTradePrice(); if (CurrencyUtil.isCryptoCurrency(currency)) { - primaryMarketDirection = direction == OfferPayload.Direction.BUY ? OfferPayload.Direction.SELL : OfferPayload.Direction.BUY; currencyPair = currency + "/" + Res.getBaseCurrencyCode(); - primaryMarketTradePrice = tradePrice.getValue(); - - primaryMarketTradeAmount = getTradeVolume() != null ? getTradeVolume().getValue() : 0; + primaryMarketTradeAmount = getTradeVolume() != null ? + getTradeVolume().getValue() : + 0; primaryMarketTradeVolume = getTradeAmount().getValue(); } else { - primaryMarketDirection = direction; currencyPair = Res.getBaseCurrencyCode() + "/" + currency; - // we use precision 4 for fiat based price but on the markets api we use precision 8 so we scale up by 10000 primaryMarketTradePrice = (long) MathUtils.scaleUpByPowerOf10(tradePrice.getValue(), 4); - primaryMarketTradeAmount = getTradeAmount().getValue(); // we use precision 4 for fiat but on the markets api we use precision 8 so we scale up by 10000 primaryMarketTradeVolume = getTradeVolume() != null ? - (long) MathUtils.scaleUpByPowerOf10(getTradeVolume().getValue(), 4) : 0; + (long) MathUtils.scaleUpByPowerOf10(getTradeVolume().getValue(), 4) : + 0; } } catch (Throwable t) { log.error(t.getMessage()); @@ -113,6 +91,10 @@ public Coin getTradeAmount() { } public Volume getTradeVolume() { - return getTradePrice().getVolumeByAmount(getTradeAmount()); + try { + return getTradePrice().getVolumeByAmount(getTradeAmount()); + } catch (Throwable t) { + return Volume.parse("0", currency); + } } } diff --git a/core/src/main/java/bisq/core/trade/statistics/TradeStatisticsManager.java b/core/src/main/java/bisq/core/trade/statistics/TradeStatisticsManager.java index 4dbe86c45bb..7e571391c84 100644 --- a/core/src/main/java/bisq/core/trade/statistics/TradeStatisticsManager.java +++ b/core/src/main/java/bisq/core/trade/statistics/TradeStatisticsManager.java @@ -32,6 +32,7 @@ import com.google.inject.Inject; import javax.inject.Named; +import javax.inject.Singleton; import javafx.collections.FXCollections; import javafx.collections.ObservableSet; @@ -40,94 +41,75 @@ import java.util.ArrayList; import java.util.List; -import java.util.Objects; -import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; import lombok.extern.slf4j.Slf4j; +@Singleton @Slf4j public class TradeStatisticsManager { - - private final JsonFileManager jsonFileManager; private final P2PService p2PService; private final PriceFeedService priceFeedService; - private final TradeStatistics2StorageService tradeStatistics2StorageService; + private final TradeStatistics3StorageService tradeStatistics3StorageService; + private final File storageDir; private final boolean dumpStatistics; - private final ObservableSet observableTradeStatisticsSet = FXCollections.observableSet(); + private final ObservableSet observableTradeStatisticsSet = FXCollections.observableSet(); + private JsonFileManager jsonFileManager; @Inject public TradeStatisticsManager(P2PService p2PService, PriceFeedService priceFeedService, - TradeStatistics2StorageService tradeStatistics2StorageService, + TradeStatistics3StorageService tradeStatistics3StorageService, AppendOnlyDataStoreService appendOnlyDataStoreService, @Named(Config.STORAGE_DIR) File storageDir, @Named(Config.DUMP_STATISTICS) boolean dumpStatistics) { this.p2PService = p2PService; this.priceFeedService = priceFeedService; - this.tradeStatistics2StorageService = tradeStatistics2StorageService; + this.tradeStatistics3StorageService = tradeStatistics3StorageService; + this.storageDir = storageDir; this.dumpStatistics = dumpStatistics; - jsonFileManager = new JsonFileManager(storageDir); - appendOnlyDataStoreService.addService(tradeStatistics2StorageService); + + appendOnlyDataStoreService.addService(tradeStatistics3StorageService); } public void onAllServicesInitialized() { p2PService.getP2PDataStorage().addAppendOnlyDataStoreListener(payload -> { - if (payload instanceof TradeStatistics2) - addToSet((TradeStatistics2) payload); + if (payload instanceof TradeStatistics3) { + TradeStatistics3 tradeStatistics = (TradeStatistics3) payload; + if (!tradeStatistics.isValid()) { + return; + } + observableTradeStatisticsSet.add(tradeStatistics); + priceFeedService.applyLatestBisqMarketPrice(observableTradeStatisticsSet); + maybeDump(); + } }); - Set set = tradeStatistics2StorageService.getMapOfAllData().values().stream() - .filter(e -> e instanceof TradeStatistics2) - .map(e -> (TradeStatistics2) e) - .map(WrapperTradeStatistics2::new) - .distinct() - .map(WrapperTradeStatistics2::unwrap) - .filter(TradeStatistics2::isValid) + Set set = tradeStatistics3StorageService.getMapOfAllData().values().stream() + .filter(e -> e instanceof TradeStatistics3) + .map(e -> (TradeStatistics3) e) + .filter(TradeStatistics3::isValid) .collect(Collectors.toSet()); observableTradeStatisticsSet.addAll(set); - priceFeedService.applyLatestBisqMarketPrice(observableTradeStatisticsSet); - - dump(); + maybeDump(); } - public ObservableSet getObservableTradeStatisticsSet() { + public ObservableSet getObservableTradeStatisticsSet() { return observableTradeStatisticsSet; } - private void addToSet(TradeStatistics2 tradeStatistics) { - if (!observableTradeStatisticsSet.contains(tradeStatistics)) { - Optional duplicate = observableTradeStatisticsSet.stream().filter( - e -> e.getOfferId().equals(tradeStatistics.getOfferId())).findAny(); - - if (duplicate.isPresent()) { - // TODO: Can be removed as soon as everyone uses v1.2.6+ - // Removes an existing object with a trade id if the new one matches the existing except - // for the deposit tx id - if (tradeStatistics.getDepositTxId() == null && - tradeStatistics.isValid() && - duplicate.get().compareTo(tradeStatistics) == 0) { - observableTradeStatisticsSet.remove(duplicate.get()); - } else { - return; - } - } - - if (!tradeStatistics.isValid()) { - return; - } - - observableTradeStatisticsSet.add(tradeStatistics); - priceFeedService.applyLatestBisqMarketPrice(observableTradeStatisticsSet); - dump(); + private void maybeDump() { + if (!dumpStatistics) { + return; } - } - private void dump() { - if (dumpStatistics) { + if (jsonFileManager == null) { + jsonFileManager = new JsonFileManager(storageDir); + + // We only dump once the currencies as they do not change during runtime ArrayList fiatCurrencyList = CurrencyUtil.getAllSortedFiatCurrencies().stream() .map(e -> new CurrencyTuple(e.getCode(), e.getName(), 8)) .collect(Collectors.toCollection(ArrayList::new)); @@ -138,44 +120,15 @@ private void dump() { .collect(Collectors.toCollection(ArrayList::new)); cryptoCurrencyList.add(0, new CurrencyTuple(Res.getBaseCurrencyCode(), Res.getBaseCurrencyName(), 8)); jsonFileManager.writeToDisc(Utilities.objectToJson(cryptoCurrencyList), "crypto_currency_list"); - - // We store the statistics as json so it is easy for further processing (e.g. for web based services) - // TODO This is just a quick solution for storing to one file. - // 1 statistic entry has 500 bytes as json. - // Need a more scalable solution later when we get more volume. - // The flag will only be activated by dedicated nodes, so it should not be too critical for the moment, but needs to - // get improved. Maybe a LevelDB like DB...? Could be impl. in a headless version only. - List list = observableTradeStatisticsSet.stream().map(TradeStatisticsForJson::new) - .sorted((o1, o2) -> (Long.compare(o2.tradeDate, o1.tradeDate))) - .collect(Collectors.toList()); - TradeStatisticsForJson[] array = new TradeStatisticsForJson[list.size()]; - list.toArray(array); - jsonFileManager.writeToDisc(Utilities.objectToJson(array), "trade_statistics"); - } - } - - static class WrapperTradeStatistics2 { - private final TradeStatistics2 tradeStatistics; - - public WrapperTradeStatistics2(TradeStatistics2 tradeStatistics) { - this.tradeStatistics = tradeStatistics; } - public TradeStatistics2 unwrap() { - return this.tradeStatistics; - } - - @Override - public boolean equals(Object obj) { - if (this == obj) return true; - if (obj == null || getClass() != obj.getClass()) return false; - var wrapper = (WrapperTradeStatistics2) obj; - return Objects.equals(tradeStatistics.getOfferId(), wrapper.tradeStatistics.getOfferId()); - } - - @Override - public int hashCode() { - return Objects.hash(tradeStatistics.getOfferId()); - } + List list = observableTradeStatisticsSet.stream() + .map(TradeStatisticsForJson::new) + .sorted((o1, o2) -> (Long.compare(o2.tradeDate, o1.tradeDate))) + .collect(Collectors.toList()); + TradeStatisticsForJson[] array = new TradeStatisticsForJson[list.size()]; + list.toArray(array); + jsonFileManager.writeToDisc(Utilities.objectToJson(array), "trade_statistics"); } + } From 52be1266673675b6dbc5a2fbce96876deb989fc0 Mon Sep 17 00:00:00 2001 From: chimp1984 Date: Mon, 5 Oct 2020 18:20:56 -0500 Subject: [PATCH 44/59] Apply TradeStatistics3 to client classes --- core/src/main/java/bisq/core/api/CoreApi.java | 4 +- .../dao/governance/asset/AssetService.java | 6 +- .../availability/DisputeAgentSelection.java | 17 ++-- .../grpc/GrpcGetTradeStatisticsService.java | 4 +- .../economy/dashboard/BsqDashboardView.java | 32 +++---- .../bisq/desktop/main/market/MarketView.java | 32 ++++--- .../main/market/trades/TradesChartsView.java | 96 +++++++------------ .../market/trades/TradesChartsViewModel.java | 33 +++---- .../main/offer/MutableOfferDataModel.java | 6 +- .../bisq/monitor/metric/P2PMarketStats.java | 6 +- .../monitor/metric/P2PSeedNodeSnapshot.java | 6 +- 11 files changed, 106 insertions(+), 136 deletions(-) diff --git a/core/src/main/java/bisq/core/api/CoreApi.java b/core/src/main/java/bisq/core/api/CoreApi.java index e944265a4b7..be694bc7130 100644 --- a/core/src/main/java/bisq/core/api/CoreApi.java +++ b/core/src/main/java/bisq/core/api/CoreApi.java @@ -22,7 +22,7 @@ import bisq.core.offer.Offer; import bisq.core.offer.OfferPayload; import bisq.core.payment.PaymentAccount; -import bisq.core.trade.statistics.TradeStatistics2; +import bisq.core.trade.statistics.TradeStatistics3; import bisq.core.trade.statistics.TradeStatisticsManager; import bisq.common.app.Version; @@ -196,7 +196,7 @@ public void removeWalletPassword(String password) { walletsService.removeWalletPassword(password); } - public List getTradeStatistics() { + public List getTradeStatistics() { return new ArrayList<>(tradeStatisticsManager.getObservableTradeStatisticsSet()); } diff --git a/core/src/main/java/bisq/core/dao/governance/asset/AssetService.java b/core/src/main/java/bisq/core/dao/governance/asset/AssetService.java index f66614ec855..cc902185057 100644 --- a/core/src/main/java/bisq/core/dao/governance/asset/AssetService.java +++ b/core/src/main/java/bisq/core/dao/governance/asset/AssetService.java @@ -154,10 +154,10 @@ public void updateAssetStates() { // TradeAmountDateTuple object holding only the data we need. Map> lookupMap = new HashMap<>(); tradeStatisticsManager.getObservableTradeStatisticsSet().stream() - .filter(e -> CurrencyUtil.isCryptoCurrency(e.getBaseCurrency())) + .filter(e -> CurrencyUtil.isCryptoCurrency(e.getCurrency())) .forEach(e -> { - lookupMap.putIfAbsent(e.getBaseCurrency(), new ArrayList<>()); - lookupMap.get(e.getBaseCurrency()).add(new TradeAmountDateTuple(e.getTradeAmount().getValue(), e.getTradeDate().getTime())); + lookupMap.putIfAbsent(e.getCurrency(), new ArrayList<>()); + lookupMap.get(e.getCurrency()).add(new TradeAmountDateTuple(e.getAmount(), e.getDate())); }); getStatefulAssets().stream() diff --git a/core/src/main/java/bisq/core/offer/availability/DisputeAgentSelection.java b/core/src/main/java/bisq/core/offer/availability/DisputeAgentSelection.java index 478038ad9ab..55996437f24 100644 --- a/core/src/main/java/bisq/core/offer/availability/DisputeAgentSelection.java +++ b/core/src/main/java/bisq/core/offer/availability/DisputeAgentSelection.java @@ -19,7 +19,7 @@ import bisq.core.support.dispute.agent.DisputeAgent; import bisq.core.support.dispute.agent.DisputeAgentManager; -import bisq.core.trade.statistics.TradeStatistics2; +import bisq.core.trade.statistics.TradeStatistics3; import bisq.core.trade.statistics.TradeStatisticsManager; import bisq.common.util.Tuple2; @@ -30,7 +30,6 @@ import java.util.Collections; import java.util.Comparator; import java.util.List; -import java.util.Objects; import java.util.Optional; import java.util.Set; import java.util.concurrent.atomic.AtomicInteger; @@ -46,22 +45,22 @@ public static T getLeastUsedMediator(TradeStatisticsMan DisputeAgentManager disputeAgentManager) { return getLeastUsedDisputeAgent(tradeStatisticsManager, disputeAgentManager, - TradeStatistics2.MEDIATOR_ADDRESS); + true); } public static T getLeastUsedRefundAgent(TradeStatisticsManager tradeStatisticsManager, DisputeAgentManager disputeAgentManager) { return getLeastUsedDisputeAgent(tradeStatisticsManager, disputeAgentManager, - TradeStatistics2.REFUND_AGENT_ADDRESS); + false); } private static T getLeastUsedDisputeAgent(TradeStatisticsManager tradeStatisticsManager, DisputeAgentManager disputeAgentManager, - String extraMapKey) { + boolean isMediator) { // We take last 100 entries from trade statistics - List list = new ArrayList<>(tradeStatisticsManager.getObservableTradeStatisticsSet()); - list.sort(Comparator.comparing(TradeStatistics2::getTradeDate)); + List list = new ArrayList<>(tradeStatisticsManager.getObservableTradeStatisticsSet()); + list.sort(Comparator.comparing(TradeStatistics3::getDate)); Collections.reverse(list); if (!list.isEmpty()) { int max = Math.min(list.size(), 100); @@ -70,9 +69,7 @@ private static T getLeastUsedDisputeAgent(TradeStatisti // We stored only first 4 chars of disputeAgents onion address List lastAddressesUsedInTrades = list.stream() - .filter(tradeStatistics2 -> tradeStatistics2.getExtraDataMap() != null) - .map(tradeStatistics2 -> tradeStatistics2.getExtraDataMap().get(extraMapKey)) - .filter(Objects::nonNull) + .map(tradeStatistics3 -> isMediator ? tradeStatistics3.getMediator() : tradeStatistics3.getRefundAgent()) .collect(Collectors.toList()); Set disputeAgents = disputeAgentManager.getObservableMap().values().stream() diff --git a/daemon/src/main/java/bisq/daemon/grpc/GrpcGetTradeStatisticsService.java b/daemon/src/main/java/bisq/daemon/grpc/GrpcGetTradeStatisticsService.java index 281fc121185..4c98e939af3 100644 --- a/daemon/src/main/java/bisq/daemon/grpc/GrpcGetTradeStatisticsService.java +++ b/daemon/src/main/java/bisq/daemon/grpc/GrpcGetTradeStatisticsService.java @@ -1,7 +1,7 @@ package bisq.daemon.grpc; import bisq.core.api.CoreApi; -import bisq.core.trade.statistics.TradeStatistics2; +import bisq.core.trade.statistics.TradeStatistics3; import bisq.proto.grpc.GetTradeStatisticsGrpc; import bisq.proto.grpc.GetTradeStatisticsReply; @@ -27,7 +27,7 @@ public void getTradeStatistics(GetTradeStatisticsRequest req, StreamObserver responseObserver) { var tradeStatistics = coreApi.getTradeStatistics().stream() - .map(TradeStatistics2::toProtoTradeStatistics2) + .map(TradeStatistics3::toProtoTradeStatistics3) .collect(Collectors.toList()); var reply = GetTradeStatisticsReply.newBuilder().addAllTradeStatistics(tradeStatistics).build(); diff --git a/desktop/src/main/java/bisq/desktop/main/dao/economy/dashboard/BsqDashboardView.java b/desktop/src/main/java/bisq/desktop/main/dao/economy/dashboard/BsqDashboardView.java index 24ef6601465..a96dfd58719 100644 --- a/desktop/src/main/java/bisq/desktop/main/dao/economy/dashboard/BsqDashboardView.java +++ b/desktop/src/main/java/bisq/desktop/main/dao/economy/dashboard/BsqDashboardView.java @@ -30,7 +30,7 @@ import bisq.core.monetary.Altcoin; import bisq.core.monetary.Price; import bisq.core.provider.price.PriceFeedService; -import bisq.core.trade.statistics.TradeStatistics2; +import bisq.core.trade.statistics.TradeStatistics3; import bisq.core.trade.statistics.TradeStatisticsManager; import bisq.core.user.Preferences; import bisq.core.util.FormattingUtils; @@ -289,17 +289,17 @@ private void updateChartData() { private void updateBsqPriceData() { seriesBSQPrice.getData().clear(); - Map> bsqPriceByDate = tradeStatisticsManager.getObservableTradeStatisticsSet().stream() - .filter(e -> e.getCurrencyCode().equals("BSQ")) - .sorted(Comparator.comparing(TradeStatistics2::getTradeDate)) - .collect(Collectors.groupingBy(item -> new java.sql.Date(item.getTradeDate().getTime()).toLocalDate() + Map> bsqPriceByDate = tradeStatisticsManager.getObservableTradeStatisticsSet().stream() + .filter(e -> e.getCurrency().equals("BSQ")) + .sorted(Comparator.comparing(TradeStatistics3::getDate)) + .collect(Collectors.groupingBy(item -> new java.sql.Date(item.getDate()).toLocalDate() .with(ADJUSTERS.get(DAY)))); List> updatedBSQPrice = bsqPriceByDate.keySet().stream() .map(e -> { ZonedDateTime zonedDateTime = e.atStartOfDay(ZoneId.systemDefault()); return new XYChart.Data(zonedDateTime.toInstant().getEpochSecond(), bsqPriceByDate.get(e).stream() - .map(TradeStatistics2::getTradePrice) + .map(TradeStatistics3::getTradePrice) .mapToDouble(Price::getValue) .average() .orElse(Double.NaN) @@ -370,12 +370,12 @@ private void updateAveragePriceFields(TextField field90, TextFieldWithIcon field private long updateAveragePriceField(TextField textField, int days, boolean isUSDField) { Date pastXDays = getPastDate(days); - List bsqTradePastXDays = tradeStatisticsManager.getObservableTradeStatisticsSet().stream() - .filter(e -> e.getCurrencyCode().equals("BSQ")) + List bsqTradePastXDays = tradeStatisticsManager.getObservableTradeStatisticsSet().stream() + .filter(e -> e.getCurrency().equals("BSQ")) .filter(e -> e.getTradeDate().after(pastXDays)) .collect(Collectors.toList()); - List usdTradePastXDays = tradeStatisticsManager.getObservableTradeStatisticsSet().stream() - .filter(e -> e.getCurrencyCode().equals("USD")) + List usdTradePastXDays = tradeStatisticsManager.getObservableTradeStatisticsSet().stream() + .filter(e -> e.getCurrency().equals("USD")) .filter(e -> e.getTradeDate().after(pastXDays)) .collect(Collectors.toList()); long average = isUSDField ? getUSDAverage(bsqTradePastXDays, usdTradePastXDays) : @@ -391,11 +391,11 @@ private long updateAveragePriceField(TextField textField, int days, boolean isUS return average; } - private long getBTCAverage(List bsqList) { + private long getBTCAverage(List bsqList) { long accumulatedVolume = 0; long accumulatedAmount = 0; - for (TradeStatistics2 item : bsqList) { + for (TradeStatistics3 item : bsqList) { accumulatedVolume += item.getTradeVolume().getValue(); accumulatedAmount += item.getTradeAmount().getValue(); // Amount of BTC traded } @@ -406,17 +406,17 @@ private long getBTCAverage(List bsqList) { return averagePrice; } - private long getUSDAverage(List bsqList, List usdList) { + private long getUSDAverage(List bsqList, List usdList) { // Use next USD/BTC print as price to calculate BSQ/USD rate // Store each trade as amount of USD and amount of BSQ traded List> usdBsqList = new ArrayList<>(bsqList.size()); - usdList.sort(Comparator.comparing(o -> o.getTradeDate().getTime())); + usdList.sort(Comparator.comparing(TradeStatistics3::getDate)); var usdBTCPrice = 10000d; // Default to 10000 USD per BTC if there is no USD feed at all - for (TradeStatistics2 item : bsqList) { + for (TradeStatistics3 item : bsqList) { // Find usdprice for trade item usdBTCPrice = usdList.stream() - .filter(usd -> usd.getTradeDate().getTime() > item.getTradeDate().getTime()) + .filter(usd -> usd.getDate() > item.getDate()) .map(usd -> MathUtils.scaleDownByPowerOf10((double) usd.getTradePrice().getValue(), Fiat.SMALLEST_UNIT_EXPONENT)) .findFirst() diff --git a/desktop/src/main/java/bisq/desktop/main/market/MarketView.java b/desktop/src/main/java/bisq/desktop/main/market/MarketView.java index 15641ac3dc1..9a3544e06d0 100644 --- a/desktop/src/main/java/bisq/desktop/main/market/MarketView.java +++ b/desktop/src/main/java/bisq/desktop/main/market/MarketView.java @@ -35,7 +35,7 @@ import bisq.core.locale.CurrencyUtil; import bisq.core.locale.Res; import bisq.core.offer.OfferPayload; -import bisq.core.trade.statistics.TradeStatistics2; +import bisq.core.trade.statistics.TradeStatistics3; import bisq.core.util.FormattingUtils; import bisq.core.util.coin.CoinFormatter; @@ -82,7 +82,10 @@ public class MarketView extends ActivatableView { @Inject - public MarketView(CachingViewLoader viewLoader, P2PService p2PService, OfferBook offerBook, @Named(FormattingUtils.BTC_FORMATTER_KEY) CoinFormatter formatter, + public MarketView(CachingViewLoader viewLoader, + P2PService p2PService, + OfferBook offerBook, + @Named(FormattingUtils.BTC_FORMATTER_KEY) CoinFormatter formatter, Navigation navigation) { this.viewLoader = viewLoader; this.p2PService = p2PService; @@ -179,20 +182,19 @@ private String getAllTradesWithReferralId() { // If both traders had set it the tradeStatistics is only delivered once. // If both traders used a different referral ID then we would get 2 objects. List list = p2PService.getP2PDataStorage().getAppendOnlyDataStoreMap().values().stream() - .filter(e -> e instanceof TradeStatistics2) - .map(e -> (TradeStatistics2) e) - .filter(tradeStatistics2 -> tradeStatistics2.getExtraDataMap() != null) - .filter(tradeStatistics2 -> tradeStatistics2.getExtraDataMap().get(OfferPayload.REFERRAL_ID) != null) - .map(trade -> { + .filter(e -> e instanceof TradeStatistics3) + .map(e -> (TradeStatistics3) e) + .filter(tradeStatistics3 -> tradeStatistics3.getExtraDataMap() != null) + .filter(tradeStatistics3 -> tradeStatistics3.getExtraDataMap().get(OfferPayload.REFERRAL_ID) != null) + .map(tradeStatistics3 -> { StringBuilder sb = new StringBuilder(); - sb.append("Trade ID: ").append(trade.getOfferId()).append("\n") - .append("Date: ").append(DisplayUtils.formatDateTime(trade.getTradeDate())).append("\n") - .append("Market: ").append(CurrencyUtil.getCurrencyPair(trade.getCurrencyCode())).append("\n") - .append("Price: ").append(FormattingUtils.formatPrice(trade.getTradePrice())).append("\n") - .append("Amount: ").append(formatter.formatCoin(trade.getTradeAmount())).append("\n") - .append("Volume: ").append(DisplayUtils.formatVolume(trade.getTradeVolume())).append("\n") - .append("Payment method: ").append(Res.get(trade.getOfferPaymentMethod())).append("\n") - .append("ReferralID: ").append(trade.getExtraDataMap().get(OfferPayload.REFERRAL_ID)); + sb.append("Date: ").append(DisplayUtils.formatDateTime(tradeStatistics3.getTradeDate())).append("\n") + .append("Market: ").append(CurrencyUtil.getCurrencyPair(tradeStatistics3.getCurrency())).append("\n") + .append("Price: ").append(FormattingUtils.formatPrice(tradeStatistics3.getTradePrice())).append("\n") + .append("Amount: ").append(formatter.formatCoin(tradeStatistics3.getTradeAmount())).append("\n") + .append("Volume: ").append(DisplayUtils.formatVolume(tradeStatistics3.getTradeVolume())).append("\n") + .append("Payment method: ").append(Res.get(tradeStatistics3.getPaymentMethod())).append("\n") + .append("ReferralID: ").append(tradeStatistics3.getExtraDataMap().get(OfferPayload.REFERRAL_ID)); return sb.toString(); }) .collect(Collectors.toList()); diff --git a/desktop/src/main/java/bisq/desktop/main/market/trades/TradesChartsView.java b/desktop/src/main/java/bisq/desktop/main/market/trades/TradesChartsView.java index f26d3c1ea05..4a70b96ff43 100644 --- a/desktop/src/main/java/bisq/desktop/main/market/trades/TradesChartsView.java +++ b/desktop/src/main/java/bisq/desktop/main/market/trades/TradesChartsView.java @@ -34,8 +34,7 @@ import bisq.core.locale.Res; import bisq.core.monetary.Price; import bisq.core.monetary.Volume; -import bisq.core.offer.OfferPayload; -import bisq.core.trade.statistics.TradeStatistics2; +import bisq.core.trade.statistics.TradeStatistics3; import bisq.core.util.FormattingUtils; import bisq.core.util.coin.CoinFormatter; @@ -103,7 +102,7 @@ public class TradesChartsView extends ActivatableViewAndModel tableView; + private TableView tableView; private AutocompleteComboBox currencyComboBox; private VolumeChart volumeChart; private CandleStickChart priceChart; @@ -118,12 +117,12 @@ public class TradesChartsView extends ActivatableViewAndModel timeUnitChangeListener; private ToggleGroup toggleGroup; private final ListChangeListener> itemsChangeListener; - private SortedList sortedList; + private SortedList sortedList; private Label nrOfTradeStatisticsLabel; - private ListChangeListener tradeStatisticsByCurrencyListener; + private ListChangeListener tradeStatisticsByCurrencyListener; private ChangeListener selectedTabIndexListener; private SingleSelectionModel tabPaneSelectionModel; - private TableColumn priceColumn, volumeColumn, marketColumn; + private TableColumn priceColumn, volumeColumn, marketColumn; @SuppressWarnings("FieldCanBeLocal") private MonadicBinding currencySelectionBinding; private Subscription currencySelectionSubscriber; @@ -550,7 +549,7 @@ private void createTable() { VBox.setVgrow(tableView, Priority.ALWAYS); // date - TableColumn dateColumn = new AutoTooltipTableColumn<>(Res.get("shared.dateTime")) { + TableColumn dateColumn = new AutoTooltipTableColumn<>(Res.get("shared.dateTime")) { { setMinWidth(240); setMaxWidth(240); @@ -561,11 +560,11 @@ private void createTable() { dateColumn.setCellFactory( new Callback<>() { @Override - public TableCell call( - TableColumn column) { + public TableCell call( + TableColumn column) { return new TableCell<>() { @Override - public void updateItem(final TradeStatistics2 item, boolean empty) { + public void updateItem(final TradeStatistics3 item, boolean empty) { super.updateItem(item, empty); if (item != null) setText(DisplayUtils.formatDateTime(item.getTradeDate())); @@ -575,7 +574,7 @@ public void updateItem(final TradeStatistics2 item, boolean empty) { }; } }); - dateColumn.setComparator(Comparator.comparing(TradeStatistics2::getTradeDate)); + dateColumn.setComparator(Comparator.comparing(TradeStatistics3::getTradeDate)); tableView.getColumns().add(dateColumn); // market @@ -590,21 +589,21 @@ public void updateItem(final TradeStatistics2 item, boolean empty) { marketColumn.setCellFactory( new Callback<>() { @Override - public TableCell call( - TableColumn column) { + public TableCell call( + TableColumn column) { return new TableCell<>() { @Override - public void updateItem(final TradeStatistics2 item, boolean empty) { + public void updateItem(final TradeStatistics3 item, boolean empty) { super.updateItem(item, empty); if (item != null) - setText(CurrencyUtil.getCurrencyPair(item.getCurrencyCode())); + setText(CurrencyUtil.getCurrencyPair(item.getCurrency())); else setText(""); } }; } }); - marketColumn.setComparator(Comparator.comparing(TradeStatistics2::getTradeDate)); + marketColumn.setComparator(Comparator.comparing(TradeStatistics3::getTradeDate)); tableView.getColumns().add(marketColumn); // price @@ -614,11 +613,11 @@ public void updateItem(final TradeStatistics2 item, boolean empty) { priceColumn.setCellFactory( new Callback<>() { @Override - public TableCell call( - TableColumn column) { + public TableCell call( + TableColumn column) { return new TableCell<>() { @Override - public void updateItem(final TradeStatistics2 item, boolean empty) { + public void updateItem(final TradeStatistics3 item, boolean empty) { super.updateItem(item, empty); if (item != null) setText(FormattingUtils.formatPrice(item.getTradePrice())); @@ -628,21 +627,21 @@ public void updateItem(final TradeStatistics2 item, boolean empty) { }; } }); - priceColumn.setComparator(Comparator.comparing(TradeStatistics2::getTradePrice)); + priceColumn.setComparator(Comparator.comparing(TradeStatistics3::getTradePrice)); tableView.getColumns().add(priceColumn); // amount - TableColumn amountColumn = new AutoTooltipTableColumn<>(Res.get("shared.amountWithCur", Res.getBaseCurrencyCode())); + TableColumn amountColumn = new AutoTooltipTableColumn<>(Res.get("shared.amountWithCur", Res.getBaseCurrencyCode())); amountColumn.getStyleClass().add("number-column"); amountColumn.setCellValueFactory((tradeStatistics) -> new ReadOnlyObjectWrapper<>(tradeStatistics.getValue())); amountColumn.setCellFactory( new Callback<>() { @Override - public TableCell call( - TableColumn column) { + public TableCell call( + TableColumn column) { return new TableCell<>() { @Override - public void updateItem(final TradeStatistics2 item, boolean empty) { + public void updateItem(final TradeStatistics3 item, boolean empty) { super.updateItem(item, empty); if (item != null) setGraphic(new ColoredDecimalPlacesWithZerosText(formatter.formatCoin(item.getTradeAmount(), @@ -653,7 +652,7 @@ public void updateItem(final TradeStatistics2 item, boolean empty) { }; } }); - amountColumn.setComparator(Comparator.comparing(TradeStatistics2::getTradeAmount)); + amountColumn.setComparator(Comparator.comparing(TradeStatistics3::getTradeAmount)); tableView.getColumns().add(amountColumn); // volume @@ -663,11 +662,11 @@ public void updateItem(final TradeStatistics2 item, boolean empty) { volumeColumn.setCellFactory( new Callback<>() { @Override - public TableCell call( - TableColumn column) { + public TableCell call( + TableColumn column) { return new TableCell<>() { @Override - public void updateItem(final TradeStatistics2 item, boolean empty) { + public void updateItem(final TradeStatistics3 item, boolean empty) { super.updateItem(item, empty); if (item != null) setText(model.showAllTradeCurrenciesProperty.get() ? @@ -687,17 +686,17 @@ public void updateItem(final TradeStatistics2 item, boolean empty) { tableView.getColumns().add(volumeColumn); // paymentMethod - TableColumn paymentMethodColumn = new AutoTooltipTableColumn<>(Res.get("shared.paymentMethod")); + TableColumn paymentMethodColumn = new AutoTooltipTableColumn<>(Res.get("shared.paymentMethod")); paymentMethodColumn.getStyleClass().add("number-column"); paymentMethodColumn.setCellValueFactory((tradeStatistics) -> new ReadOnlyObjectWrapper<>(tradeStatistics.getValue())); paymentMethodColumn.setCellFactory( new Callback<>() { @Override - public TableCell call( - TableColumn column) { + public TableCell call( + TableColumn column) { return new TableCell<>() { @Override - public void updateItem(final TradeStatistics2 item, boolean empty) { + public void updateItem(final TradeStatistics3 item, boolean empty) { super.updateItem(item, empty); if (item != null) setText(getPaymentMethodLabel(item)); @@ -710,30 +709,6 @@ public void updateItem(final TradeStatistics2 item, boolean empty) { paymentMethodColumn.setComparator(Comparator.comparing(this::getPaymentMethodLabel)); tableView.getColumns().add(paymentMethodColumn); - // direction - TableColumn directionColumn = new AutoTooltipTableColumn<>(Res.get("shared.offerType")); - directionColumn.getStyleClass().addAll("number-column", "last-column"); - directionColumn.setCellValueFactory((tradeStatistics) -> new ReadOnlyObjectWrapper<>(tradeStatistics.getValue())); - directionColumn.setCellFactory( - new Callback<>() { - @Override - public TableCell call( - TableColumn column) { - return new TableCell<>() { - @Override - public void updateItem(final TradeStatistics2 item, boolean empty) { - super.updateItem(item, empty); - if (item != null) - setText(getDirectionLabel(item)); - else - setText(""); - } - }; - } - }); - directionColumn.setComparator(Comparator.comparing(this::getDirectionLabel)); - tableView.getColumns().add(directionColumn); - tableView.setColumnResizePolicy(TableView.CONSTRAINED_RESIZE_POLICY); Label placeholder = new AutoTooltipLabel(Res.get("table.placeholder.noData")); placeholder.setWrapText(true); @@ -743,13 +718,8 @@ public void updateItem(final TradeStatistics2 item, boolean empty) { } @NotNull - private String getDirectionLabel(TradeStatistics2 item) { - return DisplayUtils.getDirectionWithCode(OfferPayload.Direction.valueOf(item.getDirection().name()), item.getCurrencyCode()); - } - - @NotNull - private String getPaymentMethodLabel(TradeStatistics2 item) { - return Res.get(item.getOfferPaymentMethod()); + private String getPaymentMethodLabel(TradeStatistics3 item) { + return Res.get(item.getPaymentMethod()); } private void layout() { diff --git a/desktop/src/main/java/bisq/desktop/main/market/trades/TradesChartsViewModel.java b/desktop/src/main/java/bisq/desktop/main/market/trades/TradesChartsViewModel.java index f070b5946b3..3bf125ad897 100644 --- a/desktop/src/main/java/bisq/desktop/main/market/trades/TradesChartsViewModel.java +++ b/desktop/src/main/java/bisq/desktop/main/market/trades/TradesChartsViewModel.java @@ -34,7 +34,7 @@ import bisq.core.locale.TradeCurrency; import bisq.core.monetary.Altcoin; import bisq.core.provider.price.PriceFeedService; -import bisq.core.trade.statistics.TradeStatistics2; +import bisq.core.trade.statistics.TradeStatistics3; import bisq.core.trade.statistics.TradeStatisticsManager; import bisq.core.user.Preferences; @@ -97,18 +97,18 @@ public enum TickUnit { private final TradeStatisticsManager tradeStatisticsManager; final Preferences preferences; - private PriceFeedService priceFeedService; - private Navigation navigation; + private final PriceFeedService priceFeedService; + private final Navigation navigation; - private final SetChangeListener setChangeListener; + private final SetChangeListener setChangeListener; final ObjectProperty selectedTradeCurrencyProperty = new SimpleObjectProperty<>(); final BooleanProperty showAllTradeCurrenciesProperty = new SimpleBooleanProperty(false); private final CurrencyList currencyListItems; private final CurrencyListItem showAllCurrencyListItem = new CurrencyListItem(new CryptoCurrency(GUIUtil.SHOW_ALL_FLAG, ""), -1); - final ObservableList tradeStatisticsByCurrency = FXCollections.observableArrayList(); + final ObservableList tradeStatisticsByCurrency = FXCollections.observableArrayList(); final ObservableList> priceItems = FXCollections.observableArrayList(); final ObservableList> volumeItems = FXCollections.observableArrayList(); - private Map>> itemsPerInterval; + private Map>> itemsPerInterval; TickUnit tickUnit; final int maxTicks = 90; @@ -119,7 +119,8 @@ public enum TickUnit { /////////////////////////////////////////////////////////////////////////////////////////// @Inject - TradesChartsViewModel(TradeStatisticsManager tradeStatisticsManager, Preferences preferences, PriceFeedService priceFeedService, Navigation navigation) { + TradesChartsViewModel(TradeStatisticsManager tradeStatisticsManager, Preferences preferences, + PriceFeedService priceFeedService, Navigation navigation) { this.tradeStatisticsManager = tradeStatisticsManager; this.preferences = preferences; this.priceFeedService = priceFeedService; @@ -145,7 +146,7 @@ public enum TickUnit { private void fillTradeCurrencies() { // Don't use a set as we need all entries List tradeCurrencyList = tradeStatisticsManager.getObservableTradeStatisticsSet().stream() - .flatMap(e -> CurrencyUtil.getTradeCurrency(e.getCurrencyCode()).stream()) + .flatMap(e -> CurrencyUtil.getTradeCurrency(e.getCurrency()).stream()) .collect(Collectors.toList()); currencyListItems.updateWithCurrencies(tradeCurrencyList, showAllCurrencyListItem); @@ -241,15 +242,15 @@ private void syncPriceFeedCurrency() { private void updateChartData() { tradeStatisticsByCurrency.setAll(tradeStatisticsManager.getObservableTradeStatisticsSet().stream() - .filter(e -> showAllTradeCurrenciesProperty.get() || e.getCurrencyCode().equals(getCurrencyCode())) + .filter(e -> showAllTradeCurrenciesProperty.get() || e.getCurrency().equals(getCurrencyCode())) .collect(Collectors.toList())); // Generate date range and create sets for all ticks itemsPerInterval = new HashMap<>(); Date time = new Date(); for (long i = maxTicks + 1; i >= 0; --i) { - Set set = new HashSet<>(); - Pair> pair = new Pair<>((Date) time.clone(), set); + Set set = new HashSet<>(); + Pair> pair = new Pair<>((Date) time.clone(), set); itemsPerInterval.put(i, pair); time.setTime(time.getTime() - 1); time = roundToTick(time, tickUnit); @@ -258,7 +259,7 @@ private void updateChartData() { // Get all entries for the defined time interval tradeStatisticsByCurrency.forEach(e -> { for (long i = maxTicks; i > 0; --i) { - Pair> p = itemsPerInterval.get(i); + Pair> p = itemsPerInterval.get(i); if (e.getTradeDate().after(p.getKey())) { p.getValue().add(e); break; @@ -283,7 +284,7 @@ private void updateChartData() { } @VisibleForTesting - CandleData getCandleData(long tick, Set set) { + CandleData getCandleData(long tick, Set set) { long open = 0; long close = 0; long high = 0; @@ -293,7 +294,7 @@ CandleData getCandleData(long tick, Set set) { long numTrades = set.size(); List tradePrices = new ArrayList<>(set.size()); - for (TradeStatistics2 item : set) { + for (TradeStatistics3 item : set) { long tradePriceAsLong = item.getTradePrice().getValue(); // Previously a check was done which inverted the low and high for cryptocurrencies. low = (low != 0) ? Math.min(low, tradePriceAsLong) : tradePriceAsLong; @@ -305,8 +306,8 @@ CandleData getCandleData(long tick, Set set) { } Collections.sort(tradePrices); - List list = new ArrayList<>(set); - list.sort(Comparator.comparingLong(o -> o.getTradeDate().getTime())); + List list = new ArrayList<>(set); + list.sort(Comparator.comparingLong(TradeStatistics3::getDate)); if (list.size() > 0) { open = list.get(0).getTradePrice().getValue(); close = list.get(list.size() - 1).getTradePrice().getValue(); diff --git a/desktop/src/main/java/bisq/desktop/main/offer/MutableOfferDataModel.java b/desktop/src/main/java/bisq/desktop/main/offer/MutableOfferDataModel.java index 44371697167..62869cd54e7 100644 --- a/desktop/src/main/java/bisq/desktop/main/offer/MutableOfferDataModel.java +++ b/desktop/src/main/java/bisq/desktop/main/offer/MutableOfferDataModel.java @@ -43,7 +43,7 @@ import bisq.core.provider.fee.FeeService; import bisq.core.provider.price.PriceFeedService; import bisq.core.trade.handlers.TransactionResultHandler; -import bisq.core.trade.statistics.TradeStatistics2; +import bisq.core.trade.statistics.TradeStatistics3; import bisq.core.trade.statistics.TradeStatisticsManager; import bisq.core.user.Preferences; import bisq.core.user.User; @@ -344,9 +344,9 @@ private void setSuggestedSecurityDeposit(PaymentAccount paymentAccount) { var blocksRange = Restrictions.getLockTime(paymentAccount.getPaymentMethod().isAsset()); var startDate = new Date(System.currentTimeMillis() - blocksRange * 10 * 60000); var sortedRangeData = tradeStatisticsManager.getObservableTradeStatisticsSet().stream() - .filter(e -> e.getCurrencyCode().equals(getTradeCurrency().getCode())) + .filter(e -> e.getCurrency().equals(getTradeCurrency().getCode())) .filter(e -> e.getTradeDate().compareTo(startDate) >= 0) - .sorted(Comparator.comparing(TradeStatistics2::getTradeDate)) + .sorted(Comparator.comparing(TradeStatistics3::getTradeDate)) .collect(Collectors.toList()); var movingAverage = new MathUtils.MovingAverage(10, 0.2); double[] extremes = {Double.MAX_VALUE, Double.MIN_VALUE}; diff --git a/monitor/src/main/java/bisq/monitor/metric/P2PMarketStats.java b/monitor/src/main/java/bisq/monitor/metric/P2PMarketStats.java index 189c739b3b0..0f7e8afb83c 100644 --- a/monitor/src/main/java/bisq/monitor/metric/P2PMarketStats.java +++ b/monitor/src/main/java/bisq/monitor/metric/P2PMarketStats.java @@ -23,7 +23,7 @@ import bisq.core.account.witness.AccountAgeWitnessStore; import bisq.core.offer.OfferPayload; import bisq.core.proto.persistable.CorePersistenceProtoResolver; -import bisq.core.trade.statistics.TradeStatistics2Store; +import bisq.core.trade.statistics.TradeStatistics3Store; import bisq.network.p2p.NodeAddress; import bisq.network.p2p.network.Connection; @@ -163,8 +163,8 @@ public void configure(Properties properties) { String networkPostfix = "_" + BaseCurrencyNetwork.values()[Version.getBaseCurrencyNetwork()].toString(); try { PersistenceManager persistenceManager = new PersistenceManager<>(dir, new CorePersistenceProtoResolver(null, null), null); - TradeStatistics2Store tradeStatistics2Store = (TradeStatistics2Store) persistenceManager.getPersisted(TradeStatistics2Store.class.getSimpleName() + networkPostfix); - hashes.addAll(tradeStatistics2Store.getMap().keySet().stream().map(byteArray -> byteArray.bytes).collect(Collectors.toList())); + TradeStatistics3Store tradeStatistics3Store = (TradeStatistics3Store) persistenceManager.getPersisted(TradeStatistics3Store.class.getSimpleName() + networkPostfix); + hashes.addAll(tradeStatistics3Store.getMap().keySet().stream().map(byteArray -> byteArray.bytes).collect(Collectors.toList())); AccountAgeWitnessStore accountAgeWitnessStore = (AccountAgeWitnessStore) persistenceManager.getPersisted(AccountAgeWitnessStore.class.getSimpleName() + networkPostfix); hashes.addAll(accountAgeWitnessStore.getMap().keySet().stream().map(byteArray -> byteArray.bytes).collect(Collectors.toList())); diff --git a/monitor/src/main/java/bisq/monitor/metric/P2PSeedNodeSnapshot.java b/monitor/src/main/java/bisq/monitor/metric/P2PSeedNodeSnapshot.java index e87d7468253..6786cb65368 100644 --- a/monitor/src/main/java/bisq/monitor/metric/P2PSeedNodeSnapshot.java +++ b/monitor/src/main/java/bisq/monitor/metric/P2PSeedNodeSnapshot.java @@ -27,7 +27,7 @@ import bisq.core.dao.monitoring.network.messages.GetProposalStateHashesRequest; import bisq.core.dao.monitoring.network.messages.GetStateHashesResponse; import bisq.core.proto.persistable.CorePersistenceProtoResolver; -import bisq.core.trade.statistics.TradeStatistics2Store; +import bisq.core.trade.statistics.TradeStatistics3Store; import bisq.network.p2p.NodeAddress; import bisq.network.p2p.network.Connection; @@ -137,8 +137,8 @@ public void configure(Properties properties) { String networkPostfix = "_" + BaseCurrencyNetwork.values()[Version.getBaseCurrencyNetwork()].toString(); try { PersistenceManager persistenceManager = new PersistenceManager<>(dir, new CorePersistenceProtoResolver(null, null), null); - TradeStatistics2Store tradeStatistics2Store = (TradeStatistics2Store) persistenceManager.getPersisted(TradeStatistics2Store.class.getSimpleName() + networkPostfix); - hashes.addAll(tradeStatistics2Store.getMap().keySet().stream().map(byteArray -> byteArray.bytes).collect(Collectors.toList())); + TradeStatistics3Store tradeStatistics3Store = (TradeStatistics3Store) persistenceManager.getPersisted(TradeStatistics3Store.class.getSimpleName() + networkPostfix); + hashes.addAll(tradeStatistics3Store.getMap().keySet().stream().map(byteArray -> byteArray.bytes).collect(Collectors.toList())); AccountAgeWitnessStore accountAgeWitnessStore = (AccountAgeWitnessStore) persistenceManager.getPersisted(AccountAgeWitnessStore.class.getSimpleName() + networkPostfix); hashes.addAll(accountAgeWitnessStore.getMap().keySet().stream().map(byteArray -> byteArray.bytes).collect(Collectors.toList())); From 0e70a99c42a002f6e38a77621e8e18168176196c Mon Sep 17 00:00:00 2001 From: chimp1984 Date: Mon, 5 Oct 2020 18:27:54 -0500 Subject: [PATCH 45/59] Adjust tests, remove tests which do not make sense anymore --- .../trade/statistics/TradeStatistics3.java | 20 +-- .../statistics/TradeStatistics2Maker.java | 91 -------------- .../statistics/TradeStatistics2Test.java | 45 ------- .../TradeStatisticsManagerTest.java | 114 ------------------ .../trades/TradesChartsViewModelTest.java | 62 ++++++++-- 5 files changed, 63 insertions(+), 269 deletions(-) delete mode 100644 core/src/test/java/bisq/core/trade/statistics/TradeStatistics2Maker.java delete mode 100644 core/src/test/java/bisq/core/trade/statistics/TradeStatistics2Test.java delete mode 100644 core/src/test/java/bisq/core/trade/statistics/TradeStatisticsManagerTest.java diff --git a/core/src/main/java/bisq/core/trade/statistics/TradeStatistics3.java b/core/src/main/java/bisq/core/trade/statistics/TradeStatistics3.java index 94d267a7a77..12a43003ad1 100644 --- a/core/src/main/java/bisq/core/trade/statistics/TradeStatistics3.java +++ b/core/src/main/java/bisq/core/trade/statistics/TradeStatistics3.java @@ -42,6 +42,7 @@ import org.bitcoinj.utils.ExchangeRate; import org.bitcoinj.utils.Fiat; +import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Charsets; import java.util.Date; @@ -173,15 +174,16 @@ public TradeStatistics3(String currency, // PROTO BUFFER /////////////////////////////////////////////////////////////////////////////////////////// - private TradeStatistics3(String currency, - long price, - long amount, - String paymentMethod, - long date, - @Nullable String mediator, - @Nullable String refundAgent, - @Nullable Map extraDataMap, - @Nullable byte[] hash) { + @VisibleForTesting + public TradeStatistics3(String currency, + long price, + long amount, + String paymentMethod, + long date, + @Nullable String mediator, + @Nullable String refundAgent, + @Nullable Map extraDataMap, + @Nullable byte[] hash) { this.currency = currency; this.price = price; this.amount = amount; diff --git a/core/src/test/java/bisq/core/trade/statistics/TradeStatistics2Maker.java b/core/src/test/java/bisq/core/trade/statistics/TradeStatistics2Maker.java deleted file mode 100644 index 3c49ce2d5a6..00000000000 --- a/core/src/test/java/bisq/core/trade/statistics/TradeStatistics2Maker.java +++ /dev/null @@ -1,91 +0,0 @@ -/* - * This file is part of Bisq. - * - * Bisq is free software: you can redistribute it and/or modify it - * under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or (at - * your option) any later version. - * - * Bisq is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public - * License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with Bisq. If not, see . - */ - -package bisq.core.trade.statistics; - -import bisq.core.monetary.Price; -import bisq.core.offer.OfferPayload; - -import org.bitcoinj.core.Coin; - -import java.util.Calendar; -import java.util.Collections; -import java.util.Date; - -import com.natpryce.makeiteasy.Instantiator; -import com.natpryce.makeiteasy.Maker; -import com.natpryce.makeiteasy.Property; - -import static com.natpryce.makeiteasy.MakeItEasy.a; - -public class TradeStatistics2Maker { - - public static final Property date = new Property<>(); - public static final Property depositTxId = new Property<>(); - public static final Property tradeAmount = new Property<>(); - - public static final Instantiator TradeStatistic2 = lookup -> { - Calendar calendar = Calendar.getInstance(); - calendar.set(2016, 3, 19); - - return new TradeStatistics2( - new OfferPayload("1234", - 0L, - null, - null, - OfferPayload.Direction.BUY, - 100000L, - 0.0, - false, - 100000L, - 100000L, - "BTC", - "USD", - null, - null, - "SEPA", - "", - null, - null, - null, - null, - null, - "", - 0L, - 0L, - 0L, - false, - 0L, - 0L, - 0L, - 0L, - false, - false, - 0L, - 0L, - false, - null, - null, - 0), - Price.valueOf("BTC", 100000L), - lookup.valueOf(tradeAmount, Coin.SATOSHI), - lookup.valueOf(date, new Date(calendar.getTimeInMillis())), - lookup.valueOf(depositTxId, "123456"), - Collections.emptyMap()); - }; - public static final Maker dayZeroTrade = a(TradeStatistic2); -} diff --git a/core/src/test/java/bisq/core/trade/statistics/TradeStatistics2Test.java b/core/src/test/java/bisq/core/trade/statistics/TradeStatistics2Test.java deleted file mode 100644 index 725d18d5c7c..00000000000 --- a/core/src/test/java/bisq/core/trade/statistics/TradeStatistics2Test.java +++ /dev/null @@ -1,45 +0,0 @@ -/* - * This file is part of Bisq. - * - * Bisq is free software: you can redistribute it and/or modify it - * under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or (at - * your option) any later version. - * - * Bisq is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public - * License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with Bisq. If not, see . - */ - -package bisq.core.trade.statistics; - -import org.junit.Test; - -import static bisq.core.trade.statistics.TradeStatistics2Maker.dayZeroTrade; -import static bisq.core.trade.statistics.TradeStatistics2Maker.depositTxId; -import static com.natpryce.makeiteasy.MakeItEasy.make; -import static com.natpryce.makeiteasy.MakeItEasy.withNull; -import static org.junit.Assert.assertTrue; - - -public class TradeStatistics2Test { - - @Test - public void isValid_WithDepositTxId() { - - TradeStatistics2 tradeStatistic = make(dayZeroTrade); - - assertTrue(tradeStatistic.isValid()); - } - - @Test - public void isValid_WithEmptyDepositTxId() { - TradeStatistics2 tradeStatistic = make(dayZeroTrade.but(withNull(depositTxId))); - - assertTrue(tradeStatistic.isValid()); - } -} diff --git a/core/src/test/java/bisq/core/trade/statistics/TradeStatisticsManagerTest.java b/core/src/test/java/bisq/core/trade/statistics/TradeStatisticsManagerTest.java deleted file mode 100644 index 44b84d13848..00000000000 --- a/core/src/test/java/bisq/core/trade/statistics/TradeStatisticsManagerTest.java +++ /dev/null @@ -1,114 +0,0 @@ -/* - * This file is part of Bisq. - * - * Bisq is free software: you can redistribute it and/or modify it - * under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or (at - * your option) any later version. - * - * Bisq is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public - * License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with Bisq. If not, see . - */ - -package bisq.core.trade.statistics; - -import bisq.core.provider.price.PriceFeedService; - -import bisq.network.p2p.P2PService; -import bisq.network.p2p.storage.P2PDataStorage; -import bisq.network.p2p.storage.persistence.AppendOnlyDataStoreListener; -import bisq.network.p2p.storage.persistence.AppendOnlyDataStoreService; - -import org.bitcoinj.core.Coin; - -import java.io.File; - -import org.mockito.ArgumentCaptor; - -import org.junit.Before; -import org.junit.Test; - -import static bisq.core.trade.statistics.TradeStatistics2Maker.dayZeroTrade; -import static bisq.core.trade.statistics.TradeStatistics2Maker.depositTxId; -import static bisq.core.trade.statistics.TradeStatistics2Maker.tradeAmount; -import static com.natpryce.makeiteasy.MakeItEasy.make; -import static com.natpryce.makeiteasy.MakeItEasy.with; -import static com.natpryce.makeiteasy.MakeItEasy.withNull; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; - -public class TradeStatisticsManagerTest { - - private TradeStatisticsManager manager; - private TradeStatistics2 tradeWithNullDepositTxId; - private ArgumentCaptor listenerArgumentCaptor; - - @Before - public void prepareMocksAndObjects() { - P2PService p2PService = mock(P2PService.class); - P2PDataStorage p2PDataStorage = mock(P2PDataStorage.class); - File storageDir = mock(File.class); - TradeStatistics2StorageService tradeStatistics2StorageService = mock(TradeStatistics2StorageService.class); - PriceFeedService priceFeedService = mock(PriceFeedService.class); - - AppendOnlyDataStoreService appendOnlyDataStoreService = mock(AppendOnlyDataStoreService.class); - when(p2PService.getP2PDataStorage()).thenReturn(p2PDataStorage); - - manager = new TradeStatisticsManager(p2PService, priceFeedService, - tradeStatistics2StorageService, appendOnlyDataStoreService, storageDir, false); - - tradeWithNullDepositTxId = make(dayZeroTrade.but(withNull(depositTxId))); - - manager.onAllServicesInitialized(); - listenerArgumentCaptor = ArgumentCaptor.forClass(AppendOnlyDataStoreListener.class); - verify(p2PDataStorage).addAppendOnlyDataStoreListener(listenerArgumentCaptor.capture()); - - } - - @Test - public void addToSet_ObjectWithNullDepositTxId() { - listenerArgumentCaptor.getValue().onAdded(tradeWithNullDepositTxId); - assertTrue(manager.getObservableTradeStatisticsSet().contains(tradeWithNullDepositTxId)); - } - - @Test - public void addToSet_RemoveExistingObjectIfObjectWithNullDepositTxIdIsAdded() { - TradeStatistics2 tradeWithDepositTxId = make(dayZeroTrade); - - listenerArgumentCaptor.getValue().onAdded(tradeWithDepositTxId); - listenerArgumentCaptor.getValue().onAdded(tradeWithNullDepositTxId); - - assertFalse(manager.getObservableTradeStatisticsSet().contains(tradeWithDepositTxId)); - assertTrue(manager.getObservableTradeStatisticsSet().contains(tradeWithNullDepositTxId)); - } - - @Test - public void addToSet_NotRemoveExistingObjectIfObjectsNotEqual() { - TradeStatistics2 tradeWithDepositTxId = make(dayZeroTrade.but(with(tradeAmount, Coin.FIFTY_COINS))); - - listenerArgumentCaptor.getValue().onAdded(tradeWithDepositTxId); - listenerArgumentCaptor.getValue().onAdded(tradeWithNullDepositTxId); - - assertTrue(manager.getObservableTradeStatisticsSet().contains(tradeWithDepositTxId)); - assertFalse(manager.getObservableTradeStatisticsSet().contains(tradeWithNullDepositTxId)); - } - - @Test - public void addToSet_IgnoreObjectIfObjectWithNullDepositTxIdAlreadyExists() { - TradeStatistics2 tradeWithDepositTxId = make(dayZeroTrade); - - listenerArgumentCaptor.getValue().onAdded(tradeWithNullDepositTxId); - listenerArgumentCaptor.getValue().onAdded(tradeWithDepositTxId); - - assertTrue(manager.getObservableTradeStatisticsSet().contains(tradeWithNullDepositTxId)); - assertFalse(manager.getObservableTradeStatisticsSet().contains(tradeWithDepositTxId)); - } -} diff --git a/desktop/src/test/java/bisq/desktop/main/market/trades/TradesChartsViewModelTest.java b/desktop/src/test/java/bisq/desktop/main/market/trades/TradesChartsViewModelTest.java index f3223f5572f..3e7dd9db5bf 100644 --- a/desktop/src/test/java/bisq/desktop/main/market/trades/TradesChartsViewModelTest.java +++ b/desktop/src/test/java/bisq/desktop/main/market/trades/TradesChartsViewModelTest.java @@ -23,8 +23,9 @@ import bisq.core.locale.FiatCurrency; import bisq.core.monetary.Price; import bisq.core.offer.OfferPayload; +import bisq.core.payment.payload.PaymentMethod; import bisq.core.provider.price.PriceFeedService; -import bisq.core.trade.statistics.TradeStatistics2; +import bisq.core.trade.statistics.TradeStatistics3; import bisq.core.trade.statistics.TradeStatisticsManager; import bisq.core.user.Preferences; @@ -106,6 +107,7 @@ public class TradesChartsViewModelTest { null, 1 ); + @Before public void setup() throws IOException { tradeStatisticsManager = mock(TradeStatisticsManager.class); @@ -134,13 +136,45 @@ public void testGetCandleData() { long volume = Fiat.parseFiat("EUR", "2200").value; boolean isBullish = true; - Set set = new HashSet<>(); + Set set = new HashSet<>(); final Date now = new Date(); - set.add(new TradeStatistics2(offer, Price.parse("EUR", "520"), Coin.parseCoin("1"), new Date(now.getTime()), null, null)); - set.add(new TradeStatistics2(offer, Price.parse("EUR", "500"), Coin.parseCoin("1"), new Date(now.getTime() + 100), null, null)); - set.add(new TradeStatistics2(offer, Price.parse("EUR", "600"), Coin.parseCoin("1"), new Date(now.getTime() + 200), null, null)); - set.add(new TradeStatistics2(offer, Price.parse("EUR", "580"), Coin.parseCoin("1"), new Date(now.getTime() + 300), null, null)); + set.add(new TradeStatistics3(offer.getCurrencyCode(), + Price.parse("EUR", "520").getValue(), + Coin.parseCoin("1").getValue(), + PaymentMethod.BLOCK_CHAINS_ID, + now.getTime(), + null, + null, + null, + null)); + set.add(new TradeStatistics3(offer.getCurrencyCode(), + Price.parse("EUR", "500").getValue(), + Coin.parseCoin("1").getValue(), + PaymentMethod.BLOCK_CHAINS_ID, + now.getTime() + 100, + null, + null, + null, + null)); + set.add(new TradeStatistics3(offer.getCurrencyCode(), + Price.parse("EUR", "600").getValue(), + Coin.parseCoin("1").getValue(), + PaymentMethod.BLOCK_CHAINS_ID, + now.getTime() + 200, + null, + null, + null, + null)); + set.add(new TradeStatistics3(offer.getCurrencyCode(), + Price.parse("EUR", "580").getValue(), + Coin.parseCoin("1").getValue(), + PaymentMethod.BLOCK_CHAINS_ID, + now.getTime() + 300, + null, + null, + null, + null)); CandleData candleData = model.getCandleData(model.roundToTick(now, TradesChartsViewModel.TickUnit.DAY).getTime(), set); assertEquals(open, candleData.open); @@ -194,11 +228,19 @@ long currentTimeMillis() { // Two trades 10 seconds apart, different YEAR, MONTH, WEEK, DAY, HOUR, MINUTE_10 trades.add(new Trade("2017-12-31T23:59:52", "1", "100", "EUR")); trades.add(new Trade("2018-01-01T00:00:02", "1", "110", "EUR")); - Set set = new HashSet<>(); + Set set = new HashSet<>(); trades.forEach(t -> - set.add(new TradeStatistics2(offer, Price.parse(t.cc, t.price), Coin.parseCoin(t.size), t.date, null, null)) + set.add(new TradeStatistics3(offer.getCurrencyCode(), + Price.parse(t.cc, t.price).getValue(), + Coin.parseCoin(t.size).getValue(), + PaymentMethod.BLOCK_CHAINS_ID, + t.date.getTime(), + null, + null, + null, + null)) ); - ObservableSet tradeStats = FXCollections.observableSet(set); + ObservableSet tradeStats = FXCollections.observableSet(set); // Run test for each tick type for (TradesChartsViewModel.TickUnit tick : TradesChartsViewModel.TickUnit.values()) { @@ -209,7 +251,7 @@ long currentTimeMillis() { // Trigger chart update model.setTickUnit(tick); - assertEquals(model.selectedTradeCurrencyProperty.get().getCode(), tradeStats.iterator().next().getCurrencyCode()); + assertEquals(model.selectedTradeCurrencyProperty.get().getCode(), tradeStats.iterator().next().getCurrency()); assertEquals(2, model.priceItems.size()); assertEquals(2, model.volumeItems.size()); } From 6766835af61b5cff514c53cee703ee2904ebfd4f Mon Sep 17 00:00:00 2001 From: chimp1984 Date: Mon, 5 Oct 2020 18:28:37 -0500 Subject: [PATCH 46/59] Use TradeStatistics3 in protobuf file --- proto/src/main/proto/grpc.proto | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/proto/src/main/proto/grpc.proto b/proto/src/main/proto/grpc.proto index 49f47e156ec..2d23a43b2d3 100644 --- a/proto/src/main/proto/grpc.proto +++ b/proto/src/main/proto/grpc.proto @@ -154,7 +154,7 @@ message GetTradeStatisticsRequest { } message GetTradeStatisticsReply { - repeated TradeStatistics2 TradeStatistics = 1; + repeated TradeStatistics3 TradeStatistics = 1; } /////////////////////////////////////////////////////////////////////////////////////////// From b14266d815ab79b3b15fa925e715d203a0daad7c Mon Sep 17 00:00:00 2001 From: chimp1984 Date: Mon, 5 Oct 2020 18:29:05 -0500 Subject: [PATCH 47/59] Remove resource file --- p2p/src/main/resources/TradeStatistics2Store_BTC_MAINNET | 3 --- 1 file changed, 3 deletions(-) delete mode 100644 p2p/src/main/resources/TradeStatistics2Store_BTC_MAINNET diff --git a/p2p/src/main/resources/TradeStatistics2Store_BTC_MAINNET b/p2p/src/main/resources/TradeStatistics2Store_BTC_MAINNET deleted file mode 100644 index b43cfb50f36..00000000000 --- a/p2p/src/main/resources/TradeStatistics2Store_BTC_MAINNET +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:c6b7362593537425e1228ee9a7a234811dc23ec1f66db31be4cc896124ff20fa -size 15989339 From 9016cb6c32e355abc100b30a36fc6cd217aea02c Mon Sep 17 00:00:00 2001 From: chimp1984 Date: Mon, 5 Oct 2020 19:55:15 -0500 Subject: [PATCH 48/59] Prune mediator and refund agent entries for all entries beside the last 100 we use for the selection algorithm. --- .../availability/DisputeAgentSelection.java | 3 +- .../trade/statistics/TradeStatistics3.java | 43 ++++++++++++++++--- .../statistics/TradeStatisticsConverter.java | 27 +++++++++--- 3 files changed, 60 insertions(+), 13 deletions(-) diff --git a/core/src/main/java/bisq/core/offer/availability/DisputeAgentSelection.java b/core/src/main/java/bisq/core/offer/availability/DisputeAgentSelection.java index 55996437f24..111260a30ea 100644 --- a/core/src/main/java/bisq/core/offer/availability/DisputeAgentSelection.java +++ b/core/src/main/java/bisq/core/offer/availability/DisputeAgentSelection.java @@ -41,6 +41,7 @@ @Slf4j public class DisputeAgentSelection { + public static final int LOOK_BACK_RANGE = 100; public static T getLeastUsedMediator(TradeStatisticsManager tradeStatisticsManager, DisputeAgentManager disputeAgentManager) { return getLeastUsedDisputeAgent(tradeStatisticsManager, @@ -63,7 +64,7 @@ private static T getLeastUsedDisputeAgent(TradeStatisti list.sort(Comparator.comparing(TradeStatistics3::getDate)); Collections.reverse(list); if (!list.isEmpty()) { - int max = Math.min(list.size(), 100); + int max = Math.min(list.size(), LOOK_BACK_RANGE); list = list.subList(0, max); } diff --git a/core/src/main/java/bisq/core/trade/statistics/TradeStatistics3.java b/core/src/main/java/bisq/core/trade/statistics/TradeStatistics3.java index 12a43003ad1..e585eff1fa1 100644 --- a/core/src/main/java/bisq/core/trade/statistics/TradeStatistics3.java +++ b/core/src/main/java/bisq/core/trade/statistics/TradeStatistics3.java @@ -45,11 +45,12 @@ import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Charsets; +import java.util.Arrays; import java.util.Date; import java.util.Map; import java.util.Optional; -import lombok.Value; +import lombok.Getter; import lombok.extern.slf4j.Slf4j; import javax.annotation.Nullable; @@ -61,7 +62,7 @@ * Data size is about 50 bytes in average */ @Slf4j -@Value +@Getter public final class TradeStatistics3 implements ProcessOncePersistableNetworkPayload, PersistableNetworkPayload, CapabilityRequiringPayload { @@ -114,10 +115,10 @@ private enum PaymentMethodMapper { // Old converted trade stat objects might not have it set @Nullable @JsonExclude - private final String mediator; // todo entries from old data could be pruned + private String mediator; @Nullable @JsonExclude - private final String refundAgent; + private String refundAgent; // todo should we add referrerId as well? get added to extra map atm but not used so far @@ -129,7 +130,7 @@ private enum PaymentMethodMapper { // field in a class would break that hash and therefore break the storage mechanism. @Nullable @JsonExclude - private Map extraDataMap; + private final Map extraDataMap; public TradeStatistics3(String currency, long price, @@ -266,6 +267,11 @@ public Capabilities getRequiredCapabilities() { return new Capabilities(Capability.TRADE_STATISTICS_3); } + public void pruneOptionalData() { + mediator = null; + refundAgent = null; + } + public String getPaymentMethod() { try { return PaymentMethodMapper.values()[Integer.parseInt(paymentMethod)].name(); @@ -305,6 +311,33 @@ public boolean isValid() { !currency.isEmpty(); } + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (!(o instanceof TradeStatistics3)) return false; + + TradeStatistics3 that = (TradeStatistics3) o; + + if (price != that.price) return false; + if (amount != that.amount) return false; + if (date != that.date) return false; + if (currency != null ? !currency.equals(that.currency) : that.currency != null) return false; + if (paymentMethod != null ? !paymentMethod.equals(that.paymentMethod) : that.paymentMethod != null) + return false; + return Arrays.equals(hash, that.hash); + } + + @Override + public int hashCode() { + int result = currency != null ? currency.hashCode() : 0; + result = 31 * result + (int) (price ^ (price >>> 32)); + result = 31 * result + (int) (amount ^ (amount >>> 32)); + result = 31 * result + (paymentMethod != null ? paymentMethod.hashCode() : 0); + result = 31 * result + (int) (date ^ (date >>> 32)); + result = 31 * result + Arrays.hashCode(hash); + return result; + } + @Override public String toString() { return "TradeStatistics3{" + diff --git a/core/src/main/java/bisq/core/trade/statistics/TradeStatisticsConverter.java b/core/src/main/java/bisq/core/trade/statistics/TradeStatisticsConverter.java index fc9e0bc4c5f..8cfad9aba54 100644 --- a/core/src/main/java/bisq/core/trade/statistics/TradeStatisticsConverter.java +++ b/core/src/main/java/bisq/core/trade/statistics/TradeStatisticsConverter.java @@ -17,6 +17,8 @@ package bisq.core.trade.statistics; +import bisq.core.offer.availability.DisputeAgentSelection; + import bisq.network.p2p.BootstrapListener; import bisq.network.p2p.P2PService; import bisq.network.p2p.storage.P2PDataStorage; @@ -29,18 +31,21 @@ import com.google.inject.Inject; import javax.inject.Named; +import javax.inject.Singleton; import java.io.File; import java.io.IOException; +import java.util.ArrayList; import java.util.Collection; +import java.util.Comparator; import java.util.HashMap; -import java.util.HashSet; +import java.util.List; import java.util.Map; -import java.util.Set; import lombok.extern.slf4j.Slf4j; +@Singleton @Slf4j public class TradeStatisticsConverter { @@ -91,8 +96,8 @@ public void onUpdatedDataReceived() { }); } - private static Set convertToTradeStatistics3(Collection persistableNetworkPayloads) { - Set result = new HashSet<>(); + private static List convertToTradeStatistics3(Collection persistableNetworkPayloads) { + List list = new ArrayList<>(); long ts = System.currentTimeMillis(); // We might have duplicate entries from both traders as the trade date was different from old clients. @@ -112,12 +117,20 @@ private static Set convertToTradeStatistics3(Collecti mapWithoutDuplicates.values().stream() .map(e -> convertToTradeStatistics3(e, false)) .filter(TradeStatistics3::isValid) - .forEach(result::add); + .forEach(list::add); log.info("Conversion to {} new trade statistic objects has been completed after {} ms", - result.size(), System.currentTimeMillis() - ts); + list.size(), System.currentTimeMillis() - ts); + + // We prune mediator and refundAgent data from all objects but the last 100 as we only use the + // last 100 entries (DisputeAgentSelection.LOOK_BACK_RANGE). + list.sort(Comparator.comparing(TradeStatistics3::getDate)); + for (int i = list.size() - DisputeAgentSelection.LOOK_BACK_RANGE; i < list.size(); i++) { + TradeStatistics3 tradeStatistics3 = list.get(i); + tradeStatistics3.pruneOptionalData(); + } - return result; + return list; } private static TradeStatistics3 convertToTradeStatistics3(TradeStatistics2 tradeStatistics2, boolean fromNetwork) { From f56fe42ae13993b472d87f8ac6145b6d2bab9d3d Mon Sep 17 00:00:00 2001 From: chimp1984 Date: Mon, 5 Oct 2020 19:56:11 -0500 Subject: [PATCH 49/59] Add injector.getInstance(TradeStatisticsConverter.class) to BisqExecutable to enforce inclusion. Cleanups, renamings --- core/src/main/java/bisq/core/app/BisqExecutable.java | 5 +++++ .../java/bisq/core/provider/price/PriceFeedService.java | 2 +- .../java/bisq/core/trade/statistics/TradeStatistics2.java | 2 +- .../bisq/core/trade/statistics/TradeStatisticsManager.java | 7 +++---- 4 files changed, 10 insertions(+), 6 deletions(-) diff --git a/core/src/main/java/bisq/core/app/BisqExecutable.java b/core/src/main/java/bisq/core/app/BisqExecutable.java index 8e403b97dc1..9ce6393c1a7 100644 --- a/core/src/main/java/bisq/core/app/BisqExecutable.java +++ b/core/src/main/java/bisq/core/app/BisqExecutable.java @@ -25,6 +25,7 @@ import bisq.core.setup.CorePersistedDataHost; import bisq.core.setup.CoreSetup; import bisq.core.support.dispute.arbitration.arbitrator.ArbitratorManager; +import bisq.core.trade.statistics.TradeStatisticsConverter; import bisq.core.trade.txproof.xmr.XmrTxProofService; import bisq.network.p2p.P2PService; @@ -154,6 +155,10 @@ protected Injector getInjector() { protected void applyInjector() { // Subclasses might configure classes with the injector here + + // As TradeStatisticsConverter is not used by any other class we need to enforce that guice is creating it by + // requesting an instance here. + injector.getInstance(TradeStatisticsConverter.class); } protected void readAllPersisted(Runnable completeHandler) { diff --git a/core/src/main/java/bisq/core/provider/price/PriceFeedService.java b/core/src/main/java/bisq/core/provider/price/PriceFeedService.java index 168812b4f43..b90b9128507 100644 --- a/core/src/main/java/bisq/core/provider/price/PriceFeedService.java +++ b/core/src/main/java/bisq/core/provider/price/PriceFeedService.java @@ -307,7 +307,7 @@ public void applyLatestBisqMarketPrice(Set tradeStatisticsSet) mapByCurrencyCode.values().stream() .filter(list -> !list.isEmpty()) .forEach(list -> { - list.sort(Comparator.comparing(o -> o.getTradeDate())); + list.sort(Comparator.comparing(TradeStatistics3::getTradeDate)); TradeStatistics3 tradeStatistics = list.get(list.size() - 1); setBisqMarketPrice(tradeStatistics.getCurrency(), tradeStatistics.getTradePrice()); }); diff --git a/core/src/main/java/bisq/core/trade/statistics/TradeStatistics2.java b/core/src/main/java/bisq/core/trade/statistics/TradeStatistics2.java index 17a878c270c..7beee9864dd 100644 --- a/core/src/main/java/bisq/core/trade/statistics/TradeStatistics2.java +++ b/core/src/main/java/bisq/core/trade/statistics/TradeStatistics2.java @@ -61,7 +61,7 @@ /** * Serialized size is about 180-210 byte. Nov 2017 we have 5500 objects */ - +@Deprecated @Slf4j @Value public final class TradeStatistics2 implements ProcessOncePersistableNetworkPayload, PersistableNetworkPayload, diff --git a/core/src/main/java/bisq/core/trade/statistics/TradeStatisticsManager.java b/core/src/main/java/bisq/core/trade/statistics/TradeStatisticsManager.java index 7e571391c84..9f5c967b13e 100644 --- a/core/src/main/java/bisq/core/trade/statistics/TradeStatisticsManager.java +++ b/core/src/main/java/bisq/core/trade/statistics/TradeStatisticsManager.java @@ -83,7 +83,7 @@ public void onAllServicesInitialized() { } observableTradeStatisticsSet.add(tradeStatistics); priceFeedService.applyLatestBisqMarketPrice(observableTradeStatisticsSet); - maybeDump(); + maybeDumpStatistics(); } }); @@ -94,14 +94,14 @@ public void onAllServicesInitialized() { .collect(Collectors.toSet()); observableTradeStatisticsSet.addAll(set); priceFeedService.applyLatestBisqMarketPrice(observableTradeStatisticsSet); - maybeDump(); + maybeDumpStatistics(); } public ObservableSet getObservableTradeStatisticsSet() { return observableTradeStatisticsSet; } - private void maybeDump() { + private void maybeDumpStatistics() { if (!dumpStatistics) { return; } @@ -130,5 +130,4 @@ private void maybeDump() { list.toArray(array); jsonFileManager.writeToDisc(Utilities.objectToJson(array), "trade_statistics"); } - } From fa374b99fb09d5d238c5adec330c9f08861fe693 Mon Sep 17 00:00:00 2001 From: chimp1984 Date: Mon, 5 Oct 2020 20:17:07 -0500 Subject: [PATCH 50/59] Do conversion in a thread to not block UI thread. takes about 4 seconds on my machine. Add shutdown method to TradeStatisticsConverter and call it via TradeStatisticsManager. Now as we have a reference to TradeStatisticsConverter in we don't need the hack anymore in applyInjector. Set log level for com.neemre.btcdcli4j.core.client.ClientConfigurator to error as there is an expected warn log because of the outdated version. --- .../bisq/common/file/JsonFileManager.java | 4 +- .../java/bisq/core/app/BisqExecutable.java | 7 +-- .../statistics/TradeStatisticsConverter.java | 45 +++++++++++++------ .../statistics/TradeStatisticsManager.java | 10 +++++ desktop/src/main/resources/logback.xml | 3 +- 5 files changed, 48 insertions(+), 21 deletions(-) diff --git a/common/src/main/java/bisq/common/file/JsonFileManager.java b/common/src/main/java/bisq/common/file/JsonFileManager.java index 9cbc91f7b29..1d1db8a9d88 100644 --- a/common/src/main/java/bisq/common/file/JsonFileManager.java +++ b/common/src/main/java/bisq/common/file/JsonFileManager.java @@ -30,7 +30,7 @@ @Slf4j public class JsonFileManager { - private final ThreadPoolExecutor executor = Utilities.getThreadPoolExecutor("JsonFileManagerExecutor", 5, 50, 60); + private final ThreadPoolExecutor executor; private final File dir; @@ -41,6 +41,8 @@ public class JsonFileManager { public JsonFileManager(File dir) { this.dir = dir; + this.executor = Utilities.getThreadPoolExecutor("JsonFileManagerExecutor", 5, 50, 60); + if (!dir.exists()) if (!dir.mkdir()) log.warn("make dir failed"); diff --git a/core/src/main/java/bisq/core/app/BisqExecutable.java b/core/src/main/java/bisq/core/app/BisqExecutable.java index 9ce6393c1a7..66c03dede43 100644 --- a/core/src/main/java/bisq/core/app/BisqExecutable.java +++ b/core/src/main/java/bisq/core/app/BisqExecutable.java @@ -25,7 +25,7 @@ import bisq.core.setup.CorePersistedDataHost; import bisq.core.setup.CoreSetup; import bisq.core.support.dispute.arbitration.arbitrator.ArbitratorManager; -import bisq.core.trade.statistics.TradeStatisticsConverter; +import bisq.core.trade.statistics.TradeStatisticsManager; import bisq.core.trade.txproof.xmr.XmrTxProofService; import bisq.network.p2p.P2PService; @@ -155,10 +155,6 @@ protected Injector getInjector() { protected void applyInjector() { // Subclasses might configure classes with the injector here - - // As TradeStatisticsConverter is not used by any other class we need to enforce that guice is creating it by - // requesting an instance here. - injector.getInstance(TradeStatisticsConverter.class); } protected void readAllPersisted(Runnable completeHandler) { @@ -225,6 +221,7 @@ public void gracefulShutDown(ResultHandler resultHandler) { try { injector.getInstance(ArbitratorManager.class).shutDown(); + injector.getInstance(TradeStatisticsManager.class).shutDown(); injector.getInstance(XmrTxProofService.class).shutDown(); injector.getInstance(DaoSetup.class).shutDown(); injector.getInstance(AvoidStandbyModeService.class).shutDown(); diff --git a/core/src/main/java/bisq/core/trade/statistics/TradeStatisticsConverter.java b/core/src/main/java/bisq/core/trade/statistics/TradeStatisticsConverter.java index 8cfad9aba54..5e85784913b 100644 --- a/core/src/main/java/bisq/core/trade/statistics/TradeStatisticsConverter.java +++ b/core/src/main/java/bisq/core/trade/statistics/TradeStatisticsConverter.java @@ -25,8 +25,10 @@ import bisq.network.p2p.storage.payload.PersistableNetworkPayload; import bisq.network.p2p.storage.persistence.AppendOnlyDataStoreService; +import bisq.common.UserThread; import bisq.common.config.Config; import bisq.common.file.FileUtil; +import bisq.common.util.Utilities; import com.google.inject.Inject; @@ -42,6 +44,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.concurrent.ExecutorService; import lombok.extern.slf4j.Slf4j; @@ -49,6 +52,8 @@ @Slf4j public class TradeStatisticsConverter { + private ExecutorService executor; + @Inject public TradeStatisticsConverter(P2PService p2PService, P2PDataStorage p2PDataStorage, @@ -60,24 +65,33 @@ public TradeStatisticsConverter(P2PService p2PService, appendOnlyDataStoreService.addService(tradeStatistics2StorageService); p2PService.addP2PServiceListener(new BootstrapListener() { + @Override public void onTorNodeReady() { if (!tradeStatistics2Store.exists()) { return; } - - // We convert early once tor is initialized but still not ready to receive data - var mapOfLiveData = tradeStatistics3StorageService.getMapOfLiveData(); - convertToTradeStatistics3(tradeStatistics2StorageService.getMapOfAllData().values()) - .forEach(e -> mapOfLiveData.put(new P2PDataStorage.ByteArray(e.getHash()), e)); - tradeStatistics3StorageService.persistNow(); - try { - log.info("We delete now the old trade statistics file as it was converted to the new format."); - FileUtil.deleteFileIfExists(tradeStatistics2Store); - } catch (IOException e) { - e.printStackTrace(); - log.error(e.toString()); - } + executor = Utilities.getSingleThreadExecutor("TradeStatisticsConverter"); + executor.submit(() -> { + // We convert early once tor is initialized but still not ready to receive data + Map tempMap = new HashMap<>(); + convertToTradeStatistics3(tradeStatistics2StorageService.getMapOfAllData().values()) + .forEach(e -> tempMap.put(new P2PDataStorage.ByteArray(e.getHash()), e)); + + // We map to user thread to avoid potential threading issues + UserThread.execute(() -> { + tradeStatistics3StorageService.getMapOfLiveData().putAll(tempMap); + tradeStatistics3StorageService.persistNow(); + }); + + try { + log.info("We delete now the old trade statistics file as it was converted to the new format."); + FileUtil.deleteFileIfExists(tradeStatistics2Store); + } catch (IOException e) { + e.printStackTrace(); + log.error(e.toString()); + } + }); } @Override @@ -96,6 +110,11 @@ public void onUpdatedDataReceived() { }); } + public void shutDown() { + if (executor != null) + executor.shutdown(); + } + private static List convertToTradeStatistics3(Collection persistableNetworkPayloads) { List list = new ArrayList<>(); long ts = System.currentTimeMillis(); diff --git a/core/src/main/java/bisq/core/trade/statistics/TradeStatisticsManager.java b/core/src/main/java/bisq/core/trade/statistics/TradeStatisticsManager.java index 9f5c967b13e..dedc250f85a 100644 --- a/core/src/main/java/bisq/core/trade/statistics/TradeStatisticsManager.java +++ b/core/src/main/java/bisq/core/trade/statistics/TradeStatisticsManager.java @@ -52,6 +52,7 @@ public class TradeStatisticsManager { private final P2PService p2PService; private final PriceFeedService priceFeedService; private final TradeStatistics3StorageService tradeStatistics3StorageService; + private final TradeStatisticsConverter tradeStatisticsConverter; private final File storageDir; private final boolean dumpStatistics; private final ObservableSet observableTradeStatisticsSet = FXCollections.observableSet(); @@ -62,11 +63,13 @@ public TradeStatisticsManager(P2PService p2PService, PriceFeedService priceFeedService, TradeStatistics3StorageService tradeStatistics3StorageService, AppendOnlyDataStoreService appendOnlyDataStoreService, + TradeStatisticsConverter tradeStatisticsConverter, @Named(Config.STORAGE_DIR) File storageDir, @Named(Config.DUMP_STATISTICS) boolean dumpStatistics) { this.p2PService = p2PService; this.priceFeedService = priceFeedService; this.tradeStatistics3StorageService = tradeStatistics3StorageService; + this.tradeStatisticsConverter = tradeStatisticsConverter; this.storageDir = storageDir; this.dumpStatistics = dumpStatistics; @@ -74,6 +77,13 @@ public TradeStatisticsManager(P2PService p2PService, appendOnlyDataStoreService.addService(tradeStatistics3StorageService); } + public void shutDown() { + tradeStatisticsConverter.shutDown(); + if (jsonFileManager != null) { + jsonFileManager.shutDown(); + } + } + public void onAllServicesInitialized() { p2PService.getP2PDataStorage().addAppendOnlyDataStoreListener(payload -> { if (payload instanceof TradeStatistics3) { diff --git a/desktop/src/main/resources/logback.xml b/desktop/src/main/resources/logback.xml index 8812d310de9..6b05588a4fe 100644 --- a/desktop/src/main/resources/logback.xml +++ b/desktop/src/main/resources/logback.xml @@ -11,7 +11,6 @@ - + From 213050c3d14a658b3035fafce647c328dc658037 Mon Sep 17 00:00:00 2001 From: chimp1984 Date: Mon, 5 Oct 2020 23:09:16 -0500 Subject: [PATCH 51/59] Add filter for excluding null objects --- .../bisq/core/offer/availability/DisputeAgentSelection.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/core/src/main/java/bisq/core/offer/availability/DisputeAgentSelection.java b/core/src/main/java/bisq/core/offer/availability/DisputeAgentSelection.java index 111260a30ea..8745bd3c565 100644 --- a/core/src/main/java/bisq/core/offer/availability/DisputeAgentSelection.java +++ b/core/src/main/java/bisq/core/offer/availability/DisputeAgentSelection.java @@ -30,6 +30,7 @@ import java.util.Collections; import java.util.Comparator; import java.util.List; +import java.util.Objects; import java.util.Optional; import java.util.Set; import java.util.concurrent.atomic.AtomicInteger; @@ -42,6 +43,7 @@ @Slf4j public class DisputeAgentSelection { public static final int LOOK_BACK_RANGE = 100; + public static T getLeastUsedMediator(TradeStatisticsManager tradeStatisticsManager, DisputeAgentManager disputeAgentManager) { return getLeastUsedDisputeAgent(tradeStatisticsManager, @@ -71,6 +73,7 @@ private static T getLeastUsedDisputeAgent(TradeStatisti // We stored only first 4 chars of disputeAgents onion address List lastAddressesUsedInTrades = list.stream() .map(tradeStatistics3 -> isMediator ? tradeStatistics3.getMediator() : tradeStatistics3.getRefundAgent()) + .filter(Objects::nonNull) .collect(Collectors.toList()); Set disputeAgents = disputeAgentManager.getObservableMap().values().stream() From 17f4ae2b541e4700562f3a13faf0d7ff5309911d Mon Sep 17 00:00:00 2001 From: chimp1984 Date: Mon, 5 Oct 2020 23:53:04 -0500 Subject: [PATCH 52/59] Add check that size is > LOOK_BACK_RANGE --- .../statistics/TradeStatisticsConverter.java | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/core/src/main/java/bisq/core/trade/statistics/TradeStatisticsConverter.java b/core/src/main/java/bisq/core/trade/statistics/TradeStatisticsConverter.java index 5e85784913b..137c7d9ee85 100644 --- a/core/src/main/java/bisq/core/trade/statistics/TradeStatisticsConverter.java +++ b/core/src/main/java/bisq/core/trade/statistics/TradeStatisticsConverter.java @@ -130,23 +130,26 @@ private static List convertToTradeStatistics3(Collection mapWithoutDuplicates.putIfAbsent(new P2PDataStorage.ByteArray(e.getHash()), e)); - log.info("We convert the existing {} trade statistics objects to the new format. " + - "This might take a bit but is only done once.", mapWithoutDuplicates.size()); + log.info("We convert the existing {} trade statistics objects to the new format.", mapWithoutDuplicates.size()); mapWithoutDuplicates.values().stream() .map(e -> convertToTradeStatistics3(e, false)) .filter(TradeStatistics3::isValid) .forEach(list::add); + int size = list.size(); log.info("Conversion to {} new trade statistic objects has been completed after {} ms", - list.size(), System.currentTimeMillis() - ts); + size, System.currentTimeMillis() - ts); // We prune mediator and refundAgent data from all objects but the last 100 as we only use the // last 100 entries (DisputeAgentSelection.LOOK_BACK_RANGE). list.sort(Comparator.comparing(TradeStatistics3::getDate)); - for (int i = list.size() - DisputeAgentSelection.LOOK_BACK_RANGE; i < list.size(); i++) { - TradeStatistics3 tradeStatistics3 = list.get(i); - tradeStatistics3.pruneOptionalData(); + if (size > DisputeAgentSelection.LOOK_BACK_RANGE) { + int start = size - DisputeAgentSelection.LOOK_BACK_RANGE; + for (int i = start; i < size; i++) { + TradeStatistics3 tradeStatistics3 = list.get(i); + tradeStatistics3.pruneOptionalData(); + } } return list; From e95ab2a0b41a6a6b9d986b7118d99fb384f506a1 Mon Sep 17 00:00:00 2001 From: chimp1984 Date: Tue, 6 Oct 2020 00:12:12 -0500 Subject: [PATCH 53/59] Add resource file for 1.4.0 (should be updated at release time) --- p2p/src/main/resources/TradeStatistics3Store_1.4.0_BTC_MAINNET | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 p2p/src/main/resources/TradeStatistics3Store_1.4.0_BTC_MAINNET diff --git a/p2p/src/main/resources/TradeStatistics3Store_1.4.0_BTC_MAINNET b/p2p/src/main/resources/TradeStatistics3Store_1.4.0_BTC_MAINNET new file mode 100644 index 00000000000..97de382b59c --- /dev/null +++ b/p2p/src/main/resources/TradeStatistics3Store_1.4.0_BTC_MAINNET @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0d58ed2b6b59006a13c31d32838959c3bf910430c6b88e357c967087e2f33a5c +size 3900357 From 58d2f1bda92521408757dc55e11662369f14105b Mon Sep 17 00:00:00 2001 From: chimp1984 Date: Thu, 8 Oct 2020 10:56:23 -0500 Subject: [PATCH 54/59] Apply codacy suggestions @ripcurl: The complaint about private constructors (using guice this is legit) should be removed IMO. --- .../main/dao/economy/dashboard/BsqDashboardView.java | 4 ++-- .../main/market/trades/TradesChartsViewModelTest.java | 3 --- .../main/java/bisq/monitor/metric/P2PSeedNodeSnapshot.java | 6 +++--- 3 files changed, 5 insertions(+), 8 deletions(-) diff --git a/desktop/src/main/java/bisq/desktop/main/dao/economy/dashboard/BsqDashboardView.java b/desktop/src/main/java/bisq/desktop/main/dao/economy/dashboard/BsqDashboardView.java index a96dfd58719..ed0257e0449 100644 --- a/desktop/src/main/java/bisq/desktop/main/dao/economy/dashboard/BsqDashboardView.java +++ b/desktop/src/main/java/bisq/desktop/main/dao/economy/dashboard/BsqDashboardView.java @@ -121,7 +121,7 @@ public class BsqDashboardView extends ActivatableView implements /////////////////////////////////////////////////////////////////////////////////////////// @Inject - private BsqDashboardView(DaoFacade daoFacade, + public BsqDashboardView(DaoFacade daoFacade, TradeStatisticsManager tradeStatisticsManager, PriceFeedService priceFeedService, Preferences preferences, @@ -173,7 +173,7 @@ private void createKPIs() { marketCapTextField = addTopLabelReadOnlyTextField(root, ++gridRow, Res.get("dao.factsAndFigures.dashboard.marketCap")).second; - availableAmountTextField = FormBuilder.addTopLabelReadOnlyTextField(root, gridRow, 1, + availableAmountTextField = addTopLabelReadOnlyTextField(root, gridRow, 1, Res.get("dao.factsAndFigures.dashboard.availableAmount")).second; } diff --git a/desktop/src/test/java/bisq/desktop/main/market/trades/TradesChartsViewModelTest.java b/desktop/src/test/java/bisq/desktop/main/market/trades/TradesChartsViewModelTest.java index 3e7dd9db5bf..8544363176c 100644 --- a/desktop/src/test/java/bisq/desktop/main/market/trades/TradesChartsViewModelTest.java +++ b/desktop/src/test/java/bisq/desktop/main/market/trades/TradesChartsViewModelTest.java @@ -64,9 +64,7 @@ public class TradesChartsViewModelTest { TradesChartsViewModel model; TradeStatisticsManager tradeStatisticsManager; - private static final Logger log = LoggerFactory.getLogger(TradesChartsViewModelTest.class); DateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss"); - private KeyRing keyRing; private File dir; OfferPayload offer = new OfferPayload(null, 0, @@ -217,7 +215,6 @@ class Trade { ArrayList trades = new ArrayList<>(); // Set predetermined time to use as "now" during test - Date test_time = dateFormat.parse("2018-01-01T00:00:05"); // Monday /* new MockUp() { @Mock long currentTimeMillis() { diff --git a/monitor/src/main/java/bisq/monitor/metric/P2PSeedNodeSnapshot.java b/monitor/src/main/java/bisq/monitor/metric/P2PSeedNodeSnapshot.java index 6786cb65368..060c97498fe 100644 --- a/monitor/src/main/java/bisq/monitor/metric/P2PSeedNodeSnapshot.java +++ b/monitor/src/main/java/bisq/monitor/metric/P2PSeedNodeSnapshot.java @@ -233,9 +233,9 @@ void report() { int oldest = (int) nodeAddressTupleMap.values().stream().min(Comparator.comparingLong(Tuple::getHeight)).get().height; // - update queried height - if(type.contains("DaoState")) + if (type.contains("DaoState")) daostateheight = oldest - 20; - else if(type.contains("Proposal")) + else if (type.contains("Proposal")) proposalheight = oldest - 20; else blindvoteheight = oldest - 20; @@ -255,7 +255,7 @@ else if(type.contains("Proposal")) List states = hitcount.entrySet().stream().sorted((o1, o2) -> o2.getValue().compareTo(o1.getValue())).map(byteBufferIntegerEntry -> byteBufferIntegerEntry.getKey()).collect(Collectors.toList()); hitcount.clear(); - + TradesChartsViewModelTest.java nodeAddressTupleMap.forEach((nodeAddress, tuple) -> daoreport.put(type + "." + OnionParser.prettyPrint(nodeAddress) + ".hash", Integer.toString(Arrays.asList(states.toArray()).indexOf(ByteBuffer.wrap(tuple.hash))))); // - report reference head From 197d8c1e0d78acc638571ea801516b445c4e28bd Mon Sep 17 00:00:00 2001 From: chimp1984 Date: Thu, 8 Oct 2020 12:19:17 -0500 Subject: [PATCH 55/59] Remove copy&past mistake --- .../src/main/java/bisq/monitor/metric/P2PSeedNodeSnapshot.java | 1 - 1 file changed, 1 deletion(-) diff --git a/monitor/src/main/java/bisq/monitor/metric/P2PSeedNodeSnapshot.java b/monitor/src/main/java/bisq/monitor/metric/P2PSeedNodeSnapshot.java index 060c97498fe..2ca12716f04 100644 --- a/monitor/src/main/java/bisq/monitor/metric/P2PSeedNodeSnapshot.java +++ b/monitor/src/main/java/bisq/monitor/metric/P2PSeedNodeSnapshot.java @@ -255,7 +255,6 @@ else if (type.contains("Proposal")) List states = hitcount.entrySet().stream().sorted((o1, o2) -> o2.getValue().compareTo(o1.getValue())).map(byteBufferIntegerEntry -> byteBufferIntegerEntry.getKey()).collect(Collectors.toList()); hitcount.clear(); - TradesChartsViewModelTest.java nodeAddressTupleMap.forEach((nodeAddress, tuple) -> daoreport.put(type + "." + OnionParser.prettyPrint(nodeAddress) + ".hash", Integer.toString(Arrays.asList(states.toArray()).indexOf(ByteBuffer.wrap(tuple.hash))))); // - report reference head From 18a27e90676c91e3b107355e1d85378a56981069 Mon Sep 17 00:00:00 2001 From: chimp1984 Date: Thu, 8 Oct 2020 13:00:31 -0500 Subject: [PATCH 56/59] Republish trade statistics from seller side if peer capability is know. This is not the case without getting PR #4609 merges as well. We only do it for 2 weeks after planned release time as then it can be assumed that enough nodes have updated that the normal publishing will distribute the object sufficiently. --- .../core/trade/protocol/SellerProtocol.java | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/core/src/main/java/bisq/core/trade/protocol/SellerProtocol.java b/core/src/main/java/bisq/core/trade/protocol/SellerProtocol.java index 1b4815f4678..9f1acab3689 100644 --- a/core/src/main/java/bisq/core/trade/protocol/SellerProtocol.java +++ b/core/src/main/java/bisq/core/trade/protocol/SellerProtocol.java @@ -39,12 +39,17 @@ import bisq.common.handlers.ErrorMessageHandler; import bisq.common.handlers.ResultHandler; +import bisq.common.util.Utilities; + +import java.util.Date; +import java.util.GregorianCalendar; import lombok.extern.slf4j.Slf4j; @Slf4j public abstract class SellerProtocol extends DisputeProtocol { enum SellerEvent implements FluentProtocol.Event { + STARTUP, PAYMENT_RECEIVED } @@ -52,6 +57,27 @@ public SellerProtocol(SellerTrade trade) { super(trade); } + @Override + protected void onInitialized() { + super.onInitialized(); + + // We get called the constructor with any possible state and phase. As we don't want to log an error for such + // cases we use the alternative 'given' method instead of 'expect'. + + // We only re-publish for about 2 weeks after 1.4.0 release until most nodes have updated to + // achieve sufficient resilience. + boolean currentDateBeforeCutOffDate = new Date().before(Utilities.getUTCDate(2020, GregorianCalendar.NOVEMBER, 1)); + given(anyPhase(Trade.Phase.DEPOSIT_PUBLISHED, + Trade.Phase.DEPOSIT_CONFIRMED, + Trade.Phase.FIAT_SENT, + Trade.Phase.FIAT_RECEIVED, + Trade.Phase.PAYOUT_PUBLISHED) + .with(SellerEvent.STARTUP) + .preCondition(currentDateBeforeCutOffDate)) + .setup(tasks(SellerPublishesTradeStatistics.class)) + .executeTasks(); + } + /////////////////////////////////////////////////////////////////////////////////////////// // Mailbox From 68a10cf5a17f914a2e1f9f1d0e559c707d89262e Mon Sep 17 00:00:00 2001 From: chimp1984 Date: Thu, 8 Oct 2020 18:51:24 -0500 Subject: [PATCH 57/59] Remove comment line --- p2p/src/main/java/bisq/network/p2p/P2PService.java | 5 ----- 1 file changed, 5 deletions(-) diff --git a/p2p/src/main/java/bisq/network/p2p/P2PService.java b/p2p/src/main/java/bisq/network/p2p/P2PService.java index bf594ecf6fc..8716c81d9d1 100644 --- a/p2p/src/main/java/bisq/network/p2p/P2PService.java +++ b/p2p/src/main/java/bisq/network/p2p/P2PService.java @@ -928,11 +928,6 @@ public Optional findPeersCapabilities(NodeAddress peer) { .findAny(); } - - /////////////////////////////////////////////////////////////////////////////////////////// - // Private - /////////////////////////////////////////////////////////////////////////////////////////// - @Value public class MailboxItem { private final ProtectedMailboxStorageEntry protectedMailboxStorageEntry; From 66740b7dc1113e72f17c2df15c92e4135568d414 Mon Sep 17 00:00:00 2001 From: chimp1984 Date: Thu, 8 Oct 2020 19:02:51 -0500 Subject: [PATCH 58/59] Remove unused variable --- .../main/market/trades/TradesChartsViewModelTest.java | 7 ------- 1 file changed, 7 deletions(-) diff --git a/desktop/src/test/java/bisq/desktop/main/market/trades/TradesChartsViewModelTest.java b/desktop/src/test/java/bisq/desktop/main/market/trades/TradesChartsViewModelTest.java index 8544363176c..8d3a7d25660 100644 --- a/desktop/src/test/java/bisq/desktop/main/market/trades/TradesChartsViewModelTest.java +++ b/desktop/src/test/java/bisq/desktop/main/market/trades/TradesChartsViewModelTest.java @@ -29,9 +29,6 @@ import bisq.core.trade.statistics.TradeStatisticsManager; import bisq.core.user.Preferences; -import bisq.common.crypto.KeyRing; -import bisq.common.crypto.KeyStorage; - import org.bitcoinj.core.Coin; import org.bitcoinj.utils.Fiat; @@ -50,9 +47,6 @@ import java.util.HashSet; import java.util.Set; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import org.junit.Before; import org.junit.Ignore; import org.junit.Test; @@ -116,7 +110,6 @@ public void setup() throws IOException { dir.delete(); //noinspection ResultOfMethodCallIgnored dir.mkdir(); - keyRing = new KeyRing(new KeyStorage(dir)); } @SuppressWarnings("ConstantConditions") From 39c8ade5ef30ac28fe3b465674ea74c3f7efcc98 Mon Sep 17 00:00:00 2001 From: chimp1984 Date: Thu, 8 Oct 2020 22:00:57 -0500 Subject: [PATCH 59/59] Cleanups: Remove outdated TODOs, fix typos --- .../src/main/java/bisq/common/app/Version.java | 1 - .../consensus/UsedForTradeContractJson.java | 3 --- .../main/java/bisq/common/crypto/Encryption.java | 1 - .../main/java/bisq/common/crypto/KeyStorage.java | 1 - .../common/proto/network/NetworkEnvelope.java | 1 - .../java/bisq/core/provider/fee/FeeService.java | 1 - .../dispute/arbitration/ArbitrationManager.java | 2 +- .../desktop/components/AddressTextField.java | 3 --- .../desktop/components/BsqAddressTextField.java | 3 --- .../components/TextFieldWithCopyIcon.java | 3 --- .../account/register/AgentRegistrationView.java | 2 -- .../desktop/main/market/spread/SpreadView.java | 2 +- .../main/market/spread/SpreadViewModel.java | 2 +- .../main/java/bisq/desktop/util/FormBuilder.java | 2 +- .../java/bisq/monitor/metric/MarketStats.java | 16 ---------------- .../java/bisq/monitor/metric/PriceNodeStats.java | 3 +-- .../bisq/monitor/metric/TorRoundTripTime.java | 3 ++- .../java/bisq/monitor/metric/TorStartupTime.java | 2 -- 18 files changed, 7 insertions(+), 44 deletions(-) diff --git a/common/src/main/java/bisq/common/app/Version.java b/common/src/main/java/bisq/common/app/Version.java index fd8cfb64634..45fc057242b 100644 --- a/common/src/main/java/bisq/common/app/Version.java +++ b/common/src/main/java/bisq/common/app/Version.java @@ -128,7 +128,6 @@ public static void printVersion() { '}'); } - //TODO move to consensus area public static final byte COMPENSATION_REQUEST = (byte) 0x01; public static final byte REIMBURSEMENT_REQUEST = (byte) 0x01; public static final byte PROPOSAL = (byte) 0x01; diff --git a/common/src/main/java/bisq/common/consensus/UsedForTradeContractJson.java b/common/src/main/java/bisq/common/consensus/UsedForTradeContractJson.java index 6aa722a4d7b..94105d34f39 100644 --- a/common/src/main/java/bisq/common/consensus/UsedForTradeContractJson.java +++ b/common/src/main/java/bisq/common/consensus/UsedForTradeContractJson.java @@ -24,8 +24,5 @@ * Better to use the excludeFromJsonDataMap (annotated with @JsonExclude; used in PaymentAccountPayload) to * add a key/value pair. */ -// TODO PubKeyRing and NodeAddress (network) are using UsedForTradeContractJson that is why it is in common module, -// which is a bit weird... Maybe we need either rename common or split it to util and common where common is common code -// used in network and core? public interface UsedForTradeContractJson { } diff --git a/common/src/main/java/bisq/common/crypto/Encryption.java b/common/src/main/java/bisq/common/crypto/Encryption.java index f4e1f8e0221..8be207fe4c2 100644 --- a/common/src/main/java/bisq/common/crypto/Encryption.java +++ b/common/src/main/java/bisq/common/crypto/Encryption.java @@ -48,7 +48,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -// TODO is Hmac needed/make sense? public class Encryption { private static final Logger log = LoggerFactory.getLogger(Encryption.class); diff --git a/common/src/main/java/bisq/common/crypto/KeyStorage.java b/common/src/main/java/bisq/common/crypto/KeyStorage.java index c132ab0042f..7b4ecece087 100644 --- a/common/src/main/java/bisq/common/crypto/KeyStorage.java +++ b/common/src/main/java/bisq/common/crypto/KeyStorage.java @@ -55,7 +55,6 @@ import static bisq.common.util.Preconditions.checkDir; -// TODO: use a password protection for key storage @Singleton public class KeyStorage { private static final Logger log = LoggerFactory.getLogger(KeyStorage.class); diff --git a/common/src/main/java/bisq/common/proto/network/NetworkEnvelope.java b/common/src/main/java/bisq/common/proto/network/NetworkEnvelope.java index 535d65d0486..330fbd72161 100644 --- a/common/src/main/java/bisq/common/proto/network/NetworkEnvelope.java +++ b/common/src/main/java/bisq/common/proto/network/NetworkEnvelope.java @@ -48,7 +48,6 @@ public Message toProtoMessage() { return getNetworkEnvelopeBuilder().build(); } - // todo remove public protobuf.NetworkEnvelope toProtoNetworkEnvelope() { return getNetworkEnvelopeBuilder().build(); } diff --git a/core/src/main/java/bisq/core/provider/fee/FeeService.java b/core/src/main/java/bisq/core/provider/fee/FeeService.java index 76427f5b3aa..b43c4491b62 100644 --- a/core/src/main/java/bisq/core/provider/fee/FeeService.java +++ b/core/src/main/java/bisq/core/provider/fee/FeeService.java @@ -52,7 +52,6 @@ import static com.google.common.base.Preconditions.checkNotNull; -// TODO use dao parameters for fee @Slf4j public class FeeService { diff --git a/core/src/main/java/bisq/core/support/dispute/arbitration/ArbitrationManager.java b/core/src/main/java/bisq/core/support/dispute/arbitration/ArbitrationManager.java index 8552003199d..29f6b6939aa 100644 --- a/core/src/main/java/bisq/core/support/dispute/arbitration/ArbitrationManager.java +++ b/core/src/main/java/bisq/core/support/dispute/arbitration/ArbitrationManager.java @@ -157,7 +157,7 @@ public void cleanupDisputes() { @Override protected String getDisputeInfo(Dispute dispute) { String role = Res.get("shared.arbitrator").toLowerCase(); - String link = "https://docs.bisq.network/trading-rules.html#legacy-arbitration"; //TODO needs to be created + String link = "https://docs.bisq.network/trading-rules.html#legacy-arbitration"; return Res.get("support.initialInfo", role, role, link); } diff --git a/desktop/src/main/java/bisq/desktop/components/AddressTextField.java b/desktop/src/main/java/bisq/desktop/components/AddressTextField.java index 9bc06ce5147..f9ca98a1130 100644 --- a/desktop/src/main/java/bisq/desktop/components/AddressTextField.java +++ b/desktop/src/main/java/bisq/desktop/components/AddressTextField.java @@ -78,9 +78,6 @@ public AddressTextField(String label) { }); textField.focusTraversableProperty().set(focusTraversableProperty().get()); - //TODO app wide focus - //focusedProperty().addListener((ov, oldValue, newValue) -> textField.requestFocus()); - Label extWalletIcon = new Label(); extWalletIcon.setLayoutY(3); extWalletIcon.getStyleClass().addAll("icon", "highlight"); diff --git a/desktop/src/main/java/bisq/desktop/components/BsqAddressTextField.java b/desktop/src/main/java/bisq/desktop/components/BsqAddressTextField.java index 97d61fd07bf..a74a4ae2e3b 100644 --- a/desktop/src/main/java/bisq/desktop/components/BsqAddressTextField.java +++ b/desktop/src/main/java/bisq/desktop/components/BsqAddressTextField.java @@ -74,9 +74,6 @@ public BsqAddressTextField() { }); textField.focusTraversableProperty().set(focusTraversableProperty().get()); - //TODO app wide focus - //focusedProperty().addListener((ov, oldValue, newValue) -> textField.requestFocus()); - Label copyIcon = new Label(); copyIcon.setLayoutY(3); diff --git a/desktop/src/main/java/bisq/desktop/components/TextFieldWithCopyIcon.java b/desktop/src/main/java/bisq/desktop/components/TextFieldWithCopyIcon.java index fe26aebeab7..7f2892379ca 100644 --- a/desktop/src/main/java/bisq/desktop/components/TextFieldWithCopyIcon.java +++ b/desktop/src/main/java/bisq/desktop/components/TextFieldWithCopyIcon.java @@ -79,9 +79,6 @@ public TextFieldWithCopyIcon(String customStyleClass) { AnchorPane.setRightAnchor(textField, 30.0); AnchorPane.setLeftAnchor(textField, 0.0); textField.focusTraversableProperty().set(focusTraversableProperty().get()); - //TODO app wide focus - //focusedProperty().addListener((ov, oldValue, newValue) -> textField.requestFocus()); - getChildren().addAll(textField, copyIcon); } diff --git a/desktop/src/main/java/bisq/desktop/main/account/register/AgentRegistrationView.java b/desktop/src/main/java/bisq/desktop/main/account/register/AgentRegistrationView.java index 4af167a5940..656d126f86d 100644 --- a/desktop/src/main/java/bisq/desktop/main/account/register/AgentRegistrationView.java +++ b/desktop/src/main/java/bisq/desktop/main/account/register/AgentRegistrationView.java @@ -64,8 +64,6 @@ import static bisq.desktop.util.FormBuilder.addTitledGroupBg; import static bisq.desktop.util.FormBuilder.addTopLabelTextField; -// TODO translation string keys should renamed to be more generic. -// Lets do it for 1.1.7 the translator have time to add new string. public abstract class AgentRegistrationView> extends ActivatableViewAndModel { diff --git a/desktop/src/main/java/bisq/desktop/main/market/spread/SpreadView.java b/desktop/src/main/java/bisq/desktop/main/market/spread/SpreadView.java index 02a2a402e51..539ab5ffad1 100644 --- a/desktop/src/main/java/bisq/desktop/main/market/spread/SpreadView.java +++ b/desktop/src/main/java/bisq/desktop/main/market/spread/SpreadView.java @@ -298,7 +298,7 @@ public TableCell call( public void updateItem(final SpreadItem item, boolean empty) { super.updateItem(item, empty); if (item != null && !empty) { - // TODO maybe show exra colums with item.priceSpread and use real amount diff + // TODO maybe show extra columns with item.priceSpread and use real amount diff // not % based if (item.priceSpread != null) setText(item.percentage); diff --git a/desktop/src/main/java/bisq/desktop/main/market/spread/SpreadViewModel.java b/desktop/src/main/java/bisq/desktop/main/market/spread/SpreadViewModel.java index 97aecedb3cb..e5ee933ca0d 100644 --- a/desktop/src/main/java/bisq/desktop/main/market/spread/SpreadViewModel.java +++ b/desktop/src/main/java/bisq/desktop/main/market/spread/SpreadViewModel.java @@ -172,7 +172,7 @@ private void update(ObservableList offerBookListItems) { else spread = bestBuyOfferPrice.subtract(bestSellOfferPrice); - // TODO maybe show extra colums with spread and use real amount diff + // TODO maybe show extra columns with spread and use real amount diff // not % based. e.g. diff between best buy and sell offer (of small amounts its a smaller gain) if (spread != null && marketPrice != null && marketPrice.isPriceAvailable()) { diff --git a/desktop/src/main/java/bisq/desktop/util/FormBuilder.java b/desktop/src/main/java/bisq/desktop/util/FormBuilder.java index 46db98c0153..f8061e04f99 100644 --- a/desktop/src/main/java/bisq/desktop/util/FormBuilder.java +++ b/desktop/src/main/java/bisq/desktop/util/FormBuilder.java @@ -332,7 +332,7 @@ public static Tuple3 addTopLabelTextField(GridPane gridP final Tuple2 topLabelWithVBox = addTopLabelWithVBox(gridPane, rowIndex, title, textField, top); - // TOD not 100% sure if that is a good idea.... + // TODO not 100% sure if that is a good idea.... //topLabelWithVBox.first.getStyleClass().add("jfx-text-field-top-label"); return new Tuple3<>(topLabelWithVBox.first, textField, topLabelWithVBox.second); diff --git a/monitor/src/main/java/bisq/monitor/metric/MarketStats.java b/monitor/src/main/java/bisq/monitor/metric/MarketStats.java index 389722329a3..f831cafd135 100644 --- a/monitor/src/main/java/bisq/monitor/metric/MarketStats.java +++ b/monitor/src/main/java/bisq/monitor/metric/MarketStats.java @@ -20,30 +20,15 @@ import bisq.monitor.Metric; import bisq.monitor.Reporter; -import bisq.asset.Asset; -import bisq.asset.AssetRegistry; - -import bisq.network.p2p.storage.payload.ProtectedStoragePayload; - -import org.berndpruenster.netlayer.tor.TorCtlException; - -import com.runjva.sourceforge.jsocks.protocol.SocksSocket; - -import java.net.HttpURLConnection; -import java.net.Socket; import java.net.URL; import java.net.URLConnection; import java.io.BufferedReader; -import java.io.BufferedWriter; import java.io.IOException; import java.io.InputStreamReader; -import java.io.OutputStreamWriter; -import java.io.PrintWriter; import java.util.Arrays; import java.util.HashMap; -import java.util.List; import java.util.Map; import java.util.concurrent.TimeUnit; import java.util.regex.Matcher; @@ -110,7 +95,6 @@ protected void execute() { } catch (IllegalStateException ignore) { // no match found } catch (IOException e) { - // TODO Auto-generated catch block e.printStackTrace(); } } diff --git a/monitor/src/main/java/bisq/monitor/metric/PriceNodeStats.java b/monitor/src/main/java/bisq/monitor/metric/PriceNodeStats.java index 9258eb5e8d5..fd10691f727 100644 --- a/monitor/src/main/java/bisq/monitor/metric/PriceNodeStats.java +++ b/monitor/src/main/java/bisq/monitor/metric/PriceNodeStats.java @@ -54,7 +54,7 @@ /** * Fetches fee and price data from the configured price nodes. * Based on the work of HarryMcFinned. - * + * * @author Florian Reimair * @author HarryMcFinned * @@ -159,7 +159,6 @@ protected void execute() { } } } catch (TorCtlException | IOException e) { - // TODO Auto-generated catch block e.printStackTrace(); } } diff --git a/monitor/src/main/java/bisq/monitor/metric/TorRoundTripTime.java b/monitor/src/main/java/bisq/monitor/metric/TorRoundTripTime.java index 16499689b08..a40c0261662 100644 --- a/monitor/src/main/java/bisq/monitor/metric/TorRoundTripTime.java +++ b/monitor/src/main/java/bisq/monitor/metric/TorRoundTripTime.java @@ -21,6 +21,7 @@ import bisq.monitor.OnionParser; import bisq.monitor.Reporter; import bisq.monitor.StatisticsHelper; + import bisq.network.p2p.NodeAddress; import org.berndpruenster.netlayer.tor.Tor; @@ -33,6 +34,7 @@ import java.util.ArrayList; import java.util.List; + import static com.google.common.base.Preconditions.checkNotNull; /** @@ -84,7 +86,6 @@ protected void execute() { reporter.report(StatisticsHelper.process(samples), getName()); } } catch (TorCtlException | IOException e) { - // TODO Auto-generated catch block e.printStackTrace(); } } diff --git a/monitor/src/main/java/bisq/monitor/metric/TorStartupTime.java b/monitor/src/main/java/bisq/monitor/metric/TorStartupTime.java index 6e53e02462b..9ea3198e197 100644 --- a/monitor/src/main/java/bisq/monitor/metric/TorStartupTime.java +++ b/monitor/src/main/java/bisq/monitor/metric/TorStartupTime.java @@ -58,7 +58,6 @@ public void configure(Properties properties) { try { torOverrides = new Torrc(overrides); } catch (IOException e) { - // TODO Auto-generated catch block e.printStackTrace(); } } @@ -79,7 +78,6 @@ protected void execute() { // stop the timer and set its timestamp reporter.report(System.currentTimeMillis() - start, getName()); } catch (TorCtlException e) { - // TODO Auto-generated catch block e.printStackTrace(); } finally { // cleanup