Merge pull request #1258 from GalRogozinski/spentaddresses-pruning

Spentaddresses are persisted upon pruning
iotaledger · Jan 8, 2019 · 56a724f · 56a724f
2 parents 7072a7d + d801de5
commit 56a724f
Show file tree

Hide file tree

Showing 17 changed files with 188 additions and 86 deletions.
diff --git a/pom.xml b/pom.xml
@@ -196,6 +196,12 @@
             <scope>test</scope>
         </dependency>
 
+        <dependency>
+            <groupId>pl.touk</groupId>
+            <artifactId>throwing-function</artifactId>
+            <version>1.3</version>
+        </dependency>
+
     </dependencies>
 
     <build>
@@ -371,6 +377,9 @@
                                         <urn>
                                             com.beust:jcommander:1.72:jar:null:compile:6375e521c1e11d6563d4f25a07ce124ccf8cd171
                                         </urn>
+                                        <urn>
+                                            pl.touk:throwing-function:1.3:jar:null:compile:32947866b8754295efde73ee7d39ea29a247a2b5
+                                        </urn>
 
                                         <!-- A check for the rules themselves -->
                                         <urn>
@@ -651,4 +660,4 @@
             </plugin>
         </plugins>
     </reporting>
-</project>
+</project>
diff --git a/src/main/java/com/iota/iri/IRI.java b/src/main/java/com/iota/iri/IRI.java
@@ -18,12 +18,12 @@
 import org.slf4j.LoggerFactory;
 
 /**
- * 
+ *
  * Main IOTA Reference Implementation (IRI) starting class.
  * <p>
  *     The IRI software enables the Tangle to operate. Individuals can run IRI to operates Nodes.
- *     The Node running the IRI software enables your device to communicate with neighbors 
- *     in the peer-to-peer network that the Tangle operates on. 
+ *     The Node running the IRI software enables your device to communicate with neighbors
+ *     in the peer-to-peer network that the Tangle operates on.
  * </p>
  * <p>
  *     IRI implements all the core functionality necessary for participating in an IOTA network as a full node.
@@ -35,7 +35,7 @@
  *         <li>Loading custom modules that extend the API.</li>
  *     </ul>
  * </p>
- * 
+ *
  * @see <a href="https://docs.iota.org/iri">Online documentation on iri</a>
  */
 public class IRI {
@@ -48,13 +48,13 @@ public class IRI {
      * The entry point of IRI.
      * Starts by configuring the logging settings, then proceeds to {@link IRILauncher#main(String[])}
      * The log level is set to INFO by default.
-     * 
+     *
      * @param args Configuration arguments. See {@link BaseIotaConfig} for a list of all options.
      * @throws Exception If we fail to start the IRI launcher.
      */
     public static void main(String[] args) throws Exception {
         // Logging is configured first before any references to Logger or LoggerFactory.
-        // Any public method or field accessors needed in IRI should be put in IRI and then delegate to IRILauncher. 
+        // Any public method or field accessors needed in IRI should be put in IRI and then delegate to IRILauncher.
         // That ensures that future code does not need to know about this setup.
         configureLogging();
         IRILauncher.main(args);
@@ -98,11 +98,11 @@ private static class IRILauncher {
          *     <li>Load the configuration.</li>
          *     <li>Create {@link Iota}, {@link IXI} and {@link API}.</li>
          *     <li>Listen for node shutdown.</li>
-         *     <li>Initialize {@link Iota}, {@link IXI} and {@link API} using their <tt>init()</tt> methods.</li> 
-         * </ul> 
-         * 
+         *     <li>Initialize {@link Iota}, {@link IXI} and {@link API} using their <tt>init()</tt> methods.</li>
+         * </ul>
+         *
          * If no exception is thrown, the node starts synchronizing with the network, and the API can be used.
-         * 
+         *
          * @param args Configuration arguments. See {@link BaseIotaConfig} for a list of all options.
          * @throws Exception If any of the <tt>init()</tt> methods failed to initialize.
          */

diff --git a/src/main/java/com/iota/iri/Iota.java b/src/main/java/com/iota/iri/Iota.java
@@ -23,7 +23,6 @@
 import com.iota.iri.service.tipselection.impl.*;
 import com.iota.iri.service.transactionpruning.TransactionPruningException;
 import com.iota.iri.service.transactionpruning.async.AsyncTransactionPruner;
-import com.iota.iri.storage.*;
 import com.iota.iri.storage.Indexable;
 import com.iota.iri.storage.Persistable;
 import com.iota.iri.storage.Tangle;
@@ -216,7 +215,8 @@ private void injectDependencies() throws SnapshotException, TransactionPruningEx
         milestoneSolidifier.init(snapshotProvider, transactionValidator);
         ledgerService.init(tangle, snapshotProvider, snapshotService, milestoneService);
         if (transactionPruner != null) {
-            transactionPruner.init(tangle, snapshotProvider, tipsViewModel, configuration).restoreState();
+            transactionPruner.init(tangle, snapshotProvider, spentAddressesService, tipsViewModel, configuration)
+                    .restoreState();
         }
         transactionRequesterWorker.init(tangle, transactionRequester, tipsViewModel, node);
     }

diff --git a/src/main/java/com/iota/iri/service/snapshot/impl/SnapshotServiceImpl.java b/src/main/java/com/iota/iri/service/snapshot/impl/SnapshotServiceImpl.java
@@ -7,24 +7,18 @@
 import com.iota.iri.controllers.TransactionViewModel;
 import com.iota.iri.model.Hash;
 import com.iota.iri.service.milestone.LatestMilestoneTracker;
-import com.iota.iri.service.snapshot.SnapshotMetaData;
-import com.iota.iri.service.snapshot.SnapshotService;
-import com.iota.iri.service.snapshot.Snapshot;
-import com.iota.iri.service.snapshot.SnapshotException;
-import com.iota.iri.service.snapshot.SnapshotProvider;
+import com.iota.iri.service.snapshot.*;
 import com.iota.iri.service.spentaddresses.SpentAddressesProvider;
 import com.iota.iri.service.spentaddresses.SpentAddressesService;
 import com.iota.iri.service.transactionpruning.TransactionPruner;
 import com.iota.iri.service.transactionpruning.TransactionPruningException;
 import com.iota.iri.service.transactionpruning.jobs.MilestonePrunerJob;
 import com.iota.iri.service.transactionpruning.jobs.UnconfirmedSubtanglePrunerJob;
 import com.iota.iri.storage.Tangle;
-import com.iota.iri.utils.log.ProgressLogger;
-import com.iota.iri.utils.log.interval.IntervalProgressLogger;
 import com.iota.iri.utils.dag.DAGHelper;
 import com.iota.iri.utils.dag.TraversalException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
+import com.iota.iri.utils.log.ProgressLogger;
+import com.iota.iri.utils.log.interval.IntervalProgressLogger;
 
 import java.util.HashMap;
 import java.util.HashSet;
@@ -33,6 +27,9 @@
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.stream.Collectors;
 
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
 /**
  * Creates a service instance that allows us to access the business logic for {@link Snapshot}s.<br />
  * <br />
@@ -500,7 +497,7 @@ private void persistLocalSnapshot(SnapshotProvider snapshotProvider, Snapshot ne
             throws SnapshotException {
 
         try {
-            spentAddressesService.calculateSpentAddresses(snapshotProvider.getInitialSnapshot().getIndex(),
+            spentAddressesService.persistSpentAddresses(snapshotProvider.getInitialSnapshot().getIndex(),
                     newSnapshot.getIndex());
 
             spentAddressesProvider.writeSpentAddressesToDisk(config.getLocalSnapshotsBasePath() +

diff --git a/src/main/java/com/iota/iri/service/spentaddresses/SpentAddressesProvider.java b/src/main/java/com/iota/iri/service/spentaddresses/SpentAddressesProvider.java
@@ -1,9 +1,9 @@
 package com.iota.iri.service.spentaddresses;
 
-import java.util.List;
-
 import com.iota.iri.model.Hash;
 
+import java.util.Collection;
+
 /**
  * Find, mark and store spent addresses
  */
@@ -32,7 +32,7 @@ public interface SpentAddressesProvider {
      * @param addressHashes The addresses we want to mark
      * @throws SpentAddressesException If the provider fails to add an address
      */
-    void addAddressesBatch(List<Hash> addressHashes) throws SpentAddressesException;
+    void addAddressesBatch(Collection<Hash> addressHashes) throws SpentAddressesException;
 
     /**
      * Writes all currently known spent addresses to disk.

diff --git a/src/main/java/com/iota/iri/service/spentaddresses/SpentAddressesService.java b/src/main/java/com/iota/iri/service/spentaddresses/SpentAddressesService.java
@@ -1,7 +1,10 @@
 package com.iota.iri.service.spentaddresses;
 
+import com.iota.iri.controllers.TransactionViewModel;
 import com.iota.iri.model.Hash;
 
+import java.util.Collection;
+
 /**
  * 
  * Check and calculate spent addresses
@@ -18,11 +21,18 @@ public interface SpentAddressesService {
     boolean wasAddressSpentFrom(Hash addressHash) throws SpentAddressesException;
 
     /**
-     * Calculate all spent addresses in between a range
+     * Calculates and persists all spent addresses in between a range that were validly signed
      * 
      * @param fromMilestoneIndex the lower bound milestone index (inclusive)
      * @param toMilestoneIndex the upper bound milestone index (exclusive)
      * @throws Exception when anything went wrong whilst calculating.
      */
-    void calculateSpentAddresses(int fromMilestoneIndex, int toMilestoneIndex) throws Exception;
+    void persistSpentAddresses(int fromMilestoneIndex, int toMilestoneIndex) throws Exception;
+
+    /**
+     * Persist all the verifiable spent from a given list of transactions
+     * @param transactions transactions to obtain spends from
+     * @throws SpentAddressesException
+     */
+    void persistSpentAddresses(Collection<TransactionViewModel> transactions) throws SpentAddressesException;
 }
diff --git a/src/main/java/com/iota/iri/service/spentaddresses/impl/SpentAddressesProviderImpl.java b/src/main/java/com/iota/iri/service/spentaddresses/impl/SpentAddressesProviderImpl.java
@@ -130,7 +130,7 @@ public void addAddress(Hash addressHash) throws SpentAddressesException {
     }
 
     @Override
-    public void addAddressesBatch(List<Hash> addressHash) throws SpentAddressesException {
+    public void addAddressesBatch(Collection<Hash> addressHash) throws SpentAddressesException {
         try {
             // Its bytes are always new byte[0], therefore identical in storage
             SpentAddress spentAddressModel = new SpentAddress();

diff --git a/src/main/java/com/iota/iri/service/spentaddresses/impl/SpentAddressesServiceImpl.java b/src/main/java/com/iota/iri/service/spentaddresses/impl/SpentAddressesServiceImpl.java
@@ -14,11 +14,13 @@
 import com.iota.iri.storage.Tangle;
 import com.iota.iri.utils.dag.DAGHelper;
 
-import java.util.HashSet;
-import java.util.Optional;
-import java.util.Set;
+import java.util.*;
 import java.util.stream.Collectors;
 
+import org.apache.commons.collections4.CollectionUtils;
+
+import pl.touk.throwing.ThrowingPredicate;
+
 /**
  * 
  * Implementation of <tt>SpentAddressesService</tt> that calculates and checks spent addresses using the {@link Tangle}
@@ -68,9 +70,9 @@ public boolean wasAddressSpentFrom(Hash addressHash) throws SpentAddressesExcept
                     }
 
                     // Transaction is pending
-                    Hash tail = findTail(hash);
-                    if (tail != null && BundleValidator.validate(tangle, snapshotProvider.getInitialSnapshot(), tail).size() != 0) {
-                        return true;
+                    Optional<Hash> tail = tailFinder.findTailFromTx(tx);
+                    if (tail.isPresent()) {
+                        return isBundleValid(tail.get());
                     }
                 }
             }
@@ -82,7 +84,7 @@ public boolean wasAddressSpentFrom(Hash addressHash) throws SpentAddressesExcept
     }
 
     @Override
-    public void calculateSpentAddresses(int fromMilestoneIndex, int toMilestoneIndex) throws SpentAddressesException {
+    public void persistSpentAddresses(int fromMilestoneIndex, int toMilestoneIndex) throws SpentAddressesException {
         Set<Hash> addressesToCheck = new HashSet<>();
         try {
             for (int i = fromMilestoneIndex; i < toMilestoneIndex; i++) {
@@ -102,15 +104,8 @@ public void calculateSpentAddresses(int fromMilestoneIndex, int toMilestoneIndex
         //Can only throw runtime exceptions in streams
         try {
             spentAddressesProvider.addAddressesBatch(addressesToCheck.stream()
-                .filter(address -> {
-                    try {
-                        return wasAddressSpentFrom(address);
-                    } catch (SpentAddressesException e) {
-                        throw new RuntimeException(e);
-                    }
-                })
+                .filter(ThrowingPredicate.unchecked(this::wasAddressSpentFrom))
                 .collect(Collectors.toList()));
-
         } catch (RuntimeException e) {
             if (e.getCause() instanceof SpentAddressesException) {
                 throw (SpentAddressesException) e.getCause();
@@ -120,16 +115,38 @@ public void calculateSpentAddresses(int fromMilestoneIndex, int toMilestoneIndex
         }
     }
 
-    /**
-     * Walks back from the hash until a tail transaction has been found or transaction aprovee is not found.
-     * A tail transaction is the first transaction in a bundle, thus with <code>index = 0</code>
-     *
-     * @param hash The transaction hash where we start the search from. If this is a tail, its hash is returned.
-     * @return The transaction hash of the tail
-     * @throws Exception When a model could not be loaded.
-     */
-    private Hash findTail(Hash hash) throws Exception {
-        Optional<Hash> optionalTail = tailFinder.findTail(hash);
-        return optionalTail.isPresent() ? optionalTail.get() : null;
+    @Override
+    public void persistSpentAddresses(Collection<TransactionViewModel> transactions) throws SpentAddressesException {
+        try {
+            Collection<Hash> spentAddresses = transactions.stream()
+                    .filter(ThrowingPredicate.unchecked(this::wasTransactionSpentFrom))
+                    .map(TransactionViewModel::getAddressHash).collect(Collectors.toSet());
+
+            spentAddressesProvider.addAddressesBatch(spentAddresses);
+        } catch (RuntimeException e) {
+            throw new SpentAddressesException("Exception while persisting spent addresses", e);
+        }
+    }
+
+    private boolean wasTransactionSpentFrom(TransactionViewModel tx) throws Exception {
+        Optional<Hash> tailFromTx = tailFinder.findTailFromTx(tx);
+        if (tailFromTx.isPresent() && tx.value() < 0) {
+            // Transaction is confirmed
+            if (tx.snapshotIndex() != 0) {
+                return true;
+            }
+
+            // transaction is pending
+            Hash tailHash = tailFromTx.get();
+            return isBundleValid(tailHash);
+        }
+
+        return false;
+    }
+
+    private boolean isBundleValid(Hash tailHash) throws Exception {
+        List<List<TransactionViewModel>> validation =
+                BundleValidator.validate(tangle, snapshotProvider.getInitialSnapshot(), tailHash);
+        return (CollectionUtils.isNotEmpty(validation) && validation.get(0).get(0).getValidity() == 1);
     }
 }
diff --git a/src/main/java/com/iota/iri/service/tipselection/TailFinder.java b/src/main/java/com/iota/iri/service/tipselection/TailFinder.java
@@ -1,22 +1,30 @@
 package com.iota.iri.service.tipselection;
 
-import java.util.Optional;
-
+import com.iota.iri.controllers.TransactionViewModel;
 import com.iota.iri.model.Hash;
 
+import java.util.Optional;
+
 /**
  * Finds the tail of a bundle
  */
 
-@FunctionalInterface
 public interface TailFinder {
     /**
-     *Method for finding a tail (current_index=0) of a bundle given any transaction in the associated bundle.
+     * Method for finding a tail (current_index=0) of a bundle given any transaction in the associated bundle.
      *
      * @param hash The transaction hash of any transaction in the bundle.
      * @return Hash of the tail transaction, or {@code Empty} if the tail is not found.
      * @throws Exception If DB fails to retrieve transactions
      */
     Optional<Hash> findTail(Hash hash) throws Exception;
 
+    /**
+     * Method for finding a tail (current_index=0) of a bundle given any transaction in the associated bundle.
+     *
+     * @param tx any transaction in the bundle.
+     * @return Hash of the tail transaction, or {@code Empty} if the tail is not found.
+     * @throws Exception If DB fails to retrieve transactions
+     */
+    Optional<Hash> findTailFromTx(TransactionViewModel tx) throws Exception;
 }
diff --git a/src/main/java/com/iota/iri/service/tipselection/impl/TailFinderImpl.java b/src/main/java/com/iota/iri/service/tipselection/impl/TailFinderImpl.java
@@ -27,6 +27,11 @@ public TailFinderImpl(Tangle tangle) {
     @Override
     public Optional<Hash> findTail(Hash hash) throws Exception {
         TransactionViewModel tx = TransactionViewModel.fromHash(tangle, hash);
+        return findTailFromTx(tx);
+    }
+
+    @Override
+    public Optional<Hash> findTailFromTx(TransactionViewModel tx) throws Exception {
         final Hash bundleHash = tx.getBundleHash();
         long index = tx.getCurrentIndex();
         while (index-- > 0 && bundleHash.equals(tx.getBundleHash())) {

diff --git a/src/main/java/com/iota/iri/service/transactionpruning/TransactionPrunerJob.java b/src/main/java/com/iota/iri/service/transactionpruning/TransactionPrunerJob.java
@@ -2,6 +2,7 @@
 
 import com.iota.iri.controllers.TipsViewModel;
 import com.iota.iri.service.snapshot.Snapshot;
+import com.iota.iri.service.spentaddresses.SpentAddressesService;
 import com.iota.iri.storage.Tangle;
 
 /**
@@ -28,6 +29,14 @@ public interface TransactionPrunerJob {
      */
     TransactionPruner getTransactionPruner();
 
+    /**
+     * Allows to set the {@link SpentAddressesService} that will ensure pruned valid transactions will have their
+     * spent addresses persisted the node
+     *
+     * @param spentAddressesService service to be injected
+     */
+    void setSpentAddressesService(SpentAddressesService spentAddressesService);
+
     /**
      * Allows to set the {@link Tangle} object that this job should work on.
      *