Merge pull request #271 from lzpap/pdecol_snapshot_pr_cleaned

Add `wereAddressesSpentFrom` check to used/new address discovery

docs/addresses.rst

@@ -17,17 +17,11 @@ any other financial service.
         These performance issues will be fixed in a future version of the library;
         please bear with us!
 
-        In the meantime, if you are using Python 3, you can install a C extension
+        In the meantime, you can install a C extension
         that boosts PyOTA's performance significantly (speedups of 60x are common!).
 
         To install the extension, run ``pip install pyota[ccurl]``.
 
-        **Important:** The extension is not yet compatible with Python 2.
-
-        If you are familiar with Python 2's C API, we'd love to hear from you!
-        Check the `GitHub issue <https://github.com/todofixthis/pyota-ccurl/issues/4>`_
-        for more information.
-
 PyOTA provides two methods for generating addresses:
 
 Using the API
@@ -60,7 +54,9 @@ method, using the following parameters:
    (defaults to 1).
 -  If ``None``, the API will generate addresses until it finds one that
    has not been used (has no transactions associated with it on the
-   Tangle). It will then return the unused address and discard the rest.
+   Tangle, and was not spent from). This makes the command safer to use after
+   a snapshot has been taken. It will then return the unused address and
+   discard the rest.
 -  ``security_level: int``: Determines the security level of the
    generated addresses. See `Security Levels <#security-levels>`__
    below.

iota/api.py

@@ -1217,8 +1217,9 @@ def get_new_addresses(
                 inside a loop.
 
             If ``None``, this method will progressively generate
-            addresses and scan the Tangle until it finds one that has no
-            transactions referencing it.
+            addresses and scan the Tangle until it finds one that is unused.
+            This is if no transactions are referencing it and it was not spent
+            from before.
 
         :param int security_level:
             Number of iterations to use when generating new addresses.

iota/commands/extended/get_account_data.py

@@ -59,7 +59,7 @@ def _execute(self, request):
             my_hashes = ft_command(addresses=my_addresses).get('hashes') or []
 
         account_balance = 0
-        if my_hashes:
+        if my_addresses:
             # Load balances for the addresses that we generated.
             gb_response = (
                 GetBalancesCommand(self.adapter)(addresses=my_addresses)

iota/commands/extended/get_new_addresses.py

@@ -9,6 +9,8 @@
 from iota import Address
 from iota.commands import FilterCommand, RequestFilter
 from iota.commands.core.find_transactions import FindTransactionsCommand
+from iota.commands.core.were_addresses_spent_from import \
+    WereAddressesSpentFromCommand
 from iota.crypto.addresses import AddressGenerator
 from iota.crypto.types import Seed
 from iota.filters import SecurityLevel, Trytes
@@ -58,17 +60,23 @@ def _find_addresses(self, seed, index, count, security_level, checksum):
         generator = AddressGenerator(seed, security_level, checksum)
 
         if count is None:
-            # Connect to Tangle and find the first address without any
-            # transactions.
+            # Connect to Tangle and find the first unused address.
             for addy in generator.create_iterator(start=index):
-                # We use addy.address here because FindTransactions does
+                # We use addy.address here because the commands do
                 # not work on an address with a checksum
+                response = WereAddressesSpentFromCommand(self.adapter)(
+                    addresses=[addy.address],
+                )
+                if response['states'][0]:
+                    continue
+
                 response = FindTransactionsCommand(self.adapter)(
                     addresses=[addy.address],
                 )
+                if response.get('hashes'):
+                    continue
 
-                if not response.get('hashes'):
-                    return [addy]
+                return [addy]
 
         return generator.get_addresses(start=index, count=count)
 

iota/commands/extended/utils.py

@@ -9,6 +9,8 @@
 from iota.adapter import BaseAdapter
 from iota.commands.core.find_transactions import FindTransactionsCommand
 from iota.commands.core.get_trytes import GetTrytesCommand
+from iota.commands.core.were_addresses_spent_from import \
+    WereAddressesSpentFromCommand
 from iota.commands.extended import FindTransactionObjectsCommand
 from iota.commands.extended.get_bundles import GetBundlesCommand
 from iota.commands.extended.get_latest_inclusion import \
@@ -25,26 +27,33 @@ def iter_used_addresses(
 ):
     # type: (...) -> Generator[Tuple[Address, List[TransactionHash]], None, None]
     """
-    Scans the Tangle for used addresses.
+    Scans the Tangle for used addresses. A used address is an address that
+    was spent from or has a transaction.
 
     This is basically the opposite of invoking ``getNewAddresses`` with
-    ``stop=None``.
+    ``count=None``.
     """
     if security_level is None:
         security_level = AddressGenerator.DEFAULT_SECURITY_LEVEL
 
     ft_command = FindTransactionsCommand(adapter)
+    wasf_command = WereAddressesSpentFromCommand(adapter)
 
     for addy in AddressGenerator(seed, security_level).create_iterator(start):
         ft_response = ft_command(addresses=[addy])
 
         if ft_response['hashes']:
             yield addy, ft_response['hashes']
         else:
-            break
+            wasf_response = wasf_command(addresses=[addy])
+            if wasf_response['states'][0]:
+                yield addy, []
+            else:
+                break
 
-        # Reset the command so that we can call it again.
+        # Reset the commands so that we can call them again.
         ft_command.reset()
+        wasf_command.reset()
 
 
 def get_bundles_from_transaction_hashes(

test/commands/extended/get_account_data_test.py

@@ -435,3 +435,28 @@ def test_no_transactions(self):
         'bundles':    [],
       },
     )
+
+  def test_balance_is_found_for_address_without_transaction(self):
+    """
+    If an address has a balance, no transactions and was spent from, the
+    balance should still be found and returned.
+    """
+    with mock.patch(
+        'iota.commands.extended.get_account_data.iter_used_addresses',
+        mock.Mock(return_value=[(self.addy1, [])]),
+    ):
+      self.adapter.seed_response('getBalances', {
+        'balances': [42],
+      })
+
+      response = self.command(seed=Seed.random())
+
+    self.assertDictEqual(
+      response,
+
+      {
+        'addresses':  [self.addy1],
+        'balance':    42,
+        'bundles':    [],
+      },
+    )

test/commands/extended/get_inputs_test.py

@@ -590,12 +590,9 @@ def test_no_stop_threshold_met(self):
     """
     No ``stop`` provided, balance meets ``threshold``.
     """
-    self.adapter.seed_response('getBalances', {
-      'balances': [42, 29],
-    })
 
-    # ``getInputs`` uses ``findTransactions`` to identify unused
-    # addresses.
+    # ``getInputs`` uses ``findTransactions`` and
+    # ``wereAddressesSpentFrom`` to identify unused addresses.
     # noinspection SpellCheckingInspection
     self.adapter.seed_response('findTransactions', {
       'hashes': [
@@ -620,6 +617,14 @@ def test_no_stop_threshold_met(self):
       'hashes': [],
     })
 
+    self.adapter.seed_response('wereAddressesSpentFrom', {
+      'states': [False],
+    })
+
+    self.adapter.seed_response('getBalances', {
+      'balances': [42, 29],
+    })
+
     # To keep the unit test nice and speedy, we will mock the address
     # generator.  We already have plenty of unit tests for that
     # functionality, so we can get away with mocking it here.
@@ -686,12 +691,9 @@ def test_no_stop_threshold_zero(self):
     """
     No ``stop`` provided, ``threshold`` is 0.
     """
-    # Note that the first address has a zero balance.
-    self.adapter.seed_response('getBalances', {
-      'balances': [0, 1],
-    })
 
-    # ``getInputs`` uses ``findTransactions`` to identify unused
+    # ``getInputs`` uses ``findTransactions`` and
+    # ``wereAddressesSpentFrom`` to identify unused addresses.
     # addresses.
     # noinspection SpellCheckingInspection
     self.adapter.seed_response('findTransactions', {
@@ -717,6 +719,15 @@ def test_no_stop_threshold_zero(self):
       'hashes': [],
     })
 
+    self.adapter.seed_response('wereAddressesSpentFrom', {
+      'states': [False],
+    })
+
+    # Note that the first address has a zero balance.
+    self.adapter.seed_response('getBalances', {
+      'balances': [0, 1],
+    })
+
     # To keep the unit test nice and speedy, we will mock the address
     # generator.  We already have plenty of unit tests for that
     # functionality, so we can get away with mocking it here.
@@ -750,12 +761,9 @@ def test_no_stop_no_threshold(self):
     """
     No ``stop`` provided, no ``threshold``.
     """
-    self.adapter.seed_response('getBalances', {
-      'balances': [42, 29],
-    })
 
-    # ``getInputs`` uses ``findTransactions`` to identify unused
-    # addresses.
+    # ``getInputs`` uses ``findTransactions`` and
+    # ``wereAddressesSpentFrom`` to identify unused addresses.
     # noinspection SpellCheckingInspection
     self.adapter.seed_response('findTransactions', {
       'hashes': [
@@ -780,6 +788,14 @@ def test_no_stop_no_threshold(self):
       'hashes': [],
     })
 
+    self.adapter.seed_response('wereAddressesSpentFrom', {
+      'states': [False],
+    })
+
+    self.adapter.seed_response('getBalances', {
+      'balances': [42, 29],
+    })
+
     # To keep the unit test nice and speedy, we will mock the address
     # generator.  We already have plenty of unit tests for that
     # functionality, so we can get away with mocking it here.
@@ -818,12 +834,9 @@ def test_start(self):
     """
     Using ``start`` to offset the key range.
     """
-    self.adapter.seed_response('getBalances', {
-      'balances': [86],
-    })
 
-    # ``getInputs`` uses ``findTransactions`` to identify unused
-    # addresses.
+    # ``getInputs`` uses ``findTransactions`` and
+    # ``wereAddressesSpentFrom`` to identify unused addresses.
     # noinspection SpellCheckingInspection
     self.adapter.seed_response('findTransactions', {
       'hashes': [
@@ -838,6 +851,14 @@ def test_start(self):
       'hashes': [],
     })
 
+    self.adapter.seed_response('wereAddressesSpentFrom', {
+      'states': [False],
+    })
+
+    self.adapter.seed_response('getBalances', {
+      'balances': [86],
+    })
+
     # To keep the unit test nice and speedy, we will mock the address
     # generator.  We already have plenty of unit tests for that
     # functionality, so we can get away with mocking it here.
@@ -926,11 +947,8 @@ def test_security_level_1_no_stop(self):
     seed = Seed.random()
     address = AddressGenerator(seed, security_level=1).get_addresses(0)[0]
 
-    self.adapter.seed_response('getBalances', {
-      'balances': [86],
-    })
-    # ``getInputs`` uses ``findTransactions`` to identify unused
-    # addresses.
+    # ``getInputs`` uses ``findTransactions`` and
+    # ``wereAddressesSpentFrom`` to identify unused addresses.
     # noinspection SpellCheckingInspection
     self.adapter.seed_response('findTransactions', {
       'hashes': [
@@ -944,6 +962,14 @@ def test_security_level_1_no_stop(self):
       'hashes': [],
     })
 
+    self.adapter.seed_response('wereAddressesSpentFrom', {
+      'states': [False],
+    })
+
+    self.adapter.seed_response('getBalances', {
+      'balances': [86],
+    })
+
     response = GetInputsCommand(self.adapter)(
       seed=seed,
       securityLevel=1,