diff --git a/docs/addresses.rst b/docs/addresses.rst index c3403ae2..01fa0fc8 100644 --- a/docs/addresses.rst +++ b/docs/addresses.rst @@ -17,17 +17,11 @@ any other financial service. These performance issues will be fixed in a future version of the library; please bear with us! - In the meantime, if you are using Python 3, you can install a C extension + In the meantime, you can install a C extension that boosts PyOTA's performance significantly (speedups of 60x are common!). To install the extension, run ``pip install pyota[ccurl]``. - **Important:** The extension is not yet compatible with Python 2. - - If you are familiar with Python 2's C API, we'd love to hear from you! - Check the `GitHub issue `_ - for more information. - PyOTA provides two methods for generating addresses: Using the API @@ -60,7 +54,9 @@ method, using the following parameters: (defaults to 1). - If ``None``, the API will generate addresses until it finds one that has not been used (has no transactions associated with it on the - Tangle). It will then return the unused address and discard the rest. + Tangle, and was not spent from). This makes the command safer to use after + a snapshot has been taken. It will then return the unused address and + discard the rest. - ``security_level: int``: Determines the security level of the generated addresses. See `Security Levels <#security-levels>`__ below. diff --git a/iota/api.py b/iota/api.py index 9b12bf79..5f8dda19 100644 --- a/iota/api.py +++ b/iota/api.py @@ -1217,8 +1217,9 @@ def get_new_addresses( inside a loop. If ``None``, this method will progressively generate - addresses and scan the Tangle until it finds one that has no - transactions referencing it. + addresses and scan the Tangle until it finds one that is unused. + This is if no transactions are referencing it and it was not spent + from before. :param int security_level: Number of iterations to use when generating new addresses. diff --git a/iota/commands/extended/get_account_data.py b/iota/commands/extended/get_account_data.py index 7235d10b..46dd8d1d 100644 --- a/iota/commands/extended/get_account_data.py +++ b/iota/commands/extended/get_account_data.py @@ -59,7 +59,7 @@ def _execute(self, request): my_hashes = ft_command(addresses=my_addresses).get('hashes') or [] account_balance = 0 - if my_hashes: + if my_addresses: # Load balances for the addresses that we generated. gb_response = ( GetBalancesCommand(self.adapter)(addresses=my_addresses) diff --git a/iota/commands/extended/get_new_addresses.py b/iota/commands/extended/get_new_addresses.py index fa9d0882..65f3c577 100644 --- a/iota/commands/extended/get_new_addresses.py +++ b/iota/commands/extended/get_new_addresses.py @@ -9,6 +9,8 @@ from iota import Address from iota.commands import FilterCommand, RequestFilter from iota.commands.core.find_transactions import FindTransactionsCommand +from iota.commands.core.were_addresses_spent_from import \ + WereAddressesSpentFromCommand from iota.crypto.addresses import AddressGenerator from iota.crypto.types import Seed from iota.filters import SecurityLevel, Trytes @@ -58,17 +60,23 @@ def _find_addresses(self, seed, index, count, security_level, checksum): generator = AddressGenerator(seed, security_level, checksum) if count is None: - # Connect to Tangle and find the first address without any - # transactions. + # Connect to Tangle and find the first unused address. for addy in generator.create_iterator(start=index): - # We use addy.address here because FindTransactions does + # We use addy.address here because the commands do # not work on an address with a checksum + response = WereAddressesSpentFromCommand(self.adapter)( + addresses=[addy.address], + ) + if response['states'][0]: + continue + response = FindTransactionsCommand(self.adapter)( addresses=[addy.address], ) + if response.get('hashes'): + continue - if not response.get('hashes'): - return [addy] + return [addy] return generator.get_addresses(start=index, count=count) diff --git a/iota/commands/extended/utils.py b/iota/commands/extended/utils.py index a8d3eb8e..2ab0afd2 100644 --- a/iota/commands/extended/utils.py +++ b/iota/commands/extended/utils.py @@ -9,6 +9,8 @@ from iota.adapter import BaseAdapter from iota.commands.core.find_transactions import FindTransactionsCommand from iota.commands.core.get_trytes import GetTrytesCommand +from iota.commands.core.were_addresses_spent_from import \ + WereAddressesSpentFromCommand from iota.commands.extended import FindTransactionObjectsCommand from iota.commands.extended.get_bundles import GetBundlesCommand from iota.commands.extended.get_latest_inclusion import \ @@ -25,15 +27,17 @@ def iter_used_addresses( ): # type: (...) -> Generator[Tuple[Address, List[TransactionHash]], None, None] """ - Scans the Tangle for used addresses. + Scans the Tangle for used addresses. A used address is an address that + was spent from or has a transaction. This is basically the opposite of invoking ``getNewAddresses`` with - ``stop=None``. + ``count=None``. """ if security_level is None: security_level = AddressGenerator.DEFAULT_SECURITY_LEVEL ft_command = FindTransactionsCommand(adapter) + wasf_command = WereAddressesSpentFromCommand(adapter) for addy in AddressGenerator(seed, security_level).create_iterator(start): ft_response = ft_command(addresses=[addy]) @@ -41,10 +45,15 @@ def iter_used_addresses( if ft_response['hashes']: yield addy, ft_response['hashes'] else: - break + wasf_response = wasf_command(addresses=[addy]) + if wasf_response['states'][0]: + yield addy, [] + else: + break - # Reset the command so that we can call it again. + # Reset the commands so that we can call them again. ft_command.reset() + wasf_command.reset() def get_bundles_from_transaction_hashes( diff --git a/test/commands/extended/get_account_data_test.py b/test/commands/extended/get_account_data_test.py index fd743f75..649ac39f 100644 --- a/test/commands/extended/get_account_data_test.py +++ b/test/commands/extended/get_account_data_test.py @@ -435,3 +435,28 @@ def test_no_transactions(self): 'bundles': [], }, ) + + def test_balance_is_found_for_address_without_transaction(self): + """ + If an address has a balance, no transactions and was spent from, the + balance should still be found and returned. + """ + with mock.patch( + 'iota.commands.extended.get_account_data.iter_used_addresses', + mock.Mock(return_value=[(self.addy1, [])]), + ): + self.adapter.seed_response('getBalances', { + 'balances': [42], + }) + + response = self.command(seed=Seed.random()) + + self.assertDictEqual( + response, + + { + 'addresses': [self.addy1], + 'balance': 42, + 'bundles': [], + }, + ) diff --git a/test/commands/extended/get_inputs_test.py b/test/commands/extended/get_inputs_test.py index de62d9f0..ee3594ab 100644 --- a/test/commands/extended/get_inputs_test.py +++ b/test/commands/extended/get_inputs_test.py @@ -590,12 +590,9 @@ def test_no_stop_threshold_met(self): """ No ``stop`` provided, balance meets ``threshold``. """ - self.adapter.seed_response('getBalances', { - 'balances': [42, 29], - }) - # ``getInputs`` uses ``findTransactions`` to identify unused - # addresses. + # ``getInputs`` uses ``findTransactions`` and + # ``wereAddressesSpentFrom`` to identify unused addresses. # noinspection SpellCheckingInspection self.adapter.seed_response('findTransactions', { 'hashes': [ @@ -620,6 +617,14 @@ def test_no_stop_threshold_met(self): 'hashes': [], }) + self.adapter.seed_response('wereAddressesSpentFrom', { + 'states': [False], + }) + + self.adapter.seed_response('getBalances', { + 'balances': [42, 29], + }) + # To keep the unit test nice and speedy, we will mock the address # generator. We already have plenty of unit tests for that # functionality, so we can get away with mocking it here. @@ -686,12 +691,9 @@ def test_no_stop_threshold_zero(self): """ No ``stop`` provided, ``threshold`` is 0. """ - # Note that the first address has a zero balance. - self.adapter.seed_response('getBalances', { - 'balances': [0, 1], - }) - # ``getInputs`` uses ``findTransactions`` to identify unused + # ``getInputs`` uses ``findTransactions`` and + # ``wereAddressesSpentFrom`` to identify unused addresses. # addresses. # noinspection SpellCheckingInspection self.adapter.seed_response('findTransactions', { @@ -717,6 +719,15 @@ def test_no_stop_threshold_zero(self): 'hashes': [], }) + self.adapter.seed_response('wereAddressesSpentFrom', { + 'states': [False], + }) + + # Note that the first address has a zero balance. + self.adapter.seed_response('getBalances', { + 'balances': [0, 1], + }) + # To keep the unit test nice and speedy, we will mock the address # generator. We already have plenty of unit tests for that # functionality, so we can get away with mocking it here. @@ -750,12 +761,9 @@ def test_no_stop_no_threshold(self): """ No ``stop`` provided, no ``threshold``. """ - self.adapter.seed_response('getBalances', { - 'balances': [42, 29], - }) - # ``getInputs`` uses ``findTransactions`` to identify unused - # addresses. + # ``getInputs`` uses ``findTransactions`` and + # ``wereAddressesSpentFrom`` to identify unused addresses. # noinspection SpellCheckingInspection self.adapter.seed_response('findTransactions', { 'hashes': [ @@ -780,6 +788,14 @@ def test_no_stop_no_threshold(self): 'hashes': [], }) + self.adapter.seed_response('wereAddressesSpentFrom', { + 'states': [False], + }) + + self.adapter.seed_response('getBalances', { + 'balances': [42, 29], + }) + # To keep the unit test nice and speedy, we will mock the address # generator. We already have plenty of unit tests for that # functionality, so we can get away with mocking it here. @@ -818,12 +834,9 @@ def test_start(self): """ Using ``start`` to offset the key range. """ - self.adapter.seed_response('getBalances', { - 'balances': [86], - }) - # ``getInputs`` uses ``findTransactions`` to identify unused - # addresses. + # ``getInputs`` uses ``findTransactions`` and + # ``wereAddressesSpentFrom`` to identify unused addresses. # noinspection SpellCheckingInspection self.adapter.seed_response('findTransactions', { 'hashes': [ @@ -838,6 +851,14 @@ def test_start(self): 'hashes': [], }) + self.adapter.seed_response('wereAddressesSpentFrom', { + 'states': [False], + }) + + self.adapter.seed_response('getBalances', { + 'balances': [86], + }) + # To keep the unit test nice and speedy, we will mock the address # generator. We already have plenty of unit tests for that # functionality, so we can get away with mocking it here. @@ -926,11 +947,8 @@ def test_security_level_1_no_stop(self): seed = Seed.random() address = AddressGenerator(seed, security_level=1).get_addresses(0)[0] - self.adapter.seed_response('getBalances', { - 'balances': [86], - }) - # ``getInputs`` uses ``findTransactions`` to identify unused - # addresses. + # ``getInputs`` uses ``findTransactions`` and + # ``wereAddressesSpentFrom`` to identify unused addresses. # noinspection SpellCheckingInspection self.adapter.seed_response('findTransactions', { 'hashes': [ @@ -944,6 +962,14 @@ def test_security_level_1_no_stop(self): 'hashes': [], }) + self.adapter.seed_response('wereAddressesSpentFrom', { + 'states': [False], + }) + + self.adapter.seed_response('getBalances', { + 'balances': [86], + }) + response = GetInputsCommand(self.adapter)( seed=seed, securityLevel=1, diff --git a/test/commands/extended/get_new_addresses_test.py b/test/commands/extended/get_new_addresses_test.py index 735aa90e..53730e78 100644 --- a/test/commands/extended/get_new_addresses_test.py +++ b/test/commands/extended/get_new_addresses_test.py @@ -423,20 +423,20 @@ def test_security_level(self): }, ) - def test_get_addresses_online(self): + def test_get_addresses_online_already_spent_from(self): """ - Generate address in online mode (filtering used addresses). + Generate address in online mode (filtering used addresses). Test if an + address that was already spent from will not be returned. """ - # Pretend that ``self.addy1`` has already been used, but not - # ``self.addy2``. - # noinspection SpellCheckingInspection - self.adapter.seed_response('findTransactions', { - 'duration': 18, + # Pretend that ``self.addy1`` has no transactions but already been + # spent from, but ``self.addy2`` is not used. - 'hashes': [ - 'TESTVALUE9DONTUSEINPRODUCTION99999ITQLQN' - 'LPPG9YNAARMKNKYQO9GSCSBIOTGMLJUFLZWSY9999', - ], + self.adapter.seed_response('wereAddressesSpentFrom', { + 'states': [True], + }) + + self.adapter.seed_response('wereAddressesSpentFrom', { + 'states': [False], }) self.adapter.seed_response('findTransactions', { @@ -461,14 +461,73 @@ def test_get_addresses_online(self): self.assertListEqual( self.adapter.requests, - # The command issued two `findTransactions` API requests: one for - # each address generated, until it found an unused address. + # The command issued a `wereAddressesSpentFrom` API request to + # check if the first address was used. Then it called `wereAddressesSpentFrom` + # and `findTransactions` to verify that the second address was + # indeed not used. [ { - 'command': 'findTransactions', + 'command': 'wereAddressesSpentFrom', 'addresses': [self.addy_1], }, + { + 'command': 'wereAddressesSpentFrom', + 'addresses': [self.addy_2], + }, + { + 'command': 'findTransactions', + 'addresses': [self.addy_2], + }, + ], + ) + + def test_get_addresses_online_has_transaction(self): + """ + Generate address in online mode (filtering used addresses). Test if an + address that has a transaction will not be returned. + """ + # Pretend that ``self.addy1`` has a transaction, but + # ``self.addy2`` is not used. + self.adapter.seed_response('wereAddressesSpentFrom', { + 'states': [False], + }) + # noinspection SpellCheckingInspection + self.adapter.seed_response('findTransactions', { + 'duration': 18, + 'hashes': [ + 'TESTVALUE9DONTUSEINPRODUCTION99999ITQLQN' + 'LPPG9YNAARMKNKYQO9GSCSBIOTGMLJUFLZWSY9999', + ], + }) + + self.adapter.seed_response('wereAddressesSpentFrom', { + 'states': [False], + }) + self.adapter.seed_response('findTransactions', { + 'hashes': [], + }) + + response = self.command(index=0, seed=self.seed) + + # The command determined that ``self.addy1`` was already used, so + # it skipped that one. + self.assertDictEqual(response, {'addresses': [self.addy_2]}) + self.assertListEqual( + self.adapter.requests, + [ + { + 'command': 'wereAddressesSpentFrom', + 'addresses': [self.addy_1], + }, + { + 'command': 'findTransactions', + 'addresses': [self.addy_1], + }, + { + 'command': 'wereAddressesSpentFrom', + 'addresses': [self.addy_2], + }, { 'command': 'findTransactions', 'addresses': [self.addy_2], diff --git a/test/commands/extended/get_transfers_test.py b/test/commands/extended/get_transfers_test.py index ff2f4f67..4b64d707 100644 --- a/test/commands/extended/get_transfers_test.py +++ b/test/commands/extended/get_transfers_test.py @@ -372,7 +372,7 @@ def create_generator(ag, start, step=1): }, ) - # The second address is unused. + # The second address is unused. It has no transactions and was not spent from. self.adapter.seed_response( 'findTransactions', @@ -381,6 +381,12 @@ def create_generator(ag, start, step=1): 'hashes': [], }, ) + self.adapter.seed_response( + 'wereAddressesSpentFrom', + { + 'states': [False], + }, + ) self.adapter.seed_response( 'getTrytes', @@ -461,6 +467,12 @@ def create_generator(ag, start, step=1): 'hashes': [], }, ) + self.adapter.seed_response( + 'wereAddressesSpentFrom', + { + 'states': [False], + }, + ) with mock.patch( 'iota.crypto.addresses.AddressGenerator.create_iterator', @@ -495,7 +507,7 @@ def create_generator(ag, start, step=1): }, ) - # The second address is unused. + # The second address is unused. It has no transactions and was not spent from. self.adapter.seed_response( 'findTransactions', @@ -505,6 +517,13 @@ def create_generator(ag, start, step=1): }, ) + self.adapter.seed_response( + 'wereAddressesSpentFrom', + { + 'states': [True], + }, + ) + self.adapter.seed_response( 'getTrytes', diff --git a/test/commands/extended/prepare_transfer_test.py b/test/commands/extended/prepare_transfer_test.py index 91f35446..35e0f9df 100644 --- a/test/commands/extended/prepare_transfer_test.py +++ b/test/commands/extended/prepare_transfer_test.py @@ -1307,11 +1307,15 @@ def mock_get_balances_execute(adapter, request): # testing for several security levels for security_level in SECURITY_LEVELS_TO_TEST: - # get_new_addresses uses `find_transactions` internaly. + # get_new_addresses uses `find_transactions` and + # `were_addresses_spent_from` internally. # The following means requested address is considered unused self.adapter.seed_response('findTransactions', { 'hashes': [], }) + self.adapter.seed_response('wereAddressesSpentFrom', { + 'states': [False], + }) self.command.reset() with mock.patch( @@ -1377,8 +1381,8 @@ def mock_get_balances_execute(adapter, request): # testing several security levels for security_level in SECURITY_LEVELS_TO_TEST: - # get_inputs use iter_used_addresses and findTransactions. - # until address without tx found + # get_inputs uses iter_used_addresses, findTransactions, + # and wereAddressesSpentFrom until an unused address is found. self.adapter.seed_response('findTransactions', { 'hashes': [ TransactionHash( @@ -1390,8 +1394,16 @@ def mock_get_balances_execute(adapter, request): self.adapter.seed_response('findTransactions', { 'hashes': [], }) + self.adapter.seed_response('wereAddressesSpentFrom', { + 'states': [False], + }) - # get_new_addresses uses `find_transactions` internaly. + # get_new_addresses uses `find_transactions`, `get_balances` and + # `were_addresses_spent_from` internally. + + self.adapter.seed_response('wereAddressesSpentFrom', { + 'states': [False], + }) self.adapter.seed_response('findTransactions', { 'hashes': [], })