diff --git a/.coveragerc b/.coveragerc index f27b66a8..4d68b106 100644 --- a/.coveragerc +++ b/.coveragerc @@ -10,6 +10,7 @@ omit = test/tumbler-test.py test/test_tumbler.py test/test_donations.py + test/ygrunner.py [report] # Regexes for lines to exclude from consideration diff --git a/cmttools/add-utxo.py b/cmttools/add-utxo.py index 108bc664..30e119c5 100644 --- a/cmttools/add-utxo.py +++ b/cmttools/add-utxo.py @@ -19,7 +19,7 @@ from optparse import OptionParser import bitcoin as btc from joinmarket import load_program_config, jm_single, get_p2pk_vbyte -from joinmarket import Wallet +from joinmarket import Wallet, sync_wallet from commitment_utils import get_utxo_info, validate_utxo_data, quit def add_external_commitments(utxo_datas): @@ -144,6 +144,12 @@ def main(): help='only validate the provided utxos (file or command line), not add', default=False ) + parser.add_option('--fast', + action='store_true', + dest='fastsync', + default=False, + help=('choose to do fast wallet sync, only for Core and ' + 'only for previously synced wallet')) (options, args) = parser.parse_args() load_program_config() utxo_data = [] @@ -172,7 +178,7 @@ def main(): options.maxmixdepth, options.gaplimit) os.chdir(os.path.join(os.getcwd(), 'cmttools')) - jm_single().bc_interface.sync_wallet(wallet) + sync_wallet(wallet, fast=options.fastsync) unsp = {} for u, av in wallet.unspent.iteritems(): addr = av['address'] diff --git a/joinmarket/__init__.py b/joinmarket/__init__.py index 23141091..23d92047 100644 --- a/joinmarket/__init__.py +++ b/joinmarket/__init__.py @@ -20,7 +20,7 @@ from .configure import load_program_config, jm_single, get_p2pk_vbyte, \ get_network, jm_single, get_network, validate_address, get_irc_mchannels, \ check_utxo_blacklist -from .blockchaininterface import BlockrInterface, BlockchainInterface +from .blockchaininterface import BlockrInterface, BlockchainInterface, sync_wallet from .yieldgenerator import YieldGenerator, ygmain # Set default logging handler to avoid "No handler found" warnings. diff --git a/joinmarket/blockchaininterface.py b/joinmarket/blockchaininterface.py index a0d61d3d..3ad87f11 100644 --- a/joinmarket/blockchaininterface.py +++ b/joinmarket/blockchaininterface.py @@ -67,6 +67,16 @@ def is_index_ahead_of_cache(wallet, mix_depth, forchange): return wallet.index[mix_depth][forchange] >= wallet.index_cache[mix_depth][ forchange] +def sync_wallet(wallet, fast=False): + """Wrapper function to choose fast syncing where it's + both possible and requested. + """ + if fast and ( + isinstance(jm_single().bc_interface, BitcoinCoreInterface) or isinstance( + jm_single().bc_interface, RegtestBitcoinCoreInterface)): + jm_single().bc_interface.sync_wallet(wallet, fast=True) + else: + jm_single().bc_interface.sync_wallet(wallet) class BlockchainInterface(object): __metaclass__ = abc.ABCMeta @@ -553,7 +563,7 @@ class BitcoinCoreInterface(BlockchainInterface): def __init__(self, jsonRpc, network): super(BitcoinCoreInterface, self).__init__() self.jsonRpc = jsonRpc - + self.fast_sync_called = False blockchainInfo = self.jsonRpc.call("getblockchaininfo", []) actualNet = blockchainInfo['chain'] @@ -589,12 +599,79 @@ def add_watchonly_addresses(self, addr_list, wallet_name): print(' otherwise just restart this joinmarket script') sys.exit(0) + def sync_wallet(self, wallet, fast=False): + #trigger fast sync if the index_cache is available + #(and not specifically disabled). + if fast and wallet.index_cache != [[0,0]] * wallet.max_mix_depth: + self.sync_wallet_fast(wallet) + self.fast_sync_called = True + return + super(BitcoinCoreInterface, self).sync_wallet(wallet) + self.fast_sync_called = False + + def sync_wallet_fast(self, wallet): + """Exploits the fact that given an index_cache, + all addresses necessary should be imported, so we + can just list all used addresses to find the right + index values. + """ + self.get_address_usages(wallet) + self.sync_unspent(wallet) + + def get_address_usages(self, wallet): + """Use rpc `listaddressgroupings` to locate all used + addresses in the account (whether spent or unspent outputs). + This will not result in a full sync if working with a new + Bitcoin Core instance, in which case "fast" should have been + specifically disabled by the user. + """ + from joinmarket.wallet import BitcoinCoreWallet + if isinstance(wallet, BitcoinCoreWallet): + return + wallet_name = self.get_wallet_name(wallet) + agd = self.rpc('listaddressgroupings', []) + #flatten all groups into a single list; then, remove duplicates + fagd = [tuple(item) for sublist in agd for item in sublist] + #"deduplicated flattened address grouping data" = dfagd + dfagd = list(set(fagd)) + #for lookup, want dict of form {"address": amount} + used_address_dict = {} + for addr_info in dfagd: + if len(addr_info) < 3 or addr_info[2] != wallet_name: + continue + used_address_dict[addr_info[0]] = (addr_info[1], addr_info[2]) + + log.debug("Fast sync in progress. Got this many used addresses: " + str( + len(used_address_dict))) + #Need to have wallet.index point to the last used address + #and fill addr_cache. + #For each branch: + #If index value is present, collect all addresses up to index+gap limit + #For each address in that list, mark used if seen in used_address_dict + used_indices = {} + for md in range(wallet.max_mix_depth): + used_indices[md] = {} + for fc in [0, 1]: + used_indices[md][fc] = [] + for i in range(wallet.index_cache[md][fc]+wallet.gaplimit): + if wallet.get_addr(md, fc, i) in used_address_dict.keys(): + used_indices[md][fc].append(i) + wallet.addr_cache[wallet.get_addr(md, fc, i)] = (md, fc, i) + if len(used_indices[md][fc]): + wallet.index[md][fc] = used_indices[md][fc][-1] + else: + wallet.index[md][fc] = 0 + if not is_index_ahead_of_cache(wallet, md, fc): + wallet.index[md][fc] = wallet.index_cache[md][fc] + self.wallet_synced = True + + def sync_addresses(self, wallet): from joinmarket.wallet import BitcoinCoreWallet if isinstance(wallet, BitcoinCoreWallet): return - log.debug('requesting wallet history') + log.debug('requesting detailed wallet history') wallet_name = self.get_wallet_name(wallet) #TODO It is worth considering making this user configurable: addr_req_count = 20 diff --git a/joinmarket/yieldgenerator.py b/joinmarket/yieldgenerator.py index cca4d002..222a0209 100644 --- a/joinmarket/yieldgenerator.py +++ b/joinmarket/yieldgenerator.py @@ -11,7 +11,7 @@ from joinmarket import BlockrInterface from joinmarket import jm_single, get_network, load_program_config from joinmarket import get_log, calc_cj_fee, debug_dump_object -from joinmarket import Wallet +from joinmarket import Wallet, sync_wallet from joinmarket import get_irc_mchannels log = get_log() @@ -102,6 +102,12 @@ def ygmain(ygclass, txfee=1000, cjfee_a=200, cjfee_r=0.002, ordertype='reloffer' parser.add_option('-g', '--gap-limit', action='store', type="int", dest='gaplimit', default=6, help='gap limit for wallet, default=6') + parser.add_option('--fast', + action='store_true', + dest='fastsync', + default=False, + help=('choose to do fast wallet sync, only for Core and ' + 'only for previously synced wallet')) (options, args) = parser.parse_args() if len(args) < 1: parser.error('Needs a wallet') @@ -142,7 +148,7 @@ def ygmain(ygclass, txfee=1000, cjfee_a=200, cjfee_r=0.002, ordertype='reloffer' return wallet = Wallet(seed, max_mix_depth=mix_levels, gaplimit=gaplimit) - jm_single().bc_interface.sync_wallet(wallet) + sync_wallet(wallet, fast=options.fastsync) mcs = [IRCMessageChannel(c, realname='btcint=' + jm_single().config.get( "BLOCKCHAIN", "blockchain_source"), diff --git a/patientsendpayment.py b/patientsendpayment.py index a7f09ba4..f1ac5f8a 100644 --- a/patientsendpayment.py +++ b/patientsendpayment.py @@ -10,7 +10,7 @@ from joinmarket import Maker, Taker, load_program_config, IRCMessageChannel from joinmarket import validate_address, jm_single from joinmarket import get_log, choose_orders, weighted_order_choose, \ - debug_dump_object + debug_dump_object, sync_wallet from joinmarket import Wallet log = get_log() @@ -194,6 +194,12 @@ def main(): help= 'Use the Bitcoin Core wallet through json rpc, instead of the internal joinmarket ' + 'wallet. Requires blockchain_source=json-rpc') + parser.add_option('--fast', + action='store_true', + dest='fastsync', + default=False, + help=('choose to do fast wallet sync, only for Core and ' + 'only for previously synced wallet')) (options, args) = parser.parse_args() if len(args) < 3: @@ -221,7 +227,7 @@ def main(): print 'not implemented yet' sys.exit(0) # wallet = BitcoinCoreWallet(fromaccount=wallet_name) - jm_single().bc_interface.sync_wallet(wallet) + sync_wallet(wallet, fast=options.fastsync) available_balance = wallet.get_balance_by_mixdepth()[options.mixdepth] if available_balance < amount: diff --git a/sendpayment.py b/sendpayment.py index 497b6762..7e9dce81 100644 --- a/sendpayment.py +++ b/sendpayment.py @@ -15,7 +15,7 @@ from joinmarket import validate_address, jm_single from joinmarket import get_log, choose_sweep_orders, choose_orders, \ pick_order, cheapest_order_choose, weighted_order_choose, debug_dump_object -from joinmarket import Wallet, BitcoinCoreWallet +from joinmarket import Wallet, BitcoinCoreWallet, sync_wallet from joinmarket.wallet import estimate_tx_fee log = get_log() @@ -276,6 +276,12 @@ def main(): help=('Use the Bitcoin Core wallet through json rpc, instead ' 'of the internal joinmarket wallet. Requires ' 'blockchain_source=json-rpc')) + parser.add_option('--fast', + action='store_true', + dest='fastsync', + default=False, + help=('choose to do fast wallet sync, only for Core and ' + 'only for previously synced wallet')) (options, args) = parser.parse_args() if len(args) < 3: @@ -316,7 +322,7 @@ def main(): wallet = Wallet(wallet_name, options.amtmixdepths, options.gaplimit) else: wallet = BitcoinCoreWallet(fromaccount=wallet_name) - jm_single().bc_interface.sync_wallet(wallet) + sync_wallet(wallet, fast=options.fastsync) mcs = [IRCMessageChannel(c) for c in get_irc_mchannels()] mcc = MessageChannelCollection(mcs) diff --git a/test/test_blockr.py b/test/test_blockr.py index 41404f26..a9b3efd8 100644 --- a/test/test_blockr.py +++ b/test/test_blockr.py @@ -11,7 +11,7 @@ import bitcoin as btc import pytest -from joinmarket import load_program_config, jm_single +from joinmarket import load_program_config, jm_single, sync_wallet from joinmarket.blockchaininterface import BlockrInterface from joinmarket import get_p2pk_vbyte, get_log, Wallet @@ -62,7 +62,7 @@ def test_blockr_estimate_fee(setup_blockr): def test_blockr_sync(setup_blockr, net, seed, gaplimit, showprivkey, method): jm_single().config.set("BLOCKCHAIN", "network", net) wallet = Wallet(seed, max_mix_depth = 5) - jm_single().bc_interface.sync_wallet(wallet) + sync_wallet(wallet) #copy pasted from wallet-tool; some boiled down form of #this should really be in wallet.py in the joinmarket module. diff --git a/test/test_donations.py b/test/test_donations.py index 95543e45..c827df90 100644 --- a/test/test_donations.py +++ b/test/test_donations.py @@ -10,7 +10,7 @@ from commontest import make_wallets from joinmarket import load_program_config, get_p2pk_vbyte, get_log, jm_single -from joinmarket import get_irc_mchannels, Taker +from joinmarket import get_irc_mchannels, Taker, sync_wallet from joinmarket.configure import donation_address log = get_log() @@ -24,7 +24,7 @@ def test_donation_address(setup_donations, amount): wallets = make_wallets(1, wallet_structures=[[1,1,1,0,0]], mean_amt=0.5) wallet = wallets[0]['wallet'] - jm_single().bc_interface.sync_wallet(wallet) + sync_wallet(wallet) #make a rdp from a simple privkey rdp_priv = "\x01"*32 reusable_donation_pubkey = binascii.hexlify(secp256k1.PrivateKey( diff --git a/test/test_podle.py b/test/test_podle.py index 37ffa4c5..0a359cea 100644 --- a/test/test_podle.py +++ b/test/test_podle.py @@ -17,7 +17,7 @@ from joinmarket import Taker, load_program_config, IRCMessageChannel from joinmarket import validate_address, jm_single, get_irc_mchannels from joinmarket import get_p2pk_vbyte, MessageChannelCollection -from joinmarket import get_log, choose_sweep_orders, choose_orders, \ +from joinmarket import get_log, choose_sweep_orders, choose_orders, sync_wallet, \ pick_order, cheapest_order_choose, weighted_order_choose, debug_dump_object import joinmarket.irc import sendpayment @@ -145,7 +145,7 @@ def test_failed_sendpayment(setup_podle, num_ygs, wallet_structures, mean_amt, log.debug('starting sendpayment') - jm_single().bc_interface.sync_wallet(wallet) + sync_wallet(wallet) #Trigger PING LAG sending artificially joinmarket.irc.PING_INTERVAL = 3 @@ -241,7 +241,7 @@ def test_external_commitment_used(setup_podle): log.debug('starting sendpayment') - jm_single().bc_interface.sync_wallet(wallet) + sync_wallet(wallet) #Trigger PING LAG sending artificially joinmarket.irc.PING_INTERVAL = 3 @@ -347,7 +347,7 @@ def test_tx_commitments_used(setup_podle, consume_tx, age_required, cmt_age): log.debug('starting sendpayment') - jm_single().bc_interface.sync_wallet(wallet) + sync_wallet(wallet) log.debug("Here is the whole wallet: \n" + str(wallet.unspent)) #Trigger PING LAG sending artificially joinmarket.irc.PING_INTERVAL = 3 diff --git a/test/test_regtest.py b/test/test_regtest.py index 2af34950..641b3baa 100644 --- a/test/test_regtest.py +++ b/test/test_regtest.py @@ -10,7 +10,7 @@ import pytest import time from joinmarket import (Taker, load_program_config, IRCMessageChannel, - BitcoinCoreWallet) + BitcoinCoreWallet, sync_wallet) from joinmarket import validate_address, jm_single, get_irc_mchannels from joinmarket import get_p2pk_vbyte, MessageChannelCollection from joinmarket import get_log, choose_sweep_orders, choose_orders, \ @@ -102,7 +102,7 @@ def test_sendpayment(setup_regtest, num_ygs, wallet_structures, mean_amt, log.debug('starting sendpayment') - jm_single().bc_interface.sync_wallet(wallet) + sync_wallet(wallet) #Trigger PING LAG sending artificially joinmarket.irc.PING_INTERVAL = 3 diff --git a/test/test_tumbler.py b/test/test_tumbler.py index 9f26cc5c..0daf878e 100644 --- a/test/test_tumbler.py +++ b/test/test_tumbler.py @@ -11,7 +11,7 @@ from joinmarket import Taker, load_program_config, IRCMessageChannel from joinmarket import validate_address, jm_single, MessageChannelCollection from joinmarket import get_p2pk_vbyte, get_irc_mchannels -from joinmarket import get_log, choose_sweep_orders, choose_orders, \ +from joinmarket import get_log, choose_sweep_orders, choose_orders, sync_wallet, \ pick_order, cheapest_order_choose, weighted_order_choose, debug_dump_object import json import tumbler @@ -147,7 +147,7 @@ def test_tumbler(setup_tumbler, num_ygs, wallet_structures, mean_amt, sdev_amt, log.debug('starting tumbler') - jm_single().bc_interface.sync_wallet(wallet) + sync_wallet(wallet) jm_single().bc_interface.pushtx_failure_prob = 0.4 mcs = [IRCMessageChannel(c) for c in get_irc_mchannels()] mcc = MessageChannelCollection(mcs) diff --git a/test/test_tx_creation.py b/test/test_tx_creation.py index 0e06a4ae..9f153659 100644 --- a/test/test_tx_creation.py +++ b/test/test_tx_creation.py @@ -15,7 +15,7 @@ import bitcoin as btc import pytest -from joinmarket import load_program_config, jm_single +from joinmarket import load_program_config, jm_single, sync_wallet from joinmarket import get_p2pk_vbyte, get_log, Wallet from joinmarket.support import chunks, select_gradual, \ select_greedy, select_greediest @@ -41,7 +41,7 @@ def test_create_p2sh_output_tx(setup_tx_creation, nw, wallet_structures, mean_amt, sdev_amt, amount, pubs, k): wallets = make_wallets(nw, wallet_structures, mean_amt, sdev_amt) for w in wallets.values(): - jm_single().bc_interface.sync_wallet(w['wallet']) + sync_wallet(w['wallet']) for k, w in enumerate(wallets.values()): wallet = w['wallet'] ins_full = wallet.select_utxos(0, amount) @@ -65,7 +65,7 @@ def test_absurd_fees(setup_tx_creation): jm_single().bc_interface.absurd_fees = True #pay into it wallet = make_wallets(1, [[2, 0, 0, 0, 1]], 3)[0]['wallet'] - jm_single().bc_interface.sync_wallet(wallet) + sync_wallet(wallet) amount = 350000000 ins_full = wallet.select_utxos(0, amount) with pytest.raises(ValueError) as e_info: @@ -76,7 +76,7 @@ def test_create_sighash_txs(setup_tx_creation): for sighash in [btc.SIGHASH_ANYONECANPAY + btc.SIGHASH_SINGLE, btc.SIGHASH_NONE, btc.SIGHASH_SINGLE]: wallet = make_wallets(1, [[2, 0, 0, 0, 1]], 3)[0]['wallet'] - jm_single().bc_interface.sync_wallet(wallet) + sync_wallet(wallet) amount = 350000000 ins_full = wallet.select_utxos(0, amount) print "using hashcode: " + str(sighash) @@ -105,7 +105,7 @@ def test_spend_p2sh_utxos(setup_tx_creation): msig_addr = btc.scriptaddr(script, magicbyte=196) #pay into it wallet = make_wallets(1, [[2, 0, 0, 0, 1]], 3)[0]['wallet'] - jm_single().bc_interface.sync_wallet(wallet) + sync_wallet(wallet) amount = 350000000 ins_full = wallet.select_utxos(0, amount) txid = make_sign_and_push(ins_full, wallet, amount, output_addr=msig_addr) diff --git a/test/test_tx_notify.py b/test/test_tx_notify.py index 30c7c954..79e386a9 100644 --- a/test/test_tx_notify.py +++ b/test/test_tx_notify.py @@ -9,7 +9,7 @@ import bitcoin as btc import pytest from joinmarket import load_program_config, jm_single -from joinmarket import get_log, Wallet +from joinmarket import get_log, Wallet, sync_wallet log = get_log() @@ -71,7 +71,7 @@ def make_tx_add_notify(): amount = 250000000 txfee = 10000 wallet = wallet_dict['wallet'] - jm_single().bc_interface.sync_wallet(wallet) + sync_wallet(wallet) inputs = wallet.select_utxos(0, amount) ins = inputs.keys() input_value = sum([i['value'] for i in inputs.values()]) diff --git a/test/test_wallets.py b/test/test_wallets.py index 0045bedb..94561355 100644 --- a/test/test_wallets.py +++ b/test/test_wallets.py @@ -12,10 +12,11 @@ import unittest from decimal import Decimal from commontest import local_command, interact, make_wallets, make_sign_and_push +import json import bitcoin as btc import pytest -from joinmarket import load_program_config, jm_single +from joinmarket import load_program_config, jm_single, sync_wallet from joinmarket import get_p2pk_vbyte, get_log, Wallet from joinmarket.support import chunks, select_gradual, \ select_greedy, select_greediest @@ -36,42 +37,6 @@ def do_tx(wallet, amount): time.sleep(2) #blocks jm_single().bc_interface.sync_unspent(wallet) -""" -@pytest.mark.parametrize( - "num_txs, gap_count, gap_limit, wallet_structure, amount, wallet_file, password", - [ - (3, 450, 461, [11,3,4,5,6], 150000000, 'test_import_wallet.json', 'import-pwd' - ), - ]) -def test_wallet_gap_sync(setup_wallets, num_txs, gap_count, gap_limit, - wallet_structure, amount, wallet_file, password): - #Starting with a nonexistent index_cache, try syncing with a large - #gap limit - setup_import(mainnet=False) - wallet = make_wallets(1,[wallet_structure], - fixed_seeds=[wallet_file], - test_wallet=True, passwords=[password])[0]['wallet'] - wallet.gaplimit = gap_limit - #Artificially insert coins at position (0, wallet_structures[0] + gap_count) - dest = wallet.get_addr(0, 0, wallet_structure[0] + gap_count) - btcamt = amount/(1e8) - jm_single().bc_interface.grab_coins(dest, amt=float(Decimal(btcamt).quantize(Decimal(10)**-8))) - time.sleep(2) - sync_count = 0 - jm_single().bc_interface.wallet_synced = False - while not jm_single().bc_interface.wallet_synced: - wallet.index = [] - for i in range(5): - wallet.index.append([0, 0]) - jm_single().bc_interface.sync_wallet(wallet) - sync_count += 1 - #avoid infinite loop - assert sync_count < 10 - log.debug("Tried " + str(sync_count) + " times") - - assert jm_single().bc_interface.wallet_synced -""" - @pytest.mark.parametrize( "num_txs, fake_count, wallet_structure, amount, wallet_file, password", [ @@ -90,7 +55,7 @@ def test_wallet_gap_sync(setup_wallets, num_txs, gap_count, gap_limit, #(25, 30, [30,20,1,1,1], 50000000, 'test_import_wallet.json', 'import-pwd' # ), ]) -def test_wallet_sync(setup_wallets, num_txs, fake_count, +def test_wallet_sync_with_fast(setup_wallets, num_txs, fake_count, wallet_structure, amount, wallet_file, password): setup_import(mainnet=False) wallet = make_wallets(1,[wallet_structure], @@ -99,13 +64,14 @@ def test_wallet_sync(setup_wallets, num_txs, fake_count, sync_count = 0 jm_single().bc_interface.wallet_synced = False while not jm_single().bc_interface.wallet_synced: - jm_single().bc_interface.sync_wallet(wallet) + sync_wallet(wallet) sync_count += 1 #avoid infinite loop assert sync_count < 10 log.debug("Tried " + str(sync_count) + " times") assert jm_single().bc_interface.wallet_synced + assert not jm_single().bc_interface.fast_sync_called #do some transactions with the wallet, then close, then resync for i in range(num_txs): do_tx(wallet, amount) @@ -152,13 +118,13 @@ def test_wallet_sync(setup_wallets, num_txs, fake_count, #script over and over again)? sync_count += 1 log.debug("TRYING SYNC NUMBER: " + str(sync_count)) - jm_single().bc_interface.sync_wallet(wallet) + sync_wallet(wallet, fast=True) + assert jm_single().bc_interface.fast_sync_called #avoid infinite loop on failure. assert sync_count < 10 - #Wallet should recognize index_cache on sync, so should not need to - #run sync process more than twice (twice if cache bump has moved us - #past the first round of imports). - assert sync_count <= 2 + #Wallet should recognize index_cache on fast sync, so should not need to + #run sync process more than once. + assert sync_count == 1 #validate the wallet index values after sync for i, ws in enumerate(wallet_structure): assert wallet.index[i][0] == ws #spends into external only @@ -181,7 +147,7 @@ def test_wallet_sync(setup_wallets, num_txs, fake_count, # [(12,3),(100,99),(7, 40), (200, 201), (10,0)] # ), ([1,3,0,2,9], 'test_import_wallet.json', 'import-pwd', - [(0,7),(100,99),(0, 0), (200, 201), (21,41)] + [(1,7),(100,99),(0, 0), (200, 201), (21,41)] ), ]) def test_wallet_sync_from_scratch(setup_wallets, wallet_structure, @@ -202,13 +168,14 @@ def test_wallet_sync_from_scratch(setup_wallets, wallet_structure, wallet.index = [] for i in range(5): wallet.index.append([0, 0]) - jm_single().bc_interface.sync_wallet(wallet) + #will call with fast=False but index_cache exists; should use slow-sync + sync_wallet(wallet) sync_count += 1 #avoid infinite loop assert sync_count < 10 log.debug("Tried " + str(sync_count) + " times") #after #586 we expect to ALWAYS succeed within 2 rounds - assert sync_count == 2 + assert sync_count <= 2 #for each external branch, the new index may be higher than #the original index_cache if there was a higher used address expected_wallet_index = [] @@ -219,6 +186,8 @@ def test_wallet_sync_from_scratch(setup_wallets, wallet_structure, expected_wallet_index.append([wallet.index_cache[i][0], wallet.index_cache[i][1]]) assert wallet.index == expected_wallet_index + log.debug("This is wallet unspent: ") + log.debug(json.dumps(wallet.unspent, indent=4)) @pytest.mark.parametrize( @@ -287,7 +256,7 @@ def test_utxo_selection(setup_wallets, nw, wallet_structures, mean_amt, wallets = make_wallets(nw, wallet_structures, mean_amt, sdev_amt) for w in wallets.values(): jm_single().bc_interface.wallet_synced = False - jm_single().bc_interface.sync_wallet(w['wallet']) + sync_wallet(w['wallet']) for k, w in enumerate(wallets.values()): for algo in [select_gradual, select_greedy, select_greediest, None]: wallet = w['wallet'] diff --git a/test/ygrunner.py b/test/ygrunner.py index c57dc4ab..308fb5cc 100644 --- a/test/ygrunner.py +++ b/test/ygrunner.py @@ -20,7 +20,7 @@ import pytest import sys import time -from joinmarket import load_program_config, jm_single +from joinmarket import load_program_config, jm_single, sync_wallet #for running bots as subprocesses python_cmd = 'python2' @@ -46,7 +46,7 @@ def test_start_ygs(setup_ygrunner, num_ygs, wallet_structures, mean_amt): wallet = wallets[num_ygs]['wallet'] print "Seed : " + wallets[num_ygs]['seed'] #useful to see the utxos on screen sometimes - jm_single().bc_interface.sync_wallet(wallet) + sync_wallet(wallet) print wallet.unspent yigen_procs = [] diff --git a/tumbler.py b/tumbler.py index 3b94f432..0654c3e4 100644 --- a/tumbler.py +++ b/tumbler.py @@ -13,7 +13,7 @@ from joinmarket import jm_single, Taker, load_program_config, \ IRCMessageChannel, MessageChannelCollection -from joinmarket import validate_address +from joinmarket import validate_address, sync_wallet from joinmarket import get_log, rand_norm_array, rand_pow_array, \ rand_exp_array, choose_orders, weighted_order_choose, choose_sweep_orders, \ debug_dump_object, get_irc_mchannels @@ -545,6 +545,12 @@ def main(): default=9, help= 'maximum amount of times to re-create a transaction before giving up, default 9') + parser.add_option('--fast', + action='store_true', + dest='fastsync', + default=False, + help=('choose to do fast wallet sync, only for Core and ' + 'only for previously synced wallet')) (options, args) = parser.parse_args() options = vars(options) @@ -631,7 +637,7 @@ def main(): # python tumbler.py -N 2 1 -c 3 0.001 -l 0.1 -M 3 -a 0 wallet_file 1xxx 1yyy wallet = Wallet(wallet_file, max_mix_depth=options['mixdepthsrc'] + options['mixdepthcount']) - jm_single().bc_interface.sync_wallet(wallet) + sync_wallet(wallet, fast=options.fastsync) jm_single().wait_for_commitments = 1 mcs = [IRCMessageChannel(c) for c in get_irc_mchannels()] mcc = MessageChannelCollection(mcs) diff --git a/wallet-tool.py b/wallet-tool.py index 03d50c68..d2617780 100644 --- a/wallet-tool.py +++ b/wallet-tool.py @@ -10,7 +10,7 @@ from joinmarket import load_program_config, get_network, Wallet, encryptData, \ get_p2pk_vbyte, jm_single, mn_decode, mn_encode, BitcoinCoreInterface, \ - JsonRpcError + JsonRpcError, sync_wallet import bitcoin as btc @@ -63,6 +63,12 @@ dest='csv', default=False, help=('When using the history method, output as csv')) +parser.add_option('--fast', + action='store_true', + dest='fastsync', + default=False, + help=('choose to do fast wallet sync, only for Core and ' + 'only for previously synced wallet')) (options, args) = parser.parse_args() # if the index_cache stored in wallet.json is longer than the default @@ -104,7 +110,8 @@ # unconfirmed balance is included in the wallet display by default if 'listunspent_args' not in jm_single().config.options('POLICY'): jm_single().config.set('POLICY','listunspent_args', '[0]') - jm_single().bc_interface.sync_wallet(wallet) + + sync_wallet(wallet, fast=options.fastsync) if method == 'showutxos': unsp = {}