add fast sync option for Core wallets

.coveragerc

@@ -10,6 +10,7 @@ omit =
  test/tumbler-test.py
  test/test_tumbler.py
  test/test_donations.py
+ test/ygrunner.py
 
 [report]
 # Regexes for lines to exclude from consideration

cmttools/add-utxo.py

@@ -19,7 +19,7 @@
 from optparse import OptionParser
 import bitcoin as btc
 from joinmarket import load_program_config, jm_single, get_p2pk_vbyte
-from joinmarket import Wallet
+from joinmarket import Wallet, sync_wallet
 from commitment_utils import get_utxo_info, validate_utxo_data, quit
 
 def add_external_commitments(utxo_datas):
@@ -144,6 +144,12 @@ def main():
         help='only validate the provided utxos (file or command line), not add',
         default=False
     )
+    parser.add_option('--fast',
+                      action='store_true',
+                      dest='fastsync',
+                      default=False,
+                      help=('choose to do fast wallet sync, only for Core and '
+                      'only for previously synced wallet'))
     (options, args) = parser.parse_args()
     load_program_config()
     utxo_data = []
@@ -172,7 +178,7 @@ def main():
                             options.maxmixdepth,
                             options.gaplimit)
         os.chdir(os.path.join(os.getcwd(), 'cmttools'))
-        jm_single().bc_interface.sync_wallet(wallet)
+        sync_wallet(wallet, fast=options.fastsync)
         unsp = {}
         for u, av in wallet.unspent.iteritems():
                     addr = av['address']

joinmarket/__init__.py

@@ -20,7 +20,7 @@
 from .configure import load_program_config, jm_single, get_p2pk_vbyte, \
     get_network, jm_single, get_network, validate_address, get_irc_mchannels, \
     check_utxo_blacklist
-from .blockchaininterface import BlockrInterface, BlockchainInterface
+from .blockchaininterface import BlockrInterface, BlockchainInterface, sync_wallet
 from .yieldgenerator import YieldGenerator, ygmain
 # Set default logging handler to avoid "No handler found" warnings.
 

joinmarket/blockchaininterface.py

@@ -67,6 +67,16 @@ def is_index_ahead_of_cache(wallet, mix_depth, forchange):
     return wallet.index[mix_depth][forchange] >= wallet.index_cache[mix_depth][
         forchange]
 
+def sync_wallet(wallet, fast=False):
+    """Wrapper function to choose fast syncing where it's
+    both possible and requested.
+    """
+    if fast and (
+        isinstance(jm_single().bc_interface, BitcoinCoreInterface) or isinstance(
+                jm_single().bc_interface, RegtestBitcoinCoreInterface)):
+        jm_single().bc_interface.sync_wallet(wallet, fast=True)
+    else:
+        jm_single().bc_interface.sync_wallet(wallet)
 
 class BlockchainInterface(object):
     __metaclass__ = abc.ABCMeta
@@ -553,7 +563,7 @@ class BitcoinCoreInterface(BlockchainInterface):
     def __init__(self, jsonRpc, network):
         super(BitcoinCoreInterface, self).__init__()
         self.jsonRpc = jsonRpc
-
+        self.fast_sync_called = False
         blockchainInfo = self.jsonRpc.call("getblockchaininfo", [])
         actualNet = blockchainInfo['chain']
 
@@ -589,12 +599,79 @@ def add_watchonly_addresses(self, addr_list, wallet_name):
             print(' otherwise just restart this joinmarket script')
             sys.exit(0)
 
+    def sync_wallet(self, wallet, fast=False):
+        #trigger fast sync if the index_cache is available
+        #(and not specifically disabled).
+        if fast and wallet.index_cache != [[0,0]] * wallet.max_mix_depth:
+            self.sync_wallet_fast(wallet)
+            self.fast_sync_called = True
+            return
+        super(BitcoinCoreInterface, self).sync_wallet(wallet)
+        self.fast_sync_called = False
+
+    def sync_wallet_fast(self, wallet):
+        """Exploits the fact that given an index_cache,
+        all addresses necessary should be imported, so we
+        can just list all used addresses to find the right
+        index values.
+        """
+        self.get_address_usages(wallet)
+        self.sync_unspent(wallet)
+
+    def get_address_usages(self, wallet):
+        """Use rpc `listaddressgroupings` to locate all used
+        addresses in the account (whether spent or unspent outputs).
+        This will not result in a full sync if working with a new
+        Bitcoin Core instance, in which case "fast" should have been
+        specifically disabled by the user.
+        """
+        from joinmarket.wallet import BitcoinCoreWallet
+        if isinstance(wallet, BitcoinCoreWallet):
+            return
+        wallet_name = self.get_wallet_name(wallet)
+        agd = self.rpc('listaddressgroupings', [])
+        #flatten all groups into a single list; then, remove duplicates
+        fagd = [tuple(item) for sublist in agd for item in sublist]
+        #"deduplicated flattened address grouping data" = dfagd
+        dfagd = list(set(fagd))
+        #for lookup, want dict of form {"address": amount}
+        used_address_dict = {}
+        for addr_info in dfagd:
+            if len(addr_info) < 3 or addr_info[2] != wallet_name:
+                continue
+            used_address_dict[addr_info[0]] = (addr_info[1], addr_info[2])
+
+        log.debug("Fast sync in progress. Got this many used addresses: " + str(
+            len(used_address_dict)))
+        #Need to have wallet.index point to the last used address
+        #and fill addr_cache.
+        #For each branch:
+        #If index value is present, collect all addresses up to index+gap limit
+        #For each address in that list, mark used if seen in used_address_dict
+        used_indices = {}
+        for md in range(wallet.max_mix_depth):
+            used_indices[md] = {}
+            for fc in [0, 1]:
+                used_indices[md][fc] = []
+                for i in range(wallet.index_cache[md][fc]+wallet.gaplimit):
+                    if wallet.get_addr(md, fc, i) in used_address_dict.keys():
+                        used_indices[md][fc].append(i)
+                        wallet.addr_cache[wallet.get_addr(md, fc, i)] = (md, fc, i)
+                if len(used_indices[md][fc]):
+                    wallet.index[md][fc] = used_indices[md][fc][-1]
+                else:
+                    wallet.index[md][fc] = 0
+                if not is_index_ahead_of_cache(wallet, md, fc):
+                    wallet.index[md][fc] = wallet.index_cache[md][fc]
+        self.wallet_synced = True
+
+
     def sync_addresses(self, wallet):
         from joinmarket.wallet import BitcoinCoreWallet
 
         if isinstance(wallet, BitcoinCoreWallet):
             return
-        log.debug('requesting wallet history')
+        log.debug('requesting detailed wallet history')
         wallet_name = self.get_wallet_name(wallet)
         #TODO It is worth considering making this user configurable:
         addr_req_count = 20

joinmarket/yieldgenerator.py

@@ -11,7 +11,7 @@
 from joinmarket import BlockrInterface
 from joinmarket import jm_single, get_network, load_program_config
 from joinmarket import get_log, calc_cj_fee, debug_dump_object
-from joinmarket import Wallet
+from joinmarket import Wallet, sync_wallet
 from joinmarket import get_irc_mchannels
 
 log = get_log()
@@ -102,6 +102,12 @@ def ygmain(ygclass, txfee=1000, cjfee_a=200, cjfee_r=0.002, ordertype='reloffer'
     parser.add_option('-g', '--gap-limit', action='store', type="int",
                       dest='gaplimit', default=6,
                       help='gap limit for wallet, default=6')
+    parser.add_option('--fast',
+                      action='store_true',
+                      dest='fastsync',
+                      default=False,
+                      help=('choose to do fast wallet sync, only for Core and '
+                      'only for previously synced wallet'))
     (options, args) = parser.parse_args()
     if len(args) < 1:
         parser.error('Needs a wallet')
@@ -142,7 +148,7 @@ def ygmain(ygclass, txfee=1000, cjfee_a=200, cjfee_r=0.002, ordertype='reloffer'
             return
 
     wallet = Wallet(seed, max_mix_depth=mix_levels, gaplimit=gaplimit)
-    jm_single().bc_interface.sync_wallet(wallet)
+    sync_wallet(wallet, fast=options.fastsync)
 
     mcs = [IRCMessageChannel(c, realname='btcint=' + jm_single().config.get(
                                  "BLOCKCHAIN", "blockchain_source"),

patientsendpayment.py

@@ -10,7 +10,7 @@
 from joinmarket import Maker, Taker, load_program_config, IRCMessageChannel
 from joinmarket import validate_address, jm_single
 from joinmarket import get_log, choose_orders, weighted_order_choose, \
-    debug_dump_object
+    debug_dump_object, sync_wallet
 from joinmarket import Wallet
 
 log = get_log()
@@ -194,6 +194,12 @@ def main():
             help=
             'Use the Bitcoin Core wallet through json rpc, instead of the internal joinmarket '
             + 'wallet. Requires blockchain_source=json-rpc')
+    parser.add_option('--fast',
+                      action='store_true',
+                      dest='fastsync',
+                      default=False,
+                      help=('choose to do fast wallet sync, only for Core and '
+                      'only for previously synced wallet'))
     (options, args) = parser.parse_args()
 
     if len(args) < 3:
@@ -221,7 +227,7 @@ def main():
         print 'not implemented yet'
         sys.exit(0)
     # wallet = BitcoinCoreWallet(fromaccount=wallet_name)
-    jm_single().bc_interface.sync_wallet(wallet)
+    sync_wallet(wallet, fast=options.fastsync)
 
     available_balance = wallet.get_balance_by_mixdepth()[options.mixdepth]
     if available_balance < amount:

sendpayment.py

@@ -15,7 +15,7 @@
 from joinmarket import validate_address, jm_single
 from joinmarket import get_log, choose_sweep_orders, choose_orders, \
     pick_order, cheapest_order_choose, weighted_order_choose, debug_dump_object
-from joinmarket import Wallet, BitcoinCoreWallet
+from joinmarket import Wallet, BitcoinCoreWallet, sync_wallet
 from joinmarket.wallet import estimate_tx_fee
 
 log = get_log()
@@ -276,6 +276,12 @@ def main():
         help=('Use the Bitcoin Core wallet through json rpc, instead '
               'of the internal joinmarket wallet. Requires '
               'blockchain_source=json-rpc'))
+    parser.add_option('--fast',
+                      action='store_true',
+                      dest='fastsync',
+                      default=False,
+                      help=('choose to do fast wallet sync, only for Core and '
+                      'only for previously synced wallet'))
     (options, args) = parser.parse_args()
 
     if len(args) < 3:
@@ -316,7 +322,7 @@ def main():
         wallet = Wallet(wallet_name, options.amtmixdepths, options.gaplimit)
     else:
         wallet = BitcoinCoreWallet(fromaccount=wallet_name)
-    jm_single().bc_interface.sync_wallet(wallet)
+    sync_wallet(wallet, fast=options.fastsync)
 
     mcs = [IRCMessageChannel(c) for c in get_irc_mchannels()]
     mcc = MessageChannelCollection(mcs)

test/test_blockr.py

@@ -11,7 +11,7 @@
 
 import bitcoin as btc
 import pytest
-from joinmarket import load_program_config, jm_single
+from joinmarket import load_program_config, jm_single, sync_wallet
 from joinmarket.blockchaininterface import BlockrInterface
 from joinmarket import get_p2pk_vbyte, get_log, Wallet
 
@@ -62,7 +62,7 @@ def test_blockr_estimate_fee(setup_blockr):
 def test_blockr_sync(setup_blockr, net, seed, gaplimit, showprivkey, method):
     jm_single().config.set("BLOCKCHAIN", "network", net)
     wallet = Wallet(seed, max_mix_depth = 5)
-    jm_single().bc_interface.sync_wallet(wallet)
+    sync_wallet(wallet)
 
     #copy pasted from wallet-tool; some boiled down form of
     #this should really be in wallet.py in the joinmarket module.

test/test_donations.py

@@ -10,7 +10,7 @@
 
 from commontest import make_wallets
 from joinmarket import load_program_config, get_p2pk_vbyte, get_log, jm_single
-from joinmarket import get_irc_mchannels, Taker
+from joinmarket import get_irc_mchannels, Taker, sync_wallet
 from joinmarket.configure import donation_address
 
 log = get_log()
@@ -24,7 +24,7 @@ def test_donation_address(setup_donations, amount):
     wallets = make_wallets(1, wallet_structures=[[1,1,1,0,0]],
                                mean_amt=0.5)
     wallet = wallets[0]['wallet']
-    jm_single().bc_interface.sync_wallet(wallet)
+    sync_wallet(wallet)
     #make a rdp from a simple privkey
     rdp_priv = "\x01"*32
     reusable_donation_pubkey = binascii.hexlify(secp256k1.PrivateKey(

test/test_podle.py

@@ -17,7 +17,7 @@
 from joinmarket import Taker, load_program_config, IRCMessageChannel
 from joinmarket import validate_address, jm_single, get_irc_mchannels
 from joinmarket import get_p2pk_vbyte, MessageChannelCollection
-from joinmarket import get_log, choose_sweep_orders, choose_orders, \
+from joinmarket import get_log, choose_sweep_orders, choose_orders, sync_wallet, \
     pick_order, cheapest_order_choose, weighted_order_choose, debug_dump_object
 import joinmarket.irc
 import sendpayment
@@ -145,7 +145,7 @@ def test_failed_sendpayment(setup_podle, num_ygs, wallet_structures, mean_amt,
 
     log.debug('starting sendpayment')
 
-    jm_single().bc_interface.sync_wallet(wallet)
+    sync_wallet(wallet)
 
     #Trigger PING LAG sending artificially
     joinmarket.irc.PING_INTERVAL = 3
@@ -241,7 +241,7 @@ def test_external_commitment_used(setup_podle):
 
     log.debug('starting sendpayment')
 
-    jm_single().bc_interface.sync_wallet(wallet)
+    sync_wallet(wallet)
 
     #Trigger PING LAG sending artificially
     joinmarket.irc.PING_INTERVAL = 3
@@ -347,7 +347,7 @@ def test_tx_commitments_used(setup_podle, consume_tx, age_required, cmt_age):
 
     log.debug('starting sendpayment')
 
-    jm_single().bc_interface.sync_wallet(wallet)
+    sync_wallet(wallet)
     log.debug("Here is the whole wallet: \n" + str(wallet.unspent))
     #Trigger PING LAG sending artificially
     joinmarket.irc.PING_INTERVAL = 3

test/test_regtest.py

@@ -10,7 +10,7 @@
 import pytest
 import time
 from joinmarket import (Taker, load_program_config, IRCMessageChannel,
-                        BitcoinCoreWallet)
+                        BitcoinCoreWallet, sync_wallet)
 from joinmarket import validate_address, jm_single, get_irc_mchannels
 from joinmarket import get_p2pk_vbyte, MessageChannelCollection
 from joinmarket import get_log, choose_sweep_orders, choose_orders, \
@@ -102,7 +102,7 @@ def test_sendpayment(setup_regtest, num_ygs, wallet_structures, mean_amt,
 
     log.debug('starting sendpayment')
 
-    jm_single().bc_interface.sync_wallet(wallet)
+    sync_wallet(wallet)
 
     #Trigger PING LAG sending artificially
     joinmarket.irc.PING_INTERVAL = 3

test/test_tumbler.py

@@ -11,7 +11,7 @@
 from joinmarket import Taker, load_program_config, IRCMessageChannel
 from joinmarket import validate_address, jm_single, MessageChannelCollection
 from joinmarket import get_p2pk_vbyte, get_irc_mchannels
-from joinmarket import get_log, choose_sweep_orders, choose_orders, \
+from joinmarket import get_log, choose_sweep_orders, choose_orders, sync_wallet, \
     pick_order, cheapest_order_choose, weighted_order_choose, debug_dump_object
 import json
 import tumbler
@@ -147,7 +147,7 @@ def test_tumbler(setup_tumbler, num_ygs, wallet_structures, mean_amt, sdev_amt,
 
     log.debug('starting tumbler')
 
-    jm_single().bc_interface.sync_wallet(wallet)
+    sync_wallet(wallet)
     jm_single().bc_interface.pushtx_failure_prob = 0.4
     mcs = [IRCMessageChannel(c) for c in get_irc_mchannels()]
     mcc = MessageChannelCollection(mcs)