Merge pull request spesmilo#282 from zebra-lucky/addr_synch_caches_op…

…timize_qt_addrs

Optimize for big PS wallets (wallet/qt address list)

electrum_dash/address_synchronizer.py

@@ -102,6 +102,8 @@ def __init__(self, db: 'WalletDB'):
         self.psman = PSManager(self)
         self.protx_manager = ProTxManager(self)
 
+        self._addrs_with_coins_cache = set()
+        self._tx_deltas_cache = defaultdict(int)
         self._get_addr_balance_cache = {}
 
         self.load_and_cleanup()
@@ -117,6 +119,8 @@ def load_and_cleanup(self):
         self.check_history()
         self.load_unverified_transactions()
         self.remove_local_transactions_we_dont_have()
+        self.populate_tx_deltas_cache()
+        self.populate_addrs_with_coins_cache()
         self.psman.load_and_cleanup()
         self.protx_manager.load()
 
@@ -384,6 +388,8 @@ def add_value_from_prev_output():
             is_new_tx = (tx_hash not in self.db.transactions)
             self.db.add_transaction(tx_hash, tx)
             self.db.add_num_inputs_to_tx(tx_hash, len(tx.inputs()))
+            self.update_tx_deltas_cache_on_tx(tx_hash, tx, is_added=True)
+            self.update_addrs_with_coins_cache_on_tx(tx)
             if is_new_tx and self.psman.enabled:
                 self.psman._add_tx_ps_data(tx_hash, tx)
             if is_new_tx and not self.is_local_tx(tx_hash):
@@ -440,6 +446,9 @@ def remove_from_spent_outpoints():
                     scripthash = bitcoin.script_to_scripthash(txo.scriptpubkey.hex())
                     prevout = TxOutpoint(bfh(tx_hash), idx)
                     self.db.remove_prevout_by_scripthash(scripthash, prevout=prevout, value=txo.value)
+            self.update_tx_deltas_cache_on_tx(tx_hash, tx, is_added=False)
+            self.update_addrs_with_coins_cache_on_tx(tx)
+
 
     def get_depending_transactions(self, tx_hash: str) -> Set[str]:
         """Returns all (grand-)children of tx_hash in this wallet."""
@@ -524,12 +533,74 @@ def remove_local_transactions_we_dont_have(self):
             if tx_height == TX_HEIGHT_LOCAL and not self.db.get_transaction(txid):
                 self.remove_transaction(txid)
 
+    @profiler
+    def populate_tx_deltas_cache(self):
+        for addr in self.get_addresses() + self.psman.get_addresses():
+            h = self.get_address_history(addr)
+            for tx_hash, height, islock in h:
+                delta = self.get_tx_delta(tx_hash, addr)
+                self._tx_deltas_cache[tx_hash] += delta
+
+    def update_tx_deltas_cache_on_tx(self, txid, tx, *, is_added):
+        self._tx_deltas_cache.pop(txid, None)
+        if not is_added:
+            return
+        mine_addrs = set()
+        for txin in tx.inputs():
+            addr = self.get_txin_address(txin)
+            if self.is_mine(addr):
+                mine_addrs.add(addr)
+        for o in tx.outputs():
+            addr = o.address
+            if self.is_mine(addr):
+                mine_addrs.add(addr)
+        for addr in mine_addrs:
+            self._tx_deltas_cache[txid] += self.get_tx_delta(txid, addr)
+
+    def is_addr_with_coins(self, addr, local_height):
+        addr_outputs = self.get_addr_outputs(addr)
+        for k, v in list(addr_outputs.items()):
+            if v.spent_height is None:
+                continue
+            if 0 < v.spent_height <= local_height:
+                addr_outputs.pop(k)
+        if addr_outputs:
+            return True
+
+    @profiler
+    def populate_addrs_with_coins_cache(self):
+        local_height = self.get_local_height()
+        for addr in self.get_addresses() + self.psman.get_addresses():
+            if self.is_addr_with_coins(addr, local_height):
+                self._addrs_with_coins_cache.add(addr)
+
+    def update_addrs_with_coins_cache_on_tx(self, tx):
+        if not tx:
+            return
+        local_height = self.get_local_height()
+        mine_addrs = set()
+        for txin in tx.inputs():
+            addr = self.get_txin_address(txin)
+            if self.is_mine(addr):
+                mine_addrs.add(addr)
+        for o in tx.outputs():
+            addr = o.address
+            if self.is_mine(addr):
+                mine_addrs.add(addr)
+        for addr in mine_addrs:
+            if self.is_addr_with_coins(addr, local_height):
+                self._addrs_with_coins_cache.add(addr)
+            elif addr in self._addrs_with_coins_cache:
+                self._addrs_with_coins_cache.remove(addr)
+
     def clear_history(self):
         with self.lock:
             with self.transaction_lock:
                 self.db.clear_history()
                 self._history_local.clear()
                 self._get_addr_balance_cache = {}  # invalidate cache
+                self._tx_deltas_cache = defaultdict(int)
+                self._addrs_with_coins_cache = set()
 
     def get_txpos(self, tx_hash, islock):
         """Returns (height, txpos) tuple, even if the tx is unverified."""
@@ -563,6 +634,7 @@ def f(self, *args, **kwargs):
     @with_lock
     @with_transaction_lock
     @with_local_height_cached
+    @profiler
     def get_history(self, *, domain=None, config=None, group_ps=False) -> Sequence[HistoryItem]:
         # get domain
         if domain is None:
@@ -571,20 +643,15 @@ def get_history(self, *, domain=None, config=None, group_ps=False) -> Sequence[H
         domain = set(domain)
         # 1. Get the history of each address in the domain, maintain the
         #    delta of a tx as the sum of its deltas on domain addresses
-        tx_deltas = defaultdict(int)  # type: Dict[str, int]
-        tx_islocks = {}
-        for addr in domain:
-            h = self.get_address_history(addr)
-            for tx_hash, height, islock in h:
-                tx_deltas[tx_hash] += self.get_tx_delta(tx_hash, addr)
-                if tx_hash not in tx_islocks:
-                    tx_islocks[tx_hash] = islock
+        if not self._tx_deltas_cache:
+            self.populate_tx_deltas_cache()
+        tx_deltas = self._tx_deltas_cache
         # 2. create sorted history
         history = []
         for tx_hash in tx_deltas:
             delta = tx_deltas[tx_hash]
             tx_mined_status = self.get_tx_height(tx_hash)
-            islock = tx_islocks[tx_hash]
+            islock = self.db.get_islock(tx_hash)
             if islock:
                 islock_sort = tx_hash if not tx_mined_status.conf else ''
             else:
@@ -1019,6 +1086,7 @@ def get_addr_balance(self, address, *, excluded_coins: Set[str] = None,
         return result
 
     @with_local_height_cached
+    @profiler
     def get_utxos(
             self,
             domain=None,
@@ -1059,6 +1127,8 @@ def get_utxos(
             domain = set(domain) - set(excluded_addresses)
         mempool_height = block_height + 1  # height of next block
         for addr in domain:
+            if addr not in self._addrs_with_coins_cache:
+                continue
             txos = self.get_addr_outputs(addr)
             for txo in txos.values():
                 if txo.address in ps_ks_domain:

electrum_dash/gui/qt/address_list.py

@@ -85,9 +85,9 @@ def ui_text(self) -> str:
 
 
 class KeystoreFilter(IntEnum):
-    ALL = 0
+    MAIN = 0
     PS_KS = 1
-    MAIN = 2
+    ALL = 2
 
     def ui_text(self) -> str:
         return {
@@ -271,26 +271,32 @@ def get_addresses(self):
 
         if show_ps_ks in [KeystoreFilter.ALL, KeystoreFilter.MAIN]:
             if show_change == AddressTypeFilter.RECEIVING:
-                all_addrs = w.get_receiving_addresses()
+                main_ks_addrs = w.get_receiving_addresses()
             elif show_change == AddressTypeFilter.CHANGE:
-                all_addrs = w.get_change_addresses()
+                main_ks_addrs = w.get_change_addresses()
             else:
-                all_addrs = w.get_addresses()
+                main_ks_addrs = w.get_addresses()
         else:
-            all_addrs = []
-        all_addrs += ps_ks_addrs
+            main_ks_addrs = []
 
         if show_ps == PSStateFilter.ALL:
-            addr_list = all_addrs
+            main_ks_addrs = [(addr, False) for addr in main_ks_addrs]
+            ps_ks_addrs = [(addr, True) for addr in ps_ks_addrs]
         elif show_ps == PSStateFilter.PS:
-            addr_list = [addr for addr in all_addrs if addr in ps_addrs]
+            main_ks_addrs = [(addr, False) for addr in main_ks_addrs
+                             if addr in ps_addrs]
+            ps_ks_addrs = [(addr, True) for addr in ps_ks_addrs
+                           if addr in ps_addrs]
         else:
-            addr_list = [addr for addr in all_addrs if addr not in ps_addrs]
+            main_ks_addrs = [(addr, False) for addr in main_ks_addrs
+                             if addr not in ps_addrs]
+            ps_ks_addrs = [(addr, True) for addr in ps_ks_addrs
+                           if addr not in ps_addrs]
 
         addrs_beyond_gap_limit = w.get_all_known_addresses_beyond_gap_limit()
         ps_ks_beyond_gap = w.psman.get_all_known_addresses_beyond_gap_limit()
         fx = self.parent.fx
-        for i, addr in enumerate(addr_list):
+        for i, (addr, is_ps_ks) in enumerate(main_ks_addrs + ps_ks_addrs):
             balance = sum(w.get_addr_balance(addr))
             is_used_and_empty = w.is_used(addr) and balance == 0
             if (show_used == AddressUsageStateFilter.UNUSED
@@ -309,7 +315,6 @@ def get_addresses(self):
             else:
                 fiat_balance = ''
 
-            is_ps_ks = True if addr in ps_ks_addrs else False
             if is_ps_ks:
                 is_beyond_limit = addr in ps_ks_beyond_gap
             else:

electrum_dash/gui/qt/network_dialog.py

@@ -57,7 +57,7 @@ class NetworkDialog(QDialog):
     def __init__(self, *, network: Network, config: 'SimpleConfig', network_updated_signal_obj):
         QDialog.__init__(self)
         self.setWindowTitle(_('Electrum Network'))
-        self.setMinimumSize(500, 500)
+        self.setMinimumSize(700, 500)
         self.nlayout = NetworkChoiceLayout(network, config)
         self.network_updated_signal_obj = network_updated_signal_obj
         vbox = QVBoxLayout(self)

electrum_dash/tests/test_wallet_vertical.py

@@ -1114,3 +1114,72 @@ def test_restoring_wallet_with_manual_delete(self, mock_save_db):
         txC = Transaction(self.transactions["a04328fbc9f28268378a8b9cf103db21ca7d673bf1cc7fa4d61b6a7265f07a6b"])
         w.add_transaction(txC)
         self.assertEqual(83500163, sum(w.get_balance()))
+
+
+class TestWalletCaches(TestCaseForTestnet):
+    transactions = [
+        # tx A is an external incoming tx funding the wallet
+        ("0cce62d61ec87ad3e391e8cd752df62e0c952ce45f52885d6d10988e02794060", "0200000001191601a44a81e061502b7bfbc6eaa1cef6d1e6af5308ef96c9342f71dbf4b9b5000000006b483045022100a6d44d0a651790a477e75334adfb8aae94d6612d01187b2c02526e340a7fd6c8022028bdf7a64a54906b13b145cd5dab21a26bd4b85d6044e9b97bceab5be44c2a9201210253e8e0254b0c95776786e40984c1aa32a7d03efa6bdacdea5f421b774917d346feffffff026b20fa04000000001976a914dc3a05eb562fb6f3ef8076946514d4730cff299988aca0860100000000001976a91421919b94ae5cefcdf0271191459157cdb41c4cbf88aca6240700"),
+        # tx B is an outgoing payment to an external address
+        ("e7f4e47f41421e37a8600b6350befd586f30db60a88d0992d54df280498f0968", "0200000001604079028e98106d5d88525fe42c950c2ef62d75cde891e3d37ac81ed662ce0c000000006b483045022100a6d44d0a651790a477e75334adfb8aae94d6612d01187b2c02526e340a7fd6c8022028bdf7a64a54906b13b145cd5dab21a26bd4b85d6044e9b97bceab5be44c2a9201210253e8e0254b0c95776786e40984c1aa32a7d03efa6bdacdea5f421b774917d346feffffff01831cfa04000000001976a914024db2e87dd7cfd0e5f266c5f212e21a31d805a588aca6240700"),
+    ]
+
+    def setUp(self):
+        super().setUp()
+        self.config = SimpleConfig({'electrum_path': self.electrum_path})
+
+    @mock.patch.object(wallet.Abstract_Wallet, 'save_db')
+    def test_addrs_with_coins_cache(self, mock_save_db):
+        w = restore_wallet_from_text("hint shock chair puzzle shock traffic drastic note dinosaur mention suggest sweet",
+                                     path='if_this_exists_mocking_failed_648151893',
+                                     gap_limit=5,
+                                     config=self.config)['wallet']  # type: Abstract_Wallet
+        w.db.put('stored_height', 100000)  # set local height
+        txidA, rawA = self.transactions[0]
+        txidB, rawB = self.transactions[1]
+
+        assert sum(w.get_balance()) == 0
+        assert w._addrs_with_coins_cache == set()
+
+        w.add_transaction(Transaction(rawA))
+        assert sum(w.get_balance()) == 83501163
+        assert w._addrs_with_coins_cache == {'ygPu1vV5ZxAgGevfWTKnRPPexsJ9JkfNJ4'}
+
+        w.add_transaction(Transaction(rawB))  # local tx
+        assert sum(w.get_balance()) == 0
+        assert w._addrs_with_coins_cache == {'ygPu1vV5ZxAgGevfWTKnRPPexsJ9JkfNJ4'}
+
+        w.add_unverified_tx(txidB, 100000)    # unverified tx with local height
+        w.add_transaction(Transaction(rawB))
+        assert w._addrs_with_coins_cache == set()
+
+        w.remove_transaction(txidB)
+        assert w._addrs_with_coins_cache == {'ygPu1vV5ZxAgGevfWTKnRPPexsJ9JkfNJ4'}
+        w.remove_transaction(txidA)
+        assert w._addrs_with_coins_cache == set()
+
+    @mock.patch.object(wallet.Abstract_Wallet, 'save_db')
+    def test_tx_deltas_cache(self, mock_save_db):
+        w = restore_wallet_from_text("hint shock chair puzzle shock traffic drastic note dinosaur mention suggest sweet",
+                                     path='if_this_exists_mocking_failed_648151893',
+                                     gap_limit=5,
+                                     config=self.config)['wallet']  # type: Abstract_Wallet
+        txidA, rawA = self.transactions[0]
+        txidB, rawB = self.transactions[1]
+
+        assert sum(w.get_balance()) == 0
+        assert w._tx_deltas_cache == {}
+
+        w.add_transaction(Transaction(rawA))
+        assert sum(w.get_balance()) == 83501163
+        assert w._tx_deltas_cache == {txidA: 83501163}
+
+        w.add_transaction(Transaction(rawB))
+        assert sum(w.get_balance()) == 0
+        assert w._tx_deltas_cache == {txidA: 83501163, txidB: -83501163}
+
+        w.remove_transaction(txidB)
+        assert w._tx_deltas_cache == {txidA: 83501163}
+
+        w.remove_transaction(txidA)
+        assert w._tx_deltas_cache == {}