Fix #438 better tracking of coinbase txos

electrumsv/constants.py

@@ -22,7 +22,7 @@ class StorageKind(IntEnum):
 
 DATABASE_EXT = ".sqlite"
 MIGRATION_FIRST = 22
-MIGRATION_CURRENT = 25
+MIGRATION_CURRENT = 26
 
 class TxFlags(IntFlag):
     Unset = 0

electrumsv/wallet.py

@@ -605,12 +605,8 @@ def _load_txo(self, row: TransactionOutputRow) -> None:
         else:
             keyinstance = self._keyinstances[row.keyinstance_id]
             script_template = self.get_script_template_for_id(row.keyinstance_id)
-            metadata = self._wallet._transaction_cache.get_metadata(row.tx_hash)
-            flags = row.flags
-            if metadata.position==0:
-                flags |= TransactionOutputFlag.IS_COINBASE
             address = script_template if isinstance(script_template, Address) else None
-            self.register_utxo(row.tx_hash, row.tx_index, row.value, flags,
+            self.register_utxo(row.tx_hash, row.tx_index, row.value, row.flags,
                 keyinstance, script_template.to_script(), address)
 
     def register_utxo(self, tx_hash: bytes, output_index: int, value: int,
@@ -640,9 +636,6 @@ def register_utxo(self, tx_hash: bytes, output_index: int, value: int,
     def create_transaction_output(self, tx_hash: bytes, output_index: int, value: int,
             flags: TransactionOutputFlag, keyinstance: KeyInstanceRow,
             script: Script, address: Optional[ScriptTemplate]=None):
-        metadata = self._wallet._transaction_cache.get_metadata(tx_hash)
-        if metadata.position == 0:
-            flags |= TransactionOutputFlag.IS_COINBASE
         if flags & TransactionOutputFlag.IS_SPENT:
             self._stxos[TxoKeyType(tx_hash, output_index)] = keyinstance.keyinstance_id
         else:
@@ -874,6 +867,8 @@ def _process_key_usage(self, tx_hash: bytes, tx: Transaction,
         key_matches = [(self.get_keyinstance(key_id),
             *self._get_cached_script(key_id)) for key_id in key_ids]
 
+        base_txo_flags = TransactionOutputFlag.IS_COINBASE if tx.is_coinbase() \
+            else TransactionOutputFlag.NONE
         tx_deltas: Dict[Tuple[bytes, int], int] = defaultdict(int)
         new_txos: List[Tuple[bytes, int, int, TransactionOutputFlag, KeyInstanceRow,
             ScriptTemplate]] = []
@@ -893,7 +888,7 @@ def _process_key_usage(self, tx_hash: bytes, tx: Transaction,
                 continue
 
             # Search the known candidates to see if we already have this txo's spending input.
-            txo_flags = TransactionOutputFlag.NONE
+            txo_flags = base_txo_flags
             for spend_tx_id, _height in self._sync_state.get_key_history(
                     keyinstance.keyinstance_id):
                 if spend_tx_id == tx_id:
@@ -909,7 +904,7 @@ def _process_key_usage(self, tx_hash: bytes, tx: Transaction,
                     continue
 
                 tx_deltas[(spend_tx_hash, keyinstance.keyinstance_id)] -= output.value
-                txo_flags = TransactionOutputFlag.IS_SPENT
+                txo_flags |= TransactionOutputFlag.IS_SPENT
                 break
 
             # TODO(rt12) BACKLOG batch create the outputs.
@@ -2502,8 +2497,7 @@ def missing_transactions(self) -> List[bytes]:
     def unverified_transactions(self) -> Dict[bytes, int]:
         '''Returns a map of tx_hash to tx_height.'''
         results = self._transaction_cache.get_unverified_entries(self.get_local_height())
-        self._logger.debug("unverified_transactions: %s",
-            [(hash_to_hex_str(r[0]), r[1]) for r in results])
+        self._logger.debug("unverified_transactions: %s", [hash_to_hex_str(r[0]) for r in results])
         return { t[0]: cast(int, t[1].metadata.height) for t in results }
 
     # Also called by network.

electrumsv/wallet_database/cache.py

@@ -505,6 +505,8 @@ def get_unverified_entries(self, watermark_height: int) \
         results = self.get_metadatas(
             flags=TxFlags.HasByteData | TxFlags.HasHeight,
             mask=TxFlags.HasByteData | TxFlags.HasPosition | TxFlags.HasHeight)
+        if len(results) > 200:
+            results = results[:200]
         return [ (tx_hash, self._cache[tx_hash]) for (tx_hash, metadata) in results
             if 0 < cast(int, metadata.height) <= watermark_height ]
 

electrumsv/wallet_database/migration.py

@@ -67,6 +67,9 @@ def update_database(db: sqlite3.Connection) -> None:
         if version == 24:
             migrations.migration_0025_invoices.execute(db)
             version += 1
+        if version == 25:
+            migrations.migration_0026_txo_coinbase_flag.execute(db)
+            version += 1
 
         if version != MIGRATION_CURRENT:
             db.rollback()

electrumsv/wallet_database/migrations/__init__.py

@@ -2,3 +2,4 @@
 from . import migration_0023_add_wallet_events
 from . import migration_0024_account_transactions
 from . import migration_0025_invoices
+from . import migration_0026_txo_coinbase_flag

electrumsv/wallet_database/migrations/migration_0026_txo_coinbase_flag.py

@@ -0,0 +1,24 @@
+import json
+try:
+    # Linux expects the latest package version of 3.31.1 (as of p)
+    import pysqlite3 as sqlite3
+except ModuleNotFoundError:
+    # MacOS expects the latest brew version of 3.32.1 (as of 2020-07-10).
+    # Windows builds use the official Python 3.7.8 builds and version of 3.31.1.
+    import sqlite3 # type: ignore
+import time
+
+from electrumsv.constants import TransactionOutputFlag
+
+MIGRATION = 26
+
+def execute(conn: sqlite3.Connection) -> None:
+    # Ensure that for all transactions in block position 0, all outputs for those transactions
+    # have the IS_COINBASE flag.
+    conn.execute("UPDATE TransactionOutputs "
+        f"SET flags=flags|{TransactionOutputFlag.IS_COINBASE} "
+        "WHERE tx_hash in (SELECT tx_hash FROM Transactions WHERE block_position = 0)")
+
+    date_updated = int(time.time())
+    conn.execute("UPDATE WalletData SET value=?, date_updated=? WHERE key=?",
+        [json.dumps(MIGRATION),date_updated,"migration"])