Merge branch 'release-0.5'

README.rst

@@ -79,7 +79,7 @@ ElectrumX should not have any need of threads.
 Roadmap
 =======
 
-- store all UTXOs, not just those with addresses
+- come up with UTXO root logic and implement it
 - test a few more performance improvement ideas
 - implement light caching of client responses
 - yield during expensive requests and/or penalize the connection

docs/RELEASE-NOTES

@@ -1,3 +1,12 @@
+version 0.5
+-----------
+
+- DB change: all UTXOs, including those that are not canonically paying to
+  an address, are stored in the DB.  So an attempt to spend a UTXO not in
+  the DB means corruption.  DB version bumped to 2; older versions will not
+  work
+- fixed issue #17: the genesis coinbase is not in the UTXO set
+
 version 0.4.3
 -------------
 

lib/coins.py

@@ -12,6 +12,7 @@
 '''
 
 from decimal import Decimal
+from functools import partial
 import inspect
 import struct
 import sys
@@ -34,6 +35,7 @@ class Coin(object):
     DEFAULT_RPC_PORT = 8332
     VALUE_PER_COIN = 100000000
     CHUNK_SIZE=2016
+    STRANGE_VERBYTE = 0xff
 
     @classmethod
     def lookup_coin_class(cls, name, net):
@@ -53,11 +55,14 @@ def hash168_handlers(cls):
             address = cls.P2PKH_hash168_from_hash160,
             script_hash = cls.P2SH_hash168_from_hash160,
             pubkey = cls.P2PKH_hash168_from_pubkey,
+            unspendable = cls.hash168_from_unspendable,
+            strange = cls.hash168_from_strange,
         )
 
     @classmethod
-    def hash168_from_script(cls, script):
-        return ScriptPubKey.pay_to(script, cls.hash168_handlers)
+    def hash168_from_script(cls):
+        '''Returns a function that is passed a script to return a hash168.'''
+        return partial(ScriptPubKey.pay_to, cls.hash168_handlers)
 
     @staticmethod
     def lookup_xverbytes(verbytes):
@@ -86,6 +91,16 @@ def hash168_to_address(cls, hash168):
         '''Return an address given a 21-byte hash.'''
         return Base58.encode_check(hash168)
 
+    @classmethod
+    def hash168_from_unspendable(cls):
+        '''Return a hash168 for an unspendable script.'''
+        return None
+
+    @classmethod
+    def hash168_from_strange(cls, script):
+        '''Return a hash168 for a strange script.'''
+        return bytes([cls.STRANGE_VERBYTE]) + hash160(script)
+
     @classmethod
     def P2PKH_hash168_from_hash160(cls, hash160):
         '''Return a hash168 if hash160 is 160 bits otherwise None.'''

lib/jsonrpc.py

@@ -147,7 +147,7 @@ def send_json_error(self, message, code, id_=None):
     def send_json(self, payload):
         '''Send a JSON payload.'''
         if self.transport.is_closing():
-            self.logger.info('send_json: connection closing, not sending')
+            # Confirmed this happens, sometimes a lot
             return False
 
         try:

lib/script.py

@@ -56,6 +56,19 @@ class ScriptError(Exception):
 assert OpCodes.OP_CHECKMULTISIG == 0xae
 
 
+def _match_ops(ops, pattern):
+    if len(ops) != len(pattern):
+        return False
+    for op, pop in zip(ops, pattern):
+        if pop != op:
+            # -1 means 'data push', whose op is an (op, data) tuple
+            if pop == -1 and isinstance(op, tuple):
+                continue
+            return False
+
+    return True
+
+
 class ScriptPubKey(object):
     '''A class for handling a tx output script that gives conditions
     necessary for spending.
@@ -66,32 +79,37 @@ class ScriptPubKey(object):
     TO_P2SH_OPS = [OpCodes.OP_HASH160, -1, OpCodes.OP_EQUAL]
     TO_PUBKEY_OPS = [-1, OpCodes.OP_CHECKSIG]
 
-    PayToHandlers = namedtuple('PayToHandlers', 'address script_hash pubkey')
+    PayToHandlers = namedtuple('PayToHandlers', 'address script_hash pubkey '
+                               'unspendable strange')
 
     @classmethod
-    def pay_to(cls, script, handlers):
+    def pay_to(cls, handlers, script):
         '''Parse a script, invoke the appropriate handler and
         return the result.
 
         One of the following handlers is invoked:
            handlers.address(hash160)
            handlers.script_hash(hash160)
            handlers.pubkey(pubkey)
-        or None is returned if the script is invalid or unregonised.
+           handlers.unspendable()
+           handlers.strange(script)
         '''
         try:
-            ops, datas = Script.get_ops(script)
+            ops = Script.get_ops(script)
         except ScriptError:
-            return None
+            return handlers.unspendable()
 
-        if Script.match_ops(ops, cls.TO_ADDRESS_OPS):
-            return handlers.address(datas[2])
-        if Script.match_ops(ops, cls.TO_P2SH_OPS):
-            return handlers.script_hash(datas[1])
-        if Script.match_ops(ops, cls.TO_PUBKEY_OPS):
-            return handlers.pubkey(datas[0])
+        match = _match_ops
 
-        return None
+        if match(ops, cls.TO_ADDRESS_OPS):
+            return handlers.address(ops[2][-1])
+        if match(ops, cls.TO_P2SH_OPS):
+            return handlers.script_hash(ops[1][-1])
+        if match(ops, cls.TO_PUBKEY_OPS):
+            return handlers.pubkey(ops[0][-1])
+        if ops and ops[0] == OpCodes.OP_RETURN:
+            return handlers.unspendable()
+        return handlers.strange(script)
 
     @classmethod
     def P2SH_script(cls, hash160):
@@ -141,54 +159,40 @@ class Script(object):
 
     @classmethod
     def get_ops(cls, script):
-        opcodes, datas = [], []
+        ops = []
 
         # The unpacks or script[n] below throw on truncated scripts
         try:
             n = 0
             while n < len(script):
-                opcode, data = script[n], None
+                op = script[n]
                 n += 1
 
-                if opcode <= OpCodes.OP_PUSHDATA4:
+                if op <= OpCodes.OP_PUSHDATA4:
                     # Raw bytes follow
-                    if opcode < OpCodes.OP_PUSHDATA1:
-                        dlen = opcode
-                    elif opcode == OpCodes.OP_PUSHDATA1:
+                    if op < OpCodes.OP_PUSHDATA1:
+                        dlen = op
+                    elif op == OpCodes.OP_PUSHDATA1:
                         dlen = script[n]
                         n += 1
-                    elif opcode == OpCodes.OP_PUSHDATA2:
-                        (dlen,) = struct.unpack('<H', script[n: n + 2])
+                    elif op == OpCodes.OP_PUSHDATA2:
+                        dlen, = struct.unpack('<H', script[n: n + 2])
                         n += 2
                     else:
-                        (dlen,) = struct.unpack('<I', script[n: n + 4])
+                        dlen, = struct.unpack('<I', script[n: n + 4])
                         n += 4
-                    data = script[n:n + dlen]
-                    if len(data) != dlen:
-                        raise ScriptError('truncated script')
+                    if n + dlen > len(script):
+                        raise IndexError
+                    op = (op, script[n:n + dlen])
                     n += dlen
 
-                opcodes.append(opcode)
-                datas.append(data)
+                ops.append(op)
         except:
             # Truncated script; e.g. tx_hash
             # ebc9fa1196a59e192352d76c0f6e73167046b9d37b8302b6bb6968dfd279b767
             raise ScriptError('truncated script')
 
-        return opcodes, datas
-
-    @classmethod
-    def match_ops(cls, ops, pattern):
-        if len(ops) != len(pattern):
-            return False
-        for op, pop in zip(ops, pattern):
-            if pop != op:
-                # -1 Indicates data push expected
-                if pop == -1 and OpCodes.OP_0 <= op <= OpCodes.OP_PUSHDATA4:
-                    continue
-                return False
-
-        return True
+        return ops
 
     @classmethod
     def push_data(cls, data):

server/block_processor.py

@@ -28,8 +28,6 @@
 # Limits single address history to ~ 65536 * HIST_ENTRIES_PER_KEY entries
 HIST_ENTRIES_PER_KEY = 1024
 HIST_VALUE_BYTES = HIST_ENTRIES_PER_KEY * 4
-NO_HASH_168 = bytes([255]) * 21
-NO_CACHE_ENTRY = NO_HASH_168 + bytes(12)
 
 
 def formatted_time(t):
@@ -209,7 +207,7 @@ async def update(self, hex_hashes):
 
         # The mempool is unordered, so process all outputs first so
         # that looking for inputs has full info.
-        script_hash168 = self.bp.coin.hash168_from_script
+        script_hash168 = self.bp.coin.hash168_from_script()
         db_utxo_lookup = self.bp.db_utxo_lookup
 
         def txout_pair(txout):
@@ -396,6 +394,11 @@ async def _wait_for_update(self):
         prefetcher only provides a non-None mempool when caught up.
         '''
         blocks, mempool_hashes = await self.prefetcher.get_blocks()
+
+        '''Strip the unspendable genesis coinbase.'''
+        if self.height == -1:
+            blocks[0] = blocks[0][:self.coin.HEADER_LEN] + bytes(1)
+
         caught_up = mempool_hashes is not None
         try:
             for block in blocks:
@@ -653,8 +656,6 @@ def backup_history(self, batch, hash168s):
         self.logger.info('backing up history to height {:,d}  tx_count {:,d}'
                          .format(self.height, self.tx_count))
 
-        # Drop any NO_CACHE entry
-        hash168s.discard(NO_CACHE_ENTRY)
         assert not self.history
 
         nremoves = 0
@@ -760,7 +761,7 @@ def advance_txs(self, tx_hashes, txs, touched):
         # Use local vars for speed in the loops
         history = self.history
         tx_num = self.tx_count
-        script_hash168 = self.coin.hash168_from_script
+        script_hash168 = self.coin.hash168_from_script()
         s_pack = pack
 
         for tx, tx_hash in zip(txs, tx_hashes):
@@ -776,15 +777,13 @@ def advance_txs(self, tx_hashes, txs, touched):
 
             # Add the new UTXOs
             for idx, txout in enumerate(tx.outputs):
-                # Get the hash168.  Ignore scripts we can't grok.
+                # Get the hash168.  Ignore unspendable outputs
                 hash168 = script_hash168(txout.pk_script)
                 if hash168:
                     hash168s.add(hash168)
                     put_utxo(tx_hash + s_pack('<H', idx),
                              hash168 + tx_numb + s_pack('<Q', txout.value))
 
-            # Drop any NO_CACHE entry
-            hash168s.discard(NO_CACHE_ENTRY)
             for hash168 in hash168s:
                 history[hash168].append(tx_num)
             self.history_size += len(hash168s)
@@ -908,15 +907,15 @@ def backup_txs(self, tx_hashes, txs, touched):
     the tx in which the UTXO was created.  As this is not unique there
     will are potential collisions when saving and looking up UTXOs;
     hence why the second table has a list as its value.  The collision
-    can be resolved with the tx_num.  The collision rate is almost
-    zero (I believe there are around 100 collisions in the whole
-    bitcoin blockchain).
+    can be resolved with the tx_num.  The collision rate is low (<0.1%).
     '''
 
     def spend_utxo(self, tx_hash, tx_idx):
         '''Spend a UTXO and return the 33-byte value.
 
-        If the UTXO is not in the cache it may be on disk.
+        If the UTXO is not in the cache it must be on disk.  We store
+        all UTXOs so not finding one indicates a logic error or DB
+        corruption.
         '''
         # Fast track is it being in the cache
         idx_packed = pack('<H', tx_idx)
@@ -930,46 +929,42 @@ def spend_utxo(self, tx_hash, tx_idx):
         # The 4 is the COMPRESSED_TX_HASH_LEN
         db_key = b'h' + tx_hash[:4] + idx_packed
         db_value = self.db_cache_get(db_key)
-        if db_value is None:
-            # Probably a strange UTXO
-            return NO_CACHE_ENTRY
-
-        # FIXME: this matches what we did previously but until we store
-        # all UTXOs isn't safe
-        if len(db_value) == 25:
-            udb_key = b'u' + db_value + idx_packed
-            utxo_value_packed = self.db.get(udb_key)
-            if utxo_value_packed:
-                # Remove the UTXO from both tables
-                self.db_deletes += 1
-                self.db_cache[db_key] = None
-                self.db_cache[udb_key] = None
-                return db_value + utxo_value_packed
-            # Fall through to below
-
-        assert len(db_value) % 25 == 0
-
-        # Find which entry, if any, the TX_HASH matches.
-        for n in range(0, len(db_value), 25):
-            tx_num, = unpack('<I', db_value[n+21:n+25])
-            hash, height = self.get_tx_hash(tx_num)
-            if hash == tx_hash:
-                match = db_value[n:n+25]
-                udb_key = b'u' + match + idx_packed
+        if db_value:
+            # FIXME: this matches what we did previously but until we store
+            # all UTXOs isn't safe
+            if len(db_value) == 25:
+                udb_key = b'u' + db_value + idx_packed
                 utxo_value_packed = self.db.get(udb_key)
                 if utxo_value_packed:
                     # Remove the UTXO from both tables
                     self.db_deletes += 1
-                    self.db_cache[db_key] = db_value[:n] + db_value[n + 25:]
+                    self.db_cache[db_key] = None
                     self.db_cache[udb_key] = None
-                    return match + utxo_value_packed
-
-                # Uh-oh, this should not happen...
-                raise self.DBError('UTXO {} / {:,d} not found, key {}'
-                                   .format(hash_to_str(tx_hash), tx_idx,
-                                           bytes(key).hex()))
-
-        return NO_CACHE_ENTRY
+                    return db_value + utxo_value_packed
+                # Fall through to below loop for error
+
+            assert len(db_value) % 25 == 0
+
+            # Find which entry, if any, the TX_HASH matches.
+            for n in range(0, len(db_value), 25):
+                tx_num, = unpack('<I', db_value[n + 21:n + 25])
+                hash, height = self.get_tx_hash(tx_num)
+                if hash == tx_hash:
+                    match = db_value[n:n+25]
+                    udb_key = b'u' + match + idx_packed
+                    utxo_value_packed = self.db.get(udb_key)
+                    if utxo_value_packed:
+                        # Remove the UTXO from both tables
+                        self.db_deletes += 1
+                        self.db_cache[db_key] = db_value[:n] + db_value[n+25:]
+                        self.db_cache[udb_key] = None
+                        return match + utxo_value_packed
+
+                    raise self.DBError('UTXO {} / {:,d} not found in "u" table'
+                                       .format(hash_to_str(tx_hash), tx_idx))
+
+        raise ChainError('UTXO {} / {:,d} not found in "h" table'
+                         .format(hash_to_str(tx_hash), tx_idx))
 
     def db_cache_get(self, key):
         '''Fetch a 'h' value from the DB through our write cache.'''

server/db.py

@@ -29,7 +29,7 @@ class DB(LoggedClass):
     it was shutdown uncleanly.
     '''
 
-    VERSIONS = [0]
+    VERSIONS = [2]
 
     class MissingUTXOError(Exception):
         '''Raised if a mempool tx input UTXO couldn't be found.'''

server/version.py

@@ -1 +1 @@
-VERSION = "ElectrumX 0.4.3"
+VERSION = "ElectrumX 0.5"