Features/tsc merkle proofs (#1056)

* Add tsc merkle proof endpoint

see: https://tsc.bitcoinassociation.net/standards/merkle-proof-standardised-format/

* Fix pylint warnings

* Fix AttributeError in pytest causing tests to fail instead of skipping

- Error message: "'MarkDecorator' object has no attribute 'lower'"

* Add tsc_format option to Merkle.branch_and_root() which adds asterixes in place of "duplicate hashes"

- SessionManager._merkle_branch now always returns branch, root and cost
- Add unittest coverage for the "blockchain.transaction.get_tsc_merkle" endpoint
- Add MacOS-specific installation for plyvel / leveldb to fix a long-standing
issue with test_compaction failing in Azure. This required leveldb 1.22
as leveldb v 1.23 runs into issues with "Symbol not found: __ZTIN7leveldb10ComparatorE"

* Add documentation for TSC merkle proof format endpoint

- Reorganise into 4 separate unittests

* Account for all cost estimates (TSC merkle endpoint)

.azure-pipelines/prepare-env.yml

@@ -27,3 +27,22 @@ steps:
   enabled: true
   continueOnError: false
   failOnStderr: false
+
+# Install plyvel depending on platform
+- bash: |
+    python -m pip install plyvel
+  condition: eq( variables['Agent.OS'], 'Linux' )
+  displayName: Install plyvel on Linux
+  enabled: true
+  continueOnError: false
+  failOnStderr: false
+- bash: |
+    brew tap bagonyi/homebrew-formulae git@github.com:bagonyi/homebrew-formulae.git
+    brew extract --version=1.22 leveldb bagonyi/formulae
+    brew install leveldb@1.22
+    pip install plyvel --no-cache-dir
+  condition: eq( variables['Agent.OS'], 'Darwin' )
+  displayName: Install plyvel on MacOS
+  enabled: true
+  continueOnError: false
+  failOnStderr: false

docs/protocol-methods.rst

@@ -674,6 +674,90 @@ and height.
     "pos": 710
   }
 
+
+blockchain.transaction.get_tsc_merkle
+=====================================
+
+Return the TSC Bitcoin Association merkle proof in standardised format for a confirmed
+transaction given its hash and height. Additional options include: txid_or_tx and target_type.
+
+See: https://tsc.bitcoinassociation.net/standards/merkle-proof-standardised-format/
+
+**Signature**
+
+  .. function:: blockchain.transaction.get_tsc_merkle(tx_hash, height, txid_or_tx="txid", target_type="block_hash")
+
+  *tx_hash*
+
+    The transaction hash as a hexadecimal string.
+
+  *height*
+
+    The height at which it was confirmed, an integer.
+
+  *txid_or_tx*
+
+    Takes two possible values: "txid" or "tx".
+    Selects whether to return the transaction hash or the full transaction as a hexadecimal string.
+
+  *target_type*
+
+    Takes three possible values: "block_hash", "block_header", "merkle_root"
+    The selected target is returned as a hexidecimal string in the response.
+
+
+**Result**
+
+  A dictionary with the following keys:
+
+  * *composite*
+
+    Included for completeness. Whether or not this is a composite merkle proof (for two or more
+    transactions). ElectrumX does not support composite proofs at this time (always False).
+
+  * *index*
+
+    The 0-based position index of the transaction in the block.
+
+  * *nodes*
+
+    The list of hash pairs making up the merkle branch. "Duplicate" hashes (see TSC merkle proof
+    format spec.) are replaced with asterixes as they can be derived by the client.
+
+  * *proofType*
+
+    Included for completeness. Specifies the proof type as either 'branch' or 'tree' type.
+    ElectrumX only supports 'branch' proof types.
+
+  * *target*
+
+    Either the block_hash, block_header or merkle_root as a hexidecimal string.
+
+  * *targetType*
+
+    Takes three possible values: "block_hash", "block_header", "merkle_root"
+
+  * *txOrId*
+
+    Either a 32 byte tx hash or a full transaction as a hexidecimal string.
+
+**Result Example**
+
+::
+
+    {
+        'composite': False,
+        'index': 4,
+        'nodes': [
+            '*',
+            '*',
+            '80c0100bc080eb0d2e205dc687056dc13c2079d0959c70cad8856fea88c74aba'],
+        'proofType': 'branch',
+        'target': '29442cb6e2ee547fcf5200dfb1b4018f4fc5ce5a220bb5ec3729a686885692fc',
+        'targetType': 'block_hash',
+        'txOrId': 'ed5a81e439e1cd9139ddb81da80bfa7cfc31e323aea544ca92a9ee1d84b9fb2f'
+    }
+
 blockchain.transaction.id_from_pos
 ==================================
 

electrumx/lib/coins.py

@@ -35,13 +35,13 @@
 from decimal import Decimal
 from hashlib import sha256
 
-import electrumx.lib.util as util
+from electrumx.lib import util
 from electrumx.lib.hash import Base58, double_sha256, hash_to_hex_str
 from electrumx.lib.hash import HASHX_LEN
 from electrumx.lib.script import ScriptPubKey
 import electrumx.lib.tx as lib_tx
 import electrumx.server.block_processor as block_proc
-import electrumx.server.daemon as daemon
+from electrumx.server import daemon
 from electrumx.server.session import ElectrumX
 
 

electrumx/lib/merkle.py

@@ -55,7 +55,7 @@ def branch_length(self, hash_count):
             raise ValueError('hash_count must be at least 1')
         return ceil(log(hash_count, 2))
 
-    def branch_and_root(self, hashes, index, length=None):
+    def branch_and_root(self, hashes, index, length=None, tsc_format=False):
         '''Return a (merkle branch, merkle_root) pair given hashes, and the
         index of one of those hashes.
         '''
@@ -79,7 +79,15 @@ def branch_and_root(self, hashes, index, length=None):
         for _ in range(length):
             if len(hashes) & 1:
                 hashes.append(hashes[-1])
-            branch.append(hashes[index ^ 1])
+
+                # Asterix used in place of "duplicated" hashes in TSC format (derivable by client)
+                is_last_node_in_level = (index ^ 1 == len(hashes)-1)
+                if tsc_format and is_last_node_in_level:
+                    branch.append(b"*")
+                else:
+                    branch.append(hashes[index ^ 1])
+            else:
+                branch.append(hashes[index ^ 1])
             index >>= 1
             hashes = [hash_func(hashes[n] + hashes[n + 1])
                       for n in range(0, len(hashes), 2)]
@@ -124,7 +132,7 @@ def level(self, hashes, depth_higher):
                 for n in range(0, len(hashes), size)]
 
     def branch_and_root_from_level(self, level, leaf_hashes, index,
-                                   depth_higher):
+                                   depth_higher, tsc_format=False):
         '''Return a (merkle branch, merkle_root) pair when a merkle-tree has a
         level cached.
 
@@ -148,9 +156,9 @@ def branch_and_root_from_level(self, level, leaf_hashes, index,
             raise TypeError("leaf_hashes must be a list")
         leaf_index = (index >> depth_higher) << depth_higher
         leaf_branch, leaf_root = self.branch_and_root(
-            leaf_hashes, index - leaf_index, depth_higher)
+            leaf_hashes, index - leaf_index, depth_higher, tsc_format=tsc_format)
         index >>= depth_higher
-        level_branch, root = self.branch_and_root(level, index)
+        level_branch, root = self.branch_and_root(level, index, tsc_format=tsc_format)
         # Check last so that we know index is in-range
         if leaf_root != level[index]:
             raise ValueError('leaf hashes inconsistent with level')
@@ -229,7 +237,7 @@ def truncate(self, length):
         self.length = length
         self.level[length >> self.depth_higher:] = []
 
-    async def branch_and_root(self, length, index):
+    async def branch_and_root(self, length, index, tsc_format=False):
         '''Return a merkle branch and root.  Length is the number of
         hashes used to calculate the merkle root, index is the position
         of the hash to calculate the branch of.
@@ -249,7 +257,7 @@ async def branch_and_root(self, length, index):
         count = min(self._segment_length(), length - leaf_start)
         leaf_hashes = await self.source_func(leaf_start, count)
         if length < self._segment_length():
-            return self.merkle.branch_and_root(leaf_hashes, index)
+            return self.merkle.branch_and_root(leaf_hashes, index, tsc_format=tsc_format)
         level = await self._level_for(length)
         return self.merkle.branch_and_root_from_level(
-            level, leaf_hashes, index, self.depth_higher)
+            level, leaf_hashes, index, self.depth_higher, tsc_format=tsc_format)

electrumx/lib/text.py

@@ -1,6 +1,6 @@
 import time
 
-import electrumx.lib.util as util
+from electrumx.lib import util
 
 
 def sessions_lines(data):

electrumx/server/db.py

@@ -20,7 +20,7 @@
 import attr
 from aiorpcx import run_in_thread, sleep
 
-import electrumx.lib.util as util
+from electrumx.lib import util
 from electrumx.lib.hash import hash_to_hex_str
 from electrumx.lib.merkle import Merkle, MerkleCache
 from electrumx.lib.util import (

electrumx/server/history.py

@@ -14,7 +14,7 @@
 import time
 from collections import defaultdict
 
-import electrumx.lib.util as util
+from electrumx.lib import util
 from electrumx.lib.util import (
     pack_be_uint16, pack_le_uint64, unpack_be_uint16_from, unpack_le_uint64,
 )

electrumx/server/session.py

@@ -29,9 +29,9 @@
 import electrumx
 from electrumx.lib.merkle import MerkleCache
 from electrumx.lib.text import sessions_lines
-import electrumx.lib.util as util
-from electrumx.lib.hash import (sha256, hash_to_hex_str, hex_str_to_hash,
-                                HASHX_LEN, Base58Error)
+from electrumx.lib import util
+from electrumx.lib.hash import (sha256, hash_to_hex_str, hex_str_to_hash, HASHX_LEN, Base58Error,
+                                double_sha256)
 from electrumx.server.daemon import DaemonError
 from electrumx.server.peers import PeerManager
 
@@ -643,7 +643,7 @@ def extra_cost(self, session):
             return 0
         return sum((group.cost() - session.cost) * group.weight for group in groups)
 
-    async def _merkle_branch(self, height, tx_hashes, tx_pos):
+    async def _merkle_branch(self, height, tx_hashes, tx_pos, tsc_format=False):
         tx_hash_count = len(tx_hashes)
         cost = tx_hash_count
 
@@ -660,12 +660,22 @@ async def tx_hashes_func(start, count):
                 merkle_cache = MerkleCache(self.db.merkle, tx_hashes_func)
                 self._merkle_cache[height] = merkle_cache
                 await merkle_cache.initialize(len(tx_hashes))
-            branch, _root = await merkle_cache.branch_and_root(tx_hash_count, tx_pos)
+            branch, root = await merkle_cache.branch_and_root(tx_hash_count, tx_pos,
+                                                              tsc_format=tsc_format)
         else:
-            branch, _root = self.db.merkle.branch_and_root(tx_hashes, tx_pos)
+            branch, root = self.db.merkle.branch_and_root(tx_hashes, tx_pos,
+                                                          tsc_format=tsc_format)
 
-        branch = [hash_to_hex_str(hash) for hash in branch]
-        return branch, cost / 2500
+        if tsc_format:
+            def converter(_hash):
+                if _hash == b"*":
+                    return _hash.decode()
+                else:
+                    return hash_to_hex_str(_hash)
+            branch = [converter(hash) for hash in branch]
+        else:
+            branch = [hash_to_hex_str(hash) for hash in branch]
+        return branch, root, cost / 2500
 
     async def merkle_branch_for_tx_hash(self, height, tx_hash):
         '''Return a triple (branch, tx_pos, cost).'''
@@ -676,9 +686,72 @@ async def merkle_branch_for_tx_hash(self, height, tx_hash):
             raise RPCError(
                 BAD_REQUEST, f'tx {hash_to_hex_str(tx_hash)} not in block at height {height:,d}'
             ) from None
-        branch, merkle_cost = await self._merkle_branch(height, tx_hashes, tx_pos)
+        branch, _root, merkle_cost = await self._merkle_branch(height, tx_hashes, tx_pos)
         return branch, tx_pos, tx_hashes_cost + merkle_cost
 
+    async def tsc_merkle_proof_for_tx_hash(self, height, tx_hash, txid_or_tx='txid',
+                                           target_type='block_hash'):
+        '''Return a pair (tsc_proof, cost) where tsc_proof is a dictionary with fields:
+            index - the position of the transaction
+            txOrId - either "txid" or "tx"
+            target - either "block_hash", "block_header" or "merkle_root"
+            nodes - the nodes in the merkle branch excluding the "target"'''
+
+        async def get_target(target_type):
+            try:
+                cost = 0.25
+                raw_header = await self.raw_header(height)
+                root_from_header = raw_header[36:36 + 32]
+                if target_type == "block_header":
+                    target = raw_header.hex()
+                elif target_type == "merkle_root":
+                    target = hash_to_hex_str(root_from_header)
+                else:  # target == block hash
+                    target = hash_to_hex_str(double_sha256(raw_header))
+            except ValueError:
+                raise RPCError(BAD_REQUEST, f'block header at height {height:,d} not found') \
+                    from None
+            return target, root_from_header, cost
+
+        def get_tx_position(tx_hash):
+            try:
+                tx_pos = tx_hashes.index(tx_hash)
+            except ValueError:
+                raise RPCError(BAD_REQUEST, f'tx {hash_to_hex_str(tx_hash)} not in block at height '
+                                            f'{height:,d}') from None
+            return tx_pos
+
+        async def get_txid_or_tx_field(tx_hash):
+            txid = hash_to_hex_str(tx_hash)
+            if txid_or_tx == "tx":
+                rawtx = await self.daemon_request('getrawtransaction', txid, False)
+                cost = 1.0
+                txid_or_tx_field = rawtx
+            else:
+                cost = 0.0
+                txid_or_tx_field = txid
+            return txid_or_tx_field, cost
+
+        tsc_proof = {}
+        tx_hashes, tx_hashes_cost = await self.tx_hashes_at_blockheight(height)
+        tx_pos = get_tx_position(tx_hash)
+        branch, root, merkle_cost = await self._merkle_branch(height, tx_hashes, tx_pos,
+                                                              tsc_format=True)
+
+        target, root_from_header, header_cost = await get_target(target_type)
+        # sanity check
+        if root != root_from_header:
+            raise RPCError(BAD_REQUEST, 'db error. Merkle root from cached block header does not '
+                                        'match the derived merkle root') from None
+
+        txid_or_tx_field, tx_fetch_cost = await get_txid_or_tx_field(tx_hash)
+
+        tsc_proof['index'] = tx_pos
+        tsc_proof['txid_or_tx'] = txid_or_tx_field
+        tsc_proof['target'] = target
+        tsc_proof['nodes'] = branch
+        return tsc_proof, tx_hashes_cost + merkle_cost + tx_fetch_cost + header_cost
+
     async def merkle_branch_for_tx_pos(self, height, tx_pos):
         '''Return a triple (branch, tx_hash_hex, cost).'''
         tx_hashes, tx_hashes_cost = await self.tx_hashes_at_blockheight(height)
@@ -688,7 +761,7 @@ async def merkle_branch_for_tx_pos(self, height, tx_pos):
             raise RPCError(
                 BAD_REQUEST, f'no tx at position {tx_pos:,d} in block at height {height:,d}'
             ) from None
-        branch, merkle_cost = await self._merkle_branch(height, tx_hashes, tx_pos)
+        branch, _root, merkle_cost = await self._merkle_branch(height, tx_hashes, tx_pos)
         return branch, hash_to_hex_str(tx_hash), tx_hashes_cost + merkle_cost
 
     async def tx_hashes_at_blockheight(self, height):
@@ -1375,6 +1448,32 @@ async def transaction_merkle(self, tx_hash, height):
 
         return {"block_height": height, "merkle": branch, "pos": tx_pos}
 
+    async def transaction_tsc_merkle(self, tx_hash, height, txid_or_tx='txid',
+                                     target_type='block_hash'):
+        '''Return the TSC merkle proof in JSON format to a confirmed transaction given its hash.
+        See: https://tsc.bitcoinassociation.net/standards/merkle-proof-standardised-format/.
+
+        tx_hash: the transaction hash as a hexadecimal string
+        include_tx: whether to include the full raw transaction in the response or txid.
+        target: options include: ('merkle_root', 'block_header', 'block_hash', 'None')
+        '''
+        tx_hash = assert_tx_hash(tx_hash)
+        height = non_negative_integer(height)
+
+        tsc_proof, cost = await self.session_mgr.tsc_merkle_proof_for_tx_hash(
+            height, tx_hash, txid_or_tx, target_type)
+        self.bump_cost(cost)
+
+        return {
+            "index": tsc_proof['index'],
+            "txOrId": tsc_proof['txid_or_tx'],
+            "target": tsc_proof['target'],
+            "nodes": tsc_proof['nodes'],  # "*" is used to represent duplicated hashes
+            "targetType": target_type,
+            "proofType": "branch",  # "tree" option is not supported by ElectrumX
+            "composite": False  # composite option is not supported by ElectrumX
+        }
+
     async def transaction_id_from_pos(self, height, tx_pos, merkle=False):
         '''Return the txid and optionally a merkle proof, given
         a block height and position in the block.
@@ -1421,6 +1520,7 @@ def set_request_handlers(self, ptuple):
             'blockchain.transaction.broadcast': self.transaction_broadcast,
             'blockchain.transaction.get': self.transaction_get,
             'blockchain.transaction.get_merkle': self.transaction_merkle,
+            'blockchain.transaction.get_tsc_merkle': self.transaction_tsc_merkle,
             'blockchain.transaction.id_from_pos': self.transaction_id_from_pos,
             'mempool.get_fee_histogram': self.compact_fee_histogram,
             'server.add_peer': self.add_peer,