Merge pull request #15 from HarryR/unique-tx-events

Fixes #4 - Duplicate merkle leafs

Makefile

@@ -22,14 +22,16 @@ end-to-end:
 	make -C solidity background-testrpc-b
 	make -C solidity deploy-b
 
-	make -C python lithium-a2b &
-	make -C python lithium-b2a &
+	make -C python lithium-a2b > /dev/null &
+	make -C python lithium-b2a > /dev/null &
 	make -C python example-pingpong
-	make -C python lithium-a2b-stop
-	make -C python lithium-b2a-stop
+	make stop-end-to-end
 
-	make -C solidity stop-testrpc-a
-	make -C solidity stop-testrpc-b
+stop-end-to-end:
+	make -C python lithium-a2b-stop || true
+	make -C python lithium-b2a-stop || true
+	make -C solidity stop-testrpc-a || true
+	make -C solidity stop-testrpc-b || true
 
 
 #######################################################################

README.md

@@ -15,6 +15,7 @@ Think of this as being a cross-chain oracle for events and transactions, or it c
 
 ### General
 
+ * [ ] Improved logging in Python code
  * [ ] Getting started guide
  * [x] ExamplePingPong FSM diagram + documentation
  * [ ] ExampleSwap FSM diagram
@@ -30,7 +31,7 @@ Think of this as being a cross-chain oracle for events and transactions, or it c
 ### Toolchain ToDo
 
  * [x] Travis CI
- * [ ] End-to-end tests under Travis
+ * [x] End-to-end tests under Travis
  * [ ] Python coverage / Codeclimate
  * [x] Solidity code coverage
  * [ ] Docker image

python/panautomata/__main__.py

@@ -1,5 +1,8 @@
 import click
 
+
+# TODO: setup logger here, must setup logger before modules are imported
+
 from .lithium.cli import daemon as lithium_daemon
 from .example.swap import COMMANDS as swap_commands
 

python/panautomata/example/pingpong.py

@@ -40,7 +40,8 @@ def main():
 
     print("Start")
     tx = alice.Start(guid, session)
-    tx.wait()
+    start_tx_receipt = tx.wait()
+    print("Start TX receipt", start_tx_receipt)
     start_proof = proof_for_tx(rpc_a, tx)
 
     print("ReceiveStart")
@@ -60,7 +61,8 @@ def main():
         link_wait(link_b, pong_proof)
         tx = bob.ReceivePong(guid, pong_proof)
         ping_proof = proof_for_event(rpc_b, tx, 0)
-        tx.wait()
+        receieve_pong_receipt = tx.wait()
+        print("ReceivePong receipt", receieve_pong_receipt)
 
 
 if __name__ == "__main__":

python/panautomata/lithium/common.py

@@ -4,13 +4,30 @@
 
 import time
 
-from sha3 import keccak_256
-
+from ..crypto import keccak_256
 from ..ethrpc import EthTransaction
-from ..utils import scan_bin, require, u256be, bytes_to_int
+from ..utils import scan_bin, require, u256be, u64be, u32be, bytes_to_int
 from ..merkle import merkle_tree, merkle_path, merkle_proof
 
 
+def pack_prefix(txn_or_log, log_idx=None):
+    """
+    Prefix for merkle leaves, which binds them to a specific log, transaction at
+    a block height.
+    """
+    if log_idx is None:
+        log_idx = int(txn_or_log.get('logIndex', '0x0'), 16)
+    assert isinstance(log_idx, int)
+
+    tx_block_height = int(txn_or_log['blockNumber'], 16)
+    tx_index = int(txn_or_log['transactionIndex'], 16)
+
+    result = u64be(tx_block_height) + u32be(tx_index) + u32be(log_idx)
+    require(len(result) == 16)
+
+    return result
+
+
 def pack_txn(txn):
     """
     Packs all the information about a transaction into a deterministic fixed-sized array of bytes
@@ -23,14 +40,15 @@ def pack_txn(txn):
     tx_input = scan_bin(txn['input'])
 
     # 104 bytes
-    result = b''.join([
+    inner_leaf = b''.join([
         tx_from,
         tx_to,
         u256be(tx_value),
         keccak_256(tx_input).digest()
     ])
-    assert len(result) == 104
-    return result
+    require(len(inner_leaf) == 104)
+
+    return pack_prefix(txn) + keccak_256(inner_leaf).digest()
 
 
 def pack_log(log):
@@ -48,13 +66,14 @@ def pack_log(log):
     Event type is a hash of the event signature, e.g. KECCAK256('MyEvent(address,uint256)')
     """
     # 84 bytes
-    result = b''.join([
+    inner_leaf = b''.join([
         scan_bin(log['address']),
         scan_bin(log['topics'][0]),
         keccak_256(scan_bin(log['data'])).digest()
     ])
-    assert len(result) == 84
-    return result
+    require(len(inner_leaf) == 84)
+
+    return pack_prefix(log) + keccak_256(inner_leaf).digest()
 
 
 def process_logs(rpc, tx_hash):
@@ -125,6 +144,7 @@ def proof_for_event(rpc, tx_hash, log_idx):
     tx_block_height = int(transaction['blockNumber'], 16)
 
     block_items, block_tx_count, block_log_count = process_block(rpc, tx_block_height)
+
     tree, root = merkle_tree(block_items)
 
     # TODO: verify the merkle root matches the on-chain merkle root submitted to LithiumLink
@@ -134,7 +154,8 @@ def proof_for_event(rpc, tx_hash, log_idx):
     require(merkle_proof(event_leaf, proof, root) is True, "Cannot confirm merkle proof")
 
     # Proof as accepted by LithiumProver instance
-    return u256be(tx_block_height) + b''.join([u256be(_) for _ in proof])
+    prefix = pack_prefix(transaction, log_idx)
+    return prefix + b''.join([u256be(_) for _ in proof])
 
 
 def proof_for_tx(rpc, tx_hash):
@@ -148,6 +169,7 @@ def proof_for_tx(rpc, tx_hash):
     tx_block_height = int(transaction['blockNumber'], 16)
 
     block_items, block_tx_count, block_log_count = process_block(rpc, tx_block_height)
+
     tree, root = merkle_tree(block_items)
 
     # TODO: verify the merkle root matches the on-chain merkle root submitted to LithiumLink
@@ -156,14 +178,15 @@ def proof_for_tx(rpc, tx_hash):
     require(merkle_proof(tx_leaf, proof, root) is True, "Cannot confirm merkle proof")
 
     # Proof as accepted by LithiumProver instance
-    return u256be(tx_block_height) + b''.join([u256be(_) for _ in proof])
+    prefix = pack_prefix(transaction)
+    return prefix + b''.join([u256be(_) for _ in proof])
 
 
 def verify_proof(root, leaf, proof):
-    require(len(proof) % 32 == 0)
-    require(len(proof) >= 64)
-    # block_height = proof[:32]
-    proof = proof[32:]
+    # Prefix is 16 bytes
+    require((len(proof) - 16) % 32 == 0)
+    require((len(proof) - 16) >= 32)
+    proof = proof[16:]
     path = []
     while len(proof):
         path.append(bytes_to_int(proof[:32]))
@@ -172,16 +195,15 @@ def verify_proof(root, leaf, proof):
     return merkle_proof(leaf, path, root)
 
 
-def link_wait(link_contract, proof):
+def link_wait(link_contract, proof, interval=1):
     """
     Wait for the LithiumLink contract to reach the height required
     to validate the proof.
     """
-    block_height = bytes_to_int(proof[:32])
-    print("waiting for block height", block_height)
+    block_height = bytes_to_int(proof[:8])
     while True:
         latest_block = link_contract.GetHeight()
         if latest_block >= block_height:
             break
-        print(".", latest_block)
-        time.sleep(1)
+        time.sleep(interval)
+        print("Waiting for block", block_height)

python/panautomata/utils.py

@@ -63,9 +63,20 @@ def zpad(x, l):
 
 
 def u256be(v):
+    # TODO: verify range
     return zpad(int_to_big_endian(v), 32)
 
 
+def u64be(v):
+    # TODO: verify range
+    return zpad(int_to_big_endian(v), 8)
+
+
+def u32be(v):
+    # TODO: verify range
+    return zpad(int_to_big_endian(v), 4)
+
+
 def flatten(l):
     return [item for sublist in l for item in sublist]
 

python/test/fakerpc.py

@@ -0,0 +1,17 @@
+
+class FakeRPC(object):
+    def __init__(self, in_blocks, in_transactions, in_receipts):
+        self._blocks_by_height = {int(_['number'], 16): _ for _ in in_blocks}
+        #self._blocks_by_hash = {_['hash']: _ for _ in in_blocks}
+        self._transactions = {_['hash']: _ for _ in in_transactions}
+        self._receipts = {_['transactionHash']: _ for _ in in_receipts}
+
+    def eth_getTransactionByHash(self, tx_hash):
+        return self._transactions[tx_hash]
+
+    def eth_getTransactionReceipt(self, tx_hash):
+        return self._receipts[tx_hash]
+
+    def eth_getBlockByNumber(self, block_height, tx_objects=True):
+        assert tx_objects is False  # is only used this way
+        return self._blocks_by_height[block_height]

python/test/test_lithium_common.py

@@ -1,14 +1,44 @@
 import unittest
 
-from binascii import unhexlify
+from binascii import unhexlify, hexlify
 
 from panautomata.utils import bytes_to_int
-from panautomata.lithium.common import verify_proof
+from panautomata.merkle import merkle_tree
+from panautomata.lithium.common import verify_proof, process_block, proof_for_tx, process_transaction
+
+from fakerpc import FakeRPC
+
+
+FAKERPC_INSTANCE = FakeRPC(
+        [   # blocks
+            {'number': '0xa', 'hash': '0x0ecee24d0107cfaa2eb4977d9a9c76e91c955b504820a15130928c180f3d3615', 'parentHash': '0xb35f11abd3f133da554106422688e84785bd973eaf4ab70d38245ae789711b2c', 'mixHash': '0x0000000000000000000000000000000000000000000000000000000000000000', 'nonce': '0x0000000000000000', 'sha3Uncles': '0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347', 'logsBloom': '0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000', 'transactionsRoot': '0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421', 'stateRoot': '0x66003709fc92807a818cf2c245b03303f5f1f3640d9cad334c16adb2be27fcec', 'receiptsRoot': '0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421', 'miner': '0x0000000000000000000000000000000000000000', 'difficulty': '0x0', 'totalDifficulty': '0x0', 'extraData': '0x', 'size': '0x3e8', 'gasLimit': '0xfffffff', 'gasUsed': '0x2095c', 'timestamp': '0x5b400a52', 'transactions': ['0x87f2dd1a154c8f11a153bdcd90fc67ab850e9f32f05a5becc79d3fe035b1c4fd'], 'uncles': []}
+        ],
+        [   # transactions
+            {'hash': '0x87f2dd1a154c8f11a153bdcd90fc67ab850e9f32f05a5becc79d3fe035b1c4fd', 'nonce': '0x09', 'blockHash': '0x0ecee24d0107cfaa2eb4977d9a9c76e91c955b504820a15130928c180f3d3615', 'blockNumber': '0x0a', 'transactionIndex': '0x0', 'from': '0x90f8bf6a479f320ead074411a4b0e7944ea8c9c1', 'to': '0xd833215cbcc3f914bd1c9ece3ee7bf8b14f841bb', 'value': '0x0', 'gas': '0x0dbba0', 'gasPrice': '0x0ba43b7400', 'input': '0x79a821d92b7763986e1a1724ddf52242eedd060cdec61fa11fd57c0eea3190653b19773b000000000000000000000000e982e462b094850f12af94d21d470e21be9d0e9c0000000000000000000000000000000000000000000000000000000000000001000000000000000000000000d833215cbcc3f914bd1c9ece3ee7bf8b14f841bb000000000000000000000000e982e462b094850f12af94d21d470e21be9d0e9c00000000000000000000000000000000000000000000000000000000000000010000000000000000000000009561c133dd8580860b6b7e504bc5aa500f0f06a70000000000000000000000000000000000000000000000000000000000000001'}
+        ],
+        [   # receipts
+            {'transactionHash': '0x87f2dd1a154c8f11a153bdcd90fc67ab850e9f32f05a5becc79d3fe035b1c4fd', 'transactionIndex': '0x0', 'blockHash': '0x0ecee24d0107cfaa2eb4977d9a9c76e91c955b504820a15130928c180f3d3615', 'blockNumber': '0xa', 'gasUsed': '0x2095c', 'cumulativeGasUsed': '0x2095c', 'contractAddress': None, 'logs': [], 'status': '0x1', 'logsBloom': '0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000'}
+        ]
+    )
 
 
 class TestLithiumCommon(unittest.TestCase):
-	def test_proof(self):
-		root = 36309678569118526148872637980630051425706212671800593816880788491663080654976
-		leaf = unhexlify(b'90f8bf6a479f320ead074411a4b0e7944ea8c9c1d833215cbcc3f914bd1c9ece3ee7bf8b14f841bb0000000000000000000000000000000000000000000000000000000000000000eda0650b624856b39dcb6a3319d210ce4aed935b7bed35ed098ec46b96596cf4')
-		proof = unhexlify(b'000000000000000000000000000000000000000000000000000000000000001ee6843bf393570479a2656a819bf8d219c1dde89fe0b6f73c6299bf202fe755d1')
-		self.assertEqual(verify_proof(root, leaf, proof), True)
+    def test_block(self):
+        items, tx_count, log_count = process_block(FAKERPC_INSTANCE, 10)
+        self.assertEqual(bytes_to_int(items[0]), 21359870359209100823974240316624348723385542012113319844900486161227333295717347719776129330291061)
+        self.assertEqual(tx_count, 1)
+        self.assertEqual(log_count, 0)
+
+    def test_proof_tx(self):
+        block_items, block_tx_count, block_log_count = process_block(FAKERPC_INSTANCE, 10)
+        tree, root = merkle_tree(block_items)
+
+        tx_hash = '0x87f2dd1a154c8f11a153bdcd90fc67ab850e9f32f05a5becc79d3fe035b1c4fd'
+        leaf = process_transaction(FAKERPC_INSTANCE, tx_hash)
+        self.assertEqual(leaf, b'\x00\x00\x00\x00\x00\x00\x00\n\x00\x00\x00\x00\x00\x00\x00\x005\x1c\xae1jW\x1a\xd6\r8\xb8\xf8vf\xd7\x9eG1Kf\xca\xfd\x00\xce\xbc\xa4\xd0\xb6\xf45\xa1u')
+
+        proof = proof_for_tx(FAKERPC_INSTANCE, tx_hash)
+        self.assertEqual(len(proof), 48)
+        self.assertEqual(proof, b'\x00\x00\x00\x00\x00\x00\x00\n\x00\x00\x00\x00\x00\x00\x00\x00\xe6\x84;\xf3\x93W\x04y\xa2ej\x81\x9b\xf8\xd2\x19\xc1\xdd\xe8\x9f\xe0\xb6\xf7<b\x99\xbf /\xe7U\xd1')
+
+        self.assertEqual(verify_proof(root, leaf, proof), True)

solidity/Makefile

@@ -16,7 +16,7 @@ all: check-prereqs node_modules lint test coverage
 .PHONY: abi
 abi: $(ABI_FILES)
 
-clean: clean-coverage stop-testrpc-a stop-testrpc-a
+clean: clean-coverage stop-testrpc-a stop-testrpc-b
 	rm -rf build *.log
 
 clean-coverage:
@@ -92,3 +92,8 @@ console: $(TRUFFLE)
 .PHONY: coverage
 coverage: clean-coverage
 	$(NPM) run coverage
+
+# Retrieve static built solidity compiler for Linux (useful...)
+solc-static-linux:
+	wget -O $@ "https://github.com/ethereum/solidity/releases/download/v$(shell ./utils/get-package-version.py package.json solc)/solc-static-linux" || rm -f $@
+	chmod 755 $@