From cb8242be811305636a710e40d345333dbf0f5343 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Mon, 31 Dec 2018 13:59:51 +0800 Subject: [PATCH 1/4] Add `get_merkle_root` --- eth/_utils/merkle.py | 89 ++++++++++++++------ eth/beacon/_utils/hash.py | 6 +- tests/core/merkle-utils/test_merkle_trees.py | 63 ++++++++------ 3 files changed, 107 insertions(+), 51 deletions(-) diff --git a/eth/_utils/merkle.py b/eth/_utils/merkle.py index 31f1e03462..8475b2b953 100644 --- a/eth/_utils/merkle.py +++ b/eth/_utils/merkle.py @@ -9,6 +9,7 @@ from typing import ( cast, Hashable, + Iterable, NewType, Sequence, ) @@ -20,8 +21,8 @@ reduce, take, ) -from eth_hash.auto import ( - keccak, +from eth.beacon._utils.hash import ( + hash_eth2, ) from eth_typing import ( Hash32, @@ -36,17 +37,23 @@ def get_root(tree: MerkleTree) -> Hash32: - """Get the root hash of a Merkle tree.""" + """ + Get the root hash of a Merkle tree. + """ return tree[0][0] -def get_branch_indices(node_index: int, depth: int) -> Sequence[int]: - """Get the indices of all ancestors up until the root for a node with a given depth.""" +def get_branch_indices(node_index: int, depth: int) -> Iterable[int]: + """ + Get the indices of all ancestors up until the root for a node with a given depth. + """ return tuple(take(depth, iterate(lambda index: index // 2, node_index))) -def get_merkle_proof(tree: MerkleTree, item_index: int) -> Sequence[Hash32]: - """Read off the Merkle proof for an item from a Merkle tree.""" +def get_merkle_proof(tree: MerkleTree, item_index: int) -> Iterable[Hash32]: + """ + Read off the Merkle proof for an item from a Merkle tree. + """ if item_index < 0 or item_index >= len(tree[-1]): raise ValidationError("Item index out of range") @@ -64,16 +71,20 @@ def get_merkle_proof(tree: MerkleTree, item_index: int) -> Sequence[Hash32]: def _calc_parent_hash(left_node: Hash32, right_node: Hash32) -> Hash32: - """Calculate the parent hash of a node and its sibling.""" - return keccak(left_node + right_node) + """ + Calculate the parent hash of a node and its sibling. + """ + return hash_eth2(left_node + right_node) def verify_merkle_proof(root: Hash32, item: Hashable, item_index: int, proof: MerkleProof) -> bool: - """Verify a Merkle proof against a root hash.""" - leaf = keccak(item) + """ + Verify a Merkle proof against a root hash. + """ + leaf = hash_eth2(item) branch_indices = get_branch_indices(item_index, len(proof)) node_orderers = [ identity if branch_index % 2 == 0 else reversed @@ -87,28 +98,56 @@ def verify_merkle_proof(root: Hash32, return proof_root == root -def _hash_layer(layer: Sequence[Hash32]) -> Sequence[Hash32]: - """Calculate the layer on top of another one.""" - return tuple(_calc_parent_hash(left, right) for left, right in partition(2, layer)) +def _hash_layer(layer: Sequence[Hash32]) -> Iterable[Hash32]: + """ + Calculate the layer on top of another one. + """ + return tuple( + _calc_parent_hash(left, right) + for left, right in partition(2, layer) + ) def calc_merkle_tree(items: Sequence[Hashable]) -> MerkleTree: - """Calculate the Merkle tree corresponding to a list of items.""" - if len(items) == 0: - raise ValidationError("No items given") - n_layers = math.log2(len(items)) + 1 + """ + Calculate the Merkle tree corresponding to a list of items. + """ + leaves = tuple(hash_eth2(item) for item in items) + return calc_merkle_tree_from_leaves(leaves) + + +def calc_merkle_root(items: Sequence[Hashable]) -> Hash32: + """ + Calculate the Merkle root corresponding to a list of items. + """ + return get_root(calc_merkle_tree(items)) + + +def calc_merkle_tree_from_leaves(leaves: Sequence[Hash32]) -> MerkleTree: + if len(leaves) == 0: + raise ValueError("No leaves given") + n_layers = math.log2(len(leaves)) + 1 if not n_layers.is_integer(): - raise ValidationError("Item number is not a power of two") + raise ValueError("Number of leaves is not a power of two") n_layers = int(n_layers) - - leaves = tuple(keccak(item) for item in items) - tree = cast(MerkleTree, tuple(take(n_layers, iterate(_hash_layer, leaves)))[::-1]) + tree = cast( + MerkleTree, + tuple( + take( + n_layers, + iterate(_hash_layer, leaves), + ) + )[::-1] + ) if len(tree[0]) != 1: raise Exception("Invariant: There must only be one root") return tree -def calc_merkle_root(items: Sequence[Hashable]) -> Hash32: - """Calculate the Merkle root corresponding to a list of items.""" - return get_root(calc_merkle_tree(items)) +def get_merkle_root(leaves: Sequence[Hash32]) -> Hash32: + """ + Return the Merkle root of the given 32-byte hashes. + Note: it has to be a full tree, i.e., `len(values)` is an exact power of 2. + """ + return get_root(calc_merkle_tree_from_leaves(leaves)) diff --git a/eth/beacon/_utils/hash.py b/eth/beacon/_utils/hash.py index 8431a4a7fa..3924940581 100644 --- a/eth/beacon/_utils/hash.py +++ b/eth/beacon/_utils/hash.py @@ -1,8 +1,12 @@ +from typing import ( + Hashable, +) + from eth_typing import Hash32 from eth_hash.auto import keccak -def hash_eth2(data: bytes) -> Hash32: +def hash_eth2(data: Hashable) -> Hash32: """ Return Keccak-256 hashed result. Note: it's a placeholder and we aim to migrate to a S[T/N]ARK-friendly hash function in diff --git a/tests/core/merkle-utils/test_merkle_trees.py b/tests/core/merkle-utils/test_merkle_trees.py index 17625888e2..289206cdd7 100644 --- a/tests/core/merkle-utils/test_merkle_trees.py +++ b/tests/core/merkle-utils/test_merkle_trees.py @@ -4,8 +4,8 @@ ValidationError, ) -from eth_hash.auto import ( - keccak, +from eth.beacon._utils.hash import ( + hash_eth2, ) from eth._utils.merkle import ( @@ -13,6 +13,7 @@ calc_merkle_tree, get_root, get_merkle_proof, + get_merkle_root, verify_merkle_proof, ) @@ -21,41 +22,41 @@ ( (b"single leaf",), ( - (keccak(b"single leaf"),), + (hash_eth2(b"single leaf"),), ), ), ( (b"left", b"right"), ( - (keccak(keccak(b"left") + keccak(b"right")),), - (keccak(b"left"), keccak(b"right")), + (hash_eth2(hash_eth2(b"left") + hash_eth2(b"right")),), + (hash_eth2(b"left"), hash_eth2(b"right")), ), ), ( (b"1", b"2", b"3", b"4"), ( ( - keccak( - keccak( - keccak(b"1") + keccak(b"2") - ) + keccak( - keccak(b"3") + keccak(b"4") + hash_eth2( + hash_eth2( + hash_eth2(b"1") + hash_eth2(b"2") + ) + hash_eth2( + hash_eth2(b"3") + hash_eth2(b"4") ) ), ), ( - keccak( - keccak(b"1") + keccak(b"2") + hash_eth2( + hash_eth2(b"1") + hash_eth2(b"2") ), - keccak( - keccak(b"3") + keccak(b"4") + hash_eth2( + hash_eth2(b"3") + hash_eth2(b"4") ), ), ( - keccak(b"1"), - keccak(b"2"), - keccak(b"3"), - keccak(b"4"), + hash_eth2(b"1"), + hash_eth2(b"2"), + hash_eth2(b"3"), + hash_eth2(b"4"), ), ), ), @@ -69,7 +70,7 @@ def test_merkle_tree_calculation(leaves, tree): @pytest.mark.parametrize("leave_number", [0, 3, 5, 6, 7, 9]) def test_invalid_merkle_root_calculation(leave_number): - with pytest.raises(ValidationError): + with pytest.raises(ValueError): calc_merkle_root((b"",) * leave_number) @@ -77,32 +78,32 @@ def test_invalid_merkle_root_calculation(leave_number): ( (b"1", b"2"), 0, - (keccak(b"2"),), + (hash_eth2(b"2"),), ), ( (b"1", b"2"), 1, - (keccak(b"1"),), + (hash_eth2(b"1"),), ), ( (b"1", b"2", b"3", b"4"), 0, - (keccak(b"2"), keccak(keccak(b"3") + keccak(b"4"))), + (hash_eth2(b"2"), hash_eth2(hash_eth2(b"3") + hash_eth2(b"4"))), ), ( (b"1", b"2", b"3", b"4"), 1, - (keccak(b"1"), keccak(keccak(b"3") + keccak(b"4"))), + (hash_eth2(b"1"), hash_eth2(hash_eth2(b"3") + hash_eth2(b"4"))), ), ( (b"1", b"2", b"3", b"4"), 2, - (keccak(b"4"), keccak(keccak(b"1") + keccak(b"2"))), + (hash_eth2(b"4"), hash_eth2(hash_eth2(b"1") + hash_eth2(b"2"))), ), ( (b"1", b"2", b"3", b"4"), 3, - (keccak(b"3"), keccak(keccak(b"1") + keccak(b"2"))), + (hash_eth2(b"3"), hash_eth2(hash_eth2(b"1") + hash_eth2(b"2"))), ), ]) def test_merkle_proofs(leaves, index, proof): @@ -142,3 +143,15 @@ def test_proof_generation_index_validation(leaves): for invalid_index in [-1, len(leaves)]: with pytest.raises(ValidationError): get_merkle_proof(tree, invalid_index) + + +def test_get_merkle_root(): + hash_0 = b"0" * 32 + leaves = (hash_0,) + root = get_merkle_root(leaves) + assert root == hash_0 + + hash_1 = b"1" * 32 + leaves = (hash_0, hash_1) + root = get_merkle_root(leaves) + assert root == hash_eth2(hash_0 + hash_1) From 40243c87cae9a7e25a5c907eb12c3581ad871879 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Mon, 7 Jan 2019 23:56:55 +0800 Subject: [PATCH 2/4] PR feedback --- eth/_utils/merkle.py | 20 ++++++++++++-------- eth/beacon/_utils/hash.py | 4 ++-- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/eth/_utils/merkle.py b/eth/_utils/merkle.py index 8475b2b953..70d7cb5ce8 100644 --- a/eth/_utils/merkle.py +++ b/eth/_utils/merkle.py @@ -8,10 +8,10 @@ import math from typing import ( cast, - Hashable, Iterable, NewType, Sequence, + Union, ) from cytoolz import ( @@ -78,7 +78,7 @@ def _calc_parent_hash(left_node: Hash32, right_node: Hash32) -> Hash32: def verify_merkle_proof(root: Hash32, - item: Hashable, + item: Union[bytes, bytearray], item_index: int, proof: MerkleProof) -> bool: """ @@ -108,7 +108,7 @@ def _hash_layer(layer: Sequence[Hash32]) -> Iterable[Hash32]: ) -def calc_merkle_tree(items: Sequence[Hashable]) -> MerkleTree: +def calc_merkle_tree(items: Sequence[Union[bytes, bytearray]]) -> MerkleTree: """ Calculate the Merkle tree corresponding to a list of items. """ @@ -116,7 +116,7 @@ def calc_merkle_tree(items: Sequence[Hashable]) -> MerkleTree: return calc_merkle_tree_from_leaves(leaves) -def calc_merkle_root(items: Sequence[Hashable]) -> Hash32: +def calc_merkle_root(items: Sequence[Union[bytes, bytearray]]) -> Hash32: """ Calculate the Merkle root corresponding to a list of items. """ @@ -133,11 +133,15 @@ def calc_merkle_tree_from_leaves(leaves: Sequence[Hash32]) -> MerkleTree: tree = cast( MerkleTree, tuple( - take( - n_layers, - iterate(_hash_layer, leaves), + reversed( + tuple( + take( + n_layers, + iterate(_hash_layer, leaves), + ) + ) ) - )[::-1] + ) ) if len(tree[0]) != 1: raise Exception("Invariant: There must only be one root") diff --git a/eth/beacon/_utils/hash.py b/eth/beacon/_utils/hash.py index 3924940581..062e7787c4 100644 --- a/eth/beacon/_utils/hash.py +++ b/eth/beacon/_utils/hash.py @@ -1,12 +1,12 @@ from typing import ( - Hashable, + Union, ) from eth_typing import Hash32 from eth_hash.auto import keccak -def hash_eth2(data: Hashable) -> Hash32: +def hash_eth2(data: Union[bytes, bytearray]) -> Hash32: """ Return Keccak-256 hashed result. Note: it's a placeholder and we aim to migrate to a S[T/N]ARK-friendly hash function in From e5c1205d494ea7b2452bc5b7f27988868968a76c Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Tue, 8 Jan 2019 18:01:10 +0800 Subject: [PATCH 3/4] PR feedback --- eth/_utils/merkle.py | 18 ++++-------------- 1 file changed, 4 insertions(+), 14 deletions(-) diff --git a/eth/_utils/merkle.py b/eth/_utils/merkle.py index 70d7cb5ce8..2151fbba8c 100644 --- a/eth/_utils/merkle.py +++ b/eth/_utils/merkle.py @@ -7,7 +7,6 @@ import math from typing import ( - cast, Iterable, NewType, Sequence, @@ -130,19 +129,10 @@ def calc_merkle_tree_from_leaves(leaves: Sequence[Hash32]) -> MerkleTree: if not n_layers.is_integer(): raise ValueError("Number of leaves is not a power of two") n_layers = int(n_layers) - tree = cast( - MerkleTree, - tuple( - reversed( - tuple( - take( - n_layers, - iterate(_hash_layer, leaves), - ) - ) - ) - ) - ) + + reversed_tree = tuple(take(n_layers, iterate(_hash_layer, leaves))) + tree = MerkleTree(tuple(reversed(reversed_tree))) + if len(tree[0]) != 1: raise Exception("Invariant: There must only be one root") From 88445f1dc8c17679fb1a051311e4f539b70161fa Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Tue, 8 Jan 2019 18:26:10 +0800 Subject: [PATCH 4/4] Rename `calc_merkle_root` to `get_merkle_root_from_items` --- eth/_utils/blobs.py | 4 ++-- eth/_utils/merkle.py | 2 +- tests/core/merkle-utils/test_merkle_trees.py | 6 +++--- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/eth/_utils/blobs.py b/eth/_utils/blobs.py index cde48da79a..7a3ee264fa 100644 --- a/eth/_utils/blobs.py +++ b/eth/_utils/blobs.py @@ -17,7 +17,7 @@ zpad_right, ) from eth._utils.merkle import ( - calc_merkle_root, + get_merkle_root_from_items, ) from eth.constants import ( @@ -44,7 +44,7 @@ def iterate_chunks(collation_body: bytes) -> Iterator[Hash32]: def calc_chunk_root(collation_body: bytes) -> Hash32: check_body_size(collation_body) chunks = list(iterate_chunks(collation_body)) - return calc_merkle_root(chunks) + return get_merkle_root_from_items(chunks) def check_body_size(body: bytes) -> bytes: diff --git a/eth/_utils/merkle.py b/eth/_utils/merkle.py index 2151fbba8c..dff8d354c2 100644 --- a/eth/_utils/merkle.py +++ b/eth/_utils/merkle.py @@ -115,7 +115,7 @@ def calc_merkle_tree(items: Sequence[Union[bytes, bytearray]]) -> MerkleTree: return calc_merkle_tree_from_leaves(leaves) -def calc_merkle_root(items: Sequence[Union[bytes, bytearray]]) -> Hash32: +def get_merkle_root_from_items(items: Sequence[Union[bytes, bytearray]]) -> Hash32: """ Calculate the Merkle root corresponding to a list of items. """ diff --git a/tests/core/merkle-utils/test_merkle_trees.py b/tests/core/merkle-utils/test_merkle_trees.py index 289206cdd7..cbef512191 100644 --- a/tests/core/merkle-utils/test_merkle_trees.py +++ b/tests/core/merkle-utils/test_merkle_trees.py @@ -9,7 +9,7 @@ ) from eth._utils.merkle import ( - calc_merkle_root, + get_merkle_root_from_items, calc_merkle_tree, get_root, get_merkle_proof, @@ -65,13 +65,13 @@ def test_merkle_tree_calculation(leaves, tree): calculated_tree = calc_merkle_tree(leaves) assert calculated_tree == tree assert get_root(tree) == tree[0][0] - assert calc_merkle_root(leaves) == get_root(tree) + assert get_merkle_root_from_items(leaves) == get_root(tree) @pytest.mark.parametrize("leave_number", [0, 3, 5, 6, 7, 9]) def test_invalid_merkle_root_calculation(leave_number): with pytest.raises(ValueError): - calc_merkle_root((b"",) * leave_number) + get_merkle_root_from_items((b"",) * leave_number) @pytest.mark.parametrize("leaves,index,proof", [