Merge branch 'master' of https://github.com/ethereum/py-evm into limitIP

LICENSE

@@ -1,6 +1,6 @@
 The MIT License (MIT)
 
-Copyright 2017 Piper Merriam
+Copyright 2017-2018 Ethereum Foundation
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

docs/api/evm/api.db.rst

@@ -0,0 +1,13 @@
+DataBase
+========
+
+
+.. toctree::
+   :maxdepth: 4
+   :name: toc-evm-api-db
+   :caption: Database
+
+   db/api.db.backends
+   db/api.db.account
+   db/api.db.journal
+   db/api.db.chain

docs/api/evm/db/api.db.account.rst

@@ -0,0 +1,14 @@
+Account
+========
+
+BaseAccountDB
+-------------
+
+.. autoclass:: evm.db.account.BaseAccountDB
+  :members:
+
+AccountDB
+-------------
+
+.. autoclass:: evm.db.account.AccountDB
+  :members:

docs/api/evm/db/api.db.backends.rst

@@ -0,0 +1,20 @@
+Backends
+========
+
+BaseDB
+------
+
+.. autoclass:: evm.db.backends.base.BaseDB
+  :members:
+
+LevelDB
+-------
+
+.. autoclass:: evm.db.backends.level.LevelDB
+  :members:
+
+MemoryDB
+--------
+
+.. autoclass:: evm.db.backends.memory.MemoryDB
+  :members:

docs/api/evm/db/api.db.chain.rst

@@ -0,0 +1,20 @@
+Chain
+=====
+
+BaseChainDB
+~~~~~~~~~~~
+
+.. autoclass:: evm.db.chain.BaseChainDB
+  :members:
+
+ChainDB
+~~~~~~~
+
+.. autoclass:: evm.db.chain.ChainDB
+  :members:
+
+AsyncChainDB
+------------
+
+.. autoclass:: evm.db.chain.AsyncChainDB
+  :members:

docs/api/evm/db/api.db.journal.rst

@@ -0,0 +1,8 @@
+Journal
+=======
+
+JournalDB
+---------
+
+.. autoclass:: evm.db.journal.JournalDB
+  :members:

docs/api/evm/index.rst

@@ -10,5 +10,6 @@ This section aims to provide a detailed description of all APIs. If you are look
 
    api.chain
    api.computation
+   api.db
    api.state
    api.vm

docs/conf.py

@@ -59,7 +59,7 @@
 
 # General information about the project.
 project = about['__name__']
-copyright = '2017, Piper Merriam, Jason Carver'
+copyright = '2017-2018 Ethereum Foundation'
 author = about['__author__']
 
 # The version info for the project you're documenting, acts as replacement for

docs/guides/trinity/quickstart.rst

@@ -199,43 +199,3 @@ For a list of JSON-RPC endpoints which are expected to work, see this issue: htt
   - Only a subset of JSON-RPC API calls are currently supported
 
 
-Release Notes
-~~~~~~~~~~~~~
-
-
-0.1.0-alpha.11
---------------
-
-- Bugfix for ``PreferredNodePeerPool`` to respect ``max_peers``
-
-
-0.1.0-alpha.10
---------------
-
-- More bugfixes to enforce ``--max-peers`` in ``PeerPool._connect_to_nodes``
-
-
-0.1.0-alpha.9
--------------
-
-- Bugfix to enforce ``--max-peers`` for incoming connections.
-
-
-0.1.0-alpha.7
--------------
-
-- Remove ``min_peers`` concept from ``PeerPool``
-- Add ``--max-peers`` and enforcement of maximum peer connections maintained by
-  the ``PeerPool``.
-
-
-0.1.0-alpha.6
--------------
-
-- Respond to ``GetBlockHeaders`` message during fast sync to prevent being disconnected as a *useless peer*.
-- Add ``--profile`` CLI flag to Trinity to enable profiling via ``cProfile``
-- Better error messaging with Trinity cannot determine the appropriate location for the data directory.
-- Handle ``ListDeserializationError`` during handshake.
-- Add ``net_version`` JSON-RPC endpoint.
-- Add ``web3_clientVersion`` JSON-RPC endpoint.
-- Handle ``rlp.DecodingError`` during handshake.

docs/index.rst

@@ -10,6 +10,7 @@ Table of contents
 
    introduction
    quickstart
+   release_notes/index
 
 .. toctree::
    :maxdepth: 1

docs/release_notes/index.rst

@@ -0,0 +1,10 @@
+Release notes
+=============
+
+Trinity and Py-EVM are moving fast. Learn about the latest improvements in the release notes.
+
+.. toctree::
+   :maxdepth: 2
+   :name: toc-release-notes
+
+   trinity.rst

docs/release_notes/trinity.rst

@@ -0,0 +1,40 @@
+Trinity 
+=======
+
+
+0.1.0-alpha.11
+--------------
+
+- Bugfix for ``PreferredNodePeerPool`` to respect ``max_peers``
+
+
+0.1.0-alpha.10
+--------------
+
+- More bugfixes to enforce ``--max-peers`` in ``PeerPool._connect_to_nodes``
+
+
+0.1.0-alpha.9
+-------------
+
+- Bugfix to enforce ``--max-peers`` for incoming connections.
+
+
+0.1.0-alpha.7
+-------------
+
+- Remove ``min_peers`` concept from ``PeerPool``
+- Add ``--max-peers`` and enforcement of maximum peer connections maintained by
+  the ``PeerPool``.
+
+
+0.1.0-alpha.6
+-------------
+
+- Respond to ``GetBlockHeaders`` message during fast sync to prevent being disconnected as a *useless peer*.
+- Add ``--profile`` CLI flag to Trinity to enable profiling via ``cProfile``
+- Better error messaging with Trinity cannot determine the appropriate location for the data directory.
+- Handle ``ListDeserializationError`` during handshake.
+- Add ``net_version`` JSON-RPC endpoint.
+- Add ``web3_clientVersion`` JSON-RPC endpoint.
+- Handle ``rlp.DecodingError`` during handshake.

evm/utils/blobs.py

@@ -1,5 +1,3 @@
-import itertools
-import math
 from io import (
     BytesIO,
 )
@@ -14,11 +12,12 @@
     int_to_big_endian,
 )
 
-from eth_hash.auto import keccak
-
 from evm.utils.padding import (
     zpad_right,
 )
+from evm.utils.merkle import (
+    calc_merkle_root,
+)
 
 from evm.constants import (
     CHUNK_SIZE,
@@ -31,11 +30,6 @@
     ValidationError,
 )
 
-from cytoolz import (
-    partition,
-    pipe,
-)
-
 from typing import (
     cast,
 )
@@ -50,31 +44,9 @@ def iterate_chunks(collation_body: bytes) -> Iterator[Hash32]:
         yield cast(Hash32, collation_body[chunk_start:chunk_start + CHUNK_SIZE])
 
 
-def hash_layer(layer: Iterable[Hash32]) -> Iterator[Hash32]:
-    for left, right in partition(2, layer):
-        yield keccak(left + right)
-
-
-def calc_merkle_root(leaves: Iterable[Hash32]) -> Hash32:
-    leaves = list(leaves)  # convert potential iterator to list
-    if len(leaves) == 0:
-        raise ValidationError("No leaves given")
-
-    n_layers = math.log2(len(leaves))
-    if not n_layers.is_integer():
-        raise ValidationError("Leave number is not a power of two")
-    n_layers = int(n_layers)
-
-    first_layer = (keccak(leaf) for leaf in leaves)
-
-    root, *extras = pipe(first_layer, *itertools.repeat(hash_layer, n_layers))
-    assert not extras, "Invariant: should only be a single value"
-    return root
-
-
 def calc_chunk_root(collation_body: bytes) -> Hash32:
     check_body_size(collation_body)
-    chunks = iterate_chunks(collation_body)
+    chunks = list(iterate_chunks(collation_body))
     return calc_merkle_root(chunks)
 
 

evm/utils/merkle.py

@@ -0,0 +1,115 @@
+"""Utilities for binary merkle trees.
+
+Merkle trees are represented as sequences of layers, from root to leaves. The root layer contains
+only a single element, the leaves as many as there are data items in the tree. The data itself is
+not considered to be part of the tree.
+"""
+
+import math
+from typing import (
+    cast,
+    Hashable,
+    NewType,
+    Sequence,
+)
+
+from cytoolz import (
+    identity,
+    iterate,
+    partition,
+    reduce,
+    take,
+)
+from eth_hash.auto import (
+    keccak,
+)
+from eth_typing import (
+    Hash32,
+)
+
+from evm.exceptions import (
+    ValidationError,
+)
+
+
+MerkleTree = NewType("MerkleTree", Sequence[Sequence[Hash32]])
+MerkleProof = NewType("MerkleProof", Sequence[Hash32])
+
+
+def get_root(tree: MerkleTree) -> Hash32:
+    """Get the root hash of a Merkle tree."""
+    return tree[0][0]
+
+
+def get_branch_indices(node_index: int, depth: int) -> Sequence[int]:
+    """Get the indices of all ancestors up until the root for a node with a given depth."""
+    return tuple(take(depth, iterate(lambda index: index // 2, node_index)))
+
+
+def get_merkle_proof(tree: MerkleTree, item_index: int) -> Sequence[Hash32]:
+    """Read off the Merkle proof for an item from a Merkle tree."""
+    if item_index < 0 or item_index >= len(tree[-1]):
+        raise ValidationError("Item index out of range")
+
+    # special case of tree consisting of only root
+    if len(tree) == 1:
+        return ()
+
+    branch_indices = get_branch_indices(item_index, len(tree))
+    proof_indices = [i ^ 1 for i in branch_indices][:-1]  # get sibling by flipping rightmost bit
+    return tuple(
+        layer[proof_index]
+        for layer, proof_index
+        in zip(reversed(tree), proof_indices)
+    )
+
+
+def _calc_parent_hash(left_node: Hash32, right_node: Hash32) -> Hash32:
+    """Calculate the parent hash of a node and its sibling."""
+    return keccak(left_node + right_node)
+
+
+def verify_merkle_proof(root: Hash32,
+                        item: Hashable,
+                        item_index: int,
+                        proof: MerkleProof) -> bool:
+    """Verify a Merkle proof against a root hash."""
+    leaf = keccak(item)
+    branch_indices = get_branch_indices(item_index, len(proof))
+    node_orderers = [
+        identity if branch_index % 2 == 0 else reversed
+        for branch_index in branch_indices
+    ]
+    proof_root = reduce(
+        lambda n1, n2_and_order: _calc_parent_hash(*n2_and_order[1]([n1, n2_and_order[0]])),
+        zip(proof, node_orderers),
+        leaf,
+    )
+    return proof_root == root
+
+
+def _hash_layer(layer: Sequence[Hash32]) -> Sequence[Hash32]:
+    """Calculate the layer on top of another one."""
+    return tuple(_calc_parent_hash(left, right) for left, right in partition(2, layer))
+
+
+def calc_merkle_tree(items: Sequence[Hashable]) -> MerkleTree:
+    """Calculate the Merkle tree corresponding to a list of items."""
+    if len(items) == 0:
+        raise ValidationError("No items given")
+    n_layers = math.log2(len(items)) + 1
+    if not n_layers.is_integer():
+        raise ValidationError("Item number is not a power of two")
+    n_layers = int(n_layers)
+
+    leaves = tuple(keccak(item) for item in items)
+    tree = cast(MerkleTree, tuple(take(n_layers, iterate(_hash_layer, leaves)))[::-1])
+    if len(tree[0]) != 1:
+        raise Exception("Invariant: There must only be one root")
+
+    return tree
+
+
+def calc_merkle_root(items: Sequence[Hashable]) -> Hash32:
+    """Calculate the Merkle root corresponding to a list of items."""
+    return get_root(calc_merkle_tree(items))