Merge pull request #2 from ethereum/carver/optional-backends

Add pysha3 backend, better auto backend selection

.travis.yml

@@ -9,21 +9,45 @@ matrix:
     # lint
     - python: "3.5"
       env: TOX_POSARGS="-e lint"
+
+    # doctest
+    - python: "3.5"
+      env: TOX_POSARGS="-e doctest"
+
     # core
     - python: "3.5"
       env: TOX_POSARGS="-e py35-core"
+
+    # backends
+    - python: "3.5"
+      env: TOX_POSARGS="-e py35-backend-pycryptodome"
+    - python: "3.5"
+      env: TOX_POSARGS="-e py35-backend-pysha3"
+
     #
     # Python 3.6 testing
     #
     # core
     - python: "3.6"
       env: TOX_POSARGS="-e py36-core"
+
+    # backends
+    - python: "3.6"
+      env: TOX_POSARGS="-e py36-backend-pycryptodome"
+    - python: "3.6"
+      env: TOX_POSARGS="-e py36-backend-pysha3"
+
     #
     # pypy3 testing
     #
     # core
     - python: "pypy3"
       env: TOX_POSARGS="-e pypy3-core"
+
+    # backends
+    - python: "pypy3"
+      env: TOX_POSARGS="-e pypy3-backend-pycryptodome"
+
 cache:
   - pip: true
 install:

Makefile

@@ -35,9 +35,11 @@ test-all:
 	tox
 
 build-docs:
+	rm docs/eth_hash.backends.rst
 	sphinx-apidoc -o docs/ . setup.py "*conftest*"
 	$(MAKE) -C docs clean
 	$(MAKE) -C docs html
+	$(MAKE) -C docs doctest
 
 docs: build-docs
 	open docs/_build/html/index.html

docs/conf.py

@@ -34,7 +34,12 @@
 # Add any Sphinx extension module names here, as strings. They can be
 # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
 # ones.
-extensions = ['sphinx.ext.autodoc', 'sphinx.ext.intersphinx']
+extensions = [
+    'sphinx.ext.autodoc',
+    'sphinx.ext.autosectionlabel',
+    'sphinx.ext.doctest',
+    'sphinx.ext.intersphinx',
+]
 
 # Add any paths that contain templates here, relative to this directory.
 templates_path = ['_templates']
@@ -271,3 +276,14 @@
 intersphinx_mapping = {
     'python': ('https://docs.python.org/3.5', None),
 }
+
+# -- Doctest configuration ----------------------------------------
+
+import doctest
+
+doctest_default_flags = (0
+    | doctest.DONT_ACCEPT_TRUE_FOR_1
+    | doctest.ELLIPSIS
+    | doctest.IGNORE_EXCEPTION_DETAIL
+    | doctest.NORMALIZE_WHITESPACE
+)

docs/eth_hash.backends.rst

@@ -0,0 +1,30 @@
+eth\_hash\.backends package
+===========================
+
+Submodules
+----------
+
+eth\_hash\.backends\.pycryptodome module
+----------------------------------------
+
+.. automodule:: eth_hash.backends.pycryptodome
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+eth\_hash\.backends\.pysha3 module
+----------------------------------
+
+.. automodule:: eth_hash.backends.pysha3
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+
+Module contents
+---------------
+
+.. automodule:: eth_hash.backends
+    :members:
+    :undoc-members:
+    :show-inheritance:

docs/eth_hash.rst

@@ -0,0 +1,18 @@
+eth\_hash package
+=================
+
+eth\_hash\.auto module
+----------------------
+
+.. automodule:: eth_hash.auto
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+eth\_hash\.main module
+----------------------
+
+.. automodule:: eth_hash.main
+    :members:
+    :undoc-members:
+    :show-inheritance:

docs/index.rst

@@ -7,10 +7,12 @@ Contents
 --------
 
 .. toctree::
-    :maxdepth: 3
+    :maxdepth: 2
 
-    eth_hash
+    quickstart
     releases
+    eth_hash.backends
+    eth_hash
 
 
 Indices and tables

docs/quickstart.rst

@@ -0,0 +1,56 @@
+Quickstart
+============
+
+Choose a hashing backend
+---------------------------
+
+If you're not sure, choose "pycryptodome" because it supports pypy3.
+
+You can find a full list of each currently supported backend in :mod:`eth_hash.backends`.
+
+Install
+----------
+
+Put the backend you would like to use in brackets during install, like:
+
+.. code-block:: shell
+
+  pip install eth-hash[pycryptodome]
+
+Compute a Keccak256 Hash
+-----------------------------
+
+.. doctest::
+
+  >>> from eth_hash.auto import keccak
+  >>> keccak(b'')
+  b"\xc5\xd2F\x01\x86\xf7#<\x92~}\xb2\xdc\xc7\x03\xc0\xe5\x00\xb6S\xca\x82';{\xfa\xd8\x04]\x85\xa4p"
+
+Select one of many installed backends
+---------------------------------------
+
+If you have several backends installed, you may want to
+explicitly specify which one to load. You can specify
+in an environment variable, or at runtime.
+
+Specify backend by environment variable
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. code-block:: shell
+
+  $ ETH_HASH_BACKEND="pysha3" python
+  >>> from eth_hash.auto import keccak
+  # This runs with the pysha3 backend
+  >>> keccak(b'')
+  b"\xc5\xd2F\x01\x86\xf7#<\x92~}\xb2\xdc\xc7\x03\xc0\xe5\x00\xb6S\xca\x82';{\xfa\xd8\x04]\x85\xa4p"
+
+Specify backend at runtime
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. code-block:: python
+
+  >>> from eth_hash.backends import pysha3
+  >>> from eth_hash import Keccak256
+  >>> keccak = Keccak256(pysha3)
+  >>> keccak(b'')
+  b"\xc5\xd2F\x01\x86\xf7#<\x92~}\xb2\xdc\xc7\x03\xc0\xe5\x00\xb6S\xca\x82';{\xfa\xd8\x04]\x85\xa4p"

docs/releases.rst

@@ -1,6 +1,16 @@
 Release Notes
 =============
 
+v0.1.0-alpha.3
+--------------
+
+Released Feb 7, 2018
+
+- Add pycryptodome backend support
+- Add pysha3 backend support
+- Can specify backend in environment variable ``ETH_HASH_BACKEND``
+- New :ref:`Quickstart` docs
+
 v0.1.0-alpha.2
 --------------
 

eth_hash/auto.py

@@ -1,8 +1,8 @@
-from .backends import (
-    pycryptodome,
-)
-from .main import (
+from eth_hash.main import (
     Keccak256,
 )
+from eth_hash.utils import (
+    auto_choose_backend,
+)
 
-keccak = Keccak256(pycryptodome)
+keccak = Keccak256(auto_choose_backend())

eth_hash/backends/__init__.py

@@ -0,0 +1,13 @@
+'''
+A collection of optional backends. You must manually select and
+install the backend you want. If the backend is not installed,
+then trying to import the module for that backend will cause
+an :class:`ImportError`.
+
+See :ref:`Choose a hashing backend` for more.
+'''
+
+SUPPORTED_BACKENDS = [
+    'pycryptodome',  # prefer this over pysha3, for pypy3 support
+    'pysha3',
+]

eth_hash/backends/pycryptodome.py

@@ -4,6 +4,6 @@
 
 
 def keccak256(prehash: bytes) -> bytes:
-    hash = keccak.new(digest_bits=256)
-    hash.update(prehash)
-    return hash.digest()
+    hasher = keccak.new(digest_bits=256)
+    hasher.update(prehash)
+    return hasher.digest()

eth_hash/backends/pysha3.py

@@ -0,0 +1,7 @@
+from sha3 import (
+    keccak_256,
+)
+
+
+def keccak256(prehash: bytes) -> bytes:
+    return keccak_256(prehash).digest()

eth_hash/utils.py

@@ -0,0 +1,48 @@
+import importlib
+import os
+
+from eth_hash.backends import (
+    SUPPORTED_BACKENDS,
+)
+
+
+def auto_choose_backend():
+    env_backend = get_backend_in_environment()
+
+    if env_backend:
+        return load_environment_backend(env_backend)
+    else:
+        return choose_available_backend()
+
+
+def get_backend_in_environment():
+    return os.environ.get('ETH_HASH_BACKEND', None)
+
+
+def load_backend(backend_name):
+    return importlib.import_module('eth_hash.backends.%s' % backend_name)
+
+
+def load_environment_backend(env_backend):
+    if env_backend in SUPPORTED_BACKENDS:
+        try:
+            return load_backend(env_backend)
+        except ImportError as e:
+            raise ImportError(
+                "The backend specified in ETH_HASH_BACKEND, '{0}', is not installed. "
+                "Install with `pip install eth-hash[{0}]`.".format(env_backend)
+            )
+    else:
+        raise ValueError(
+            "The backend specified in ETH_HASH_BACKEND, %r, is not supported. "
+            "Choose one of: %r" % (env_backend, SUPPORTED_BACKENDS)
+        )
+
+
+def choose_available_backend():
+    for backend in SUPPORTED_BACKENDS:
+        try:
+            return load_backend(backend)
+        except ImportError:
+            pass
+    raise ImportError("None of these backends are available: %r" % SUPPORTED_BACKENDS)

setup.py

@@ -25,6 +25,13 @@
         "wheel",
         "ipython",
     ],
+    # optional backends:
+    'pycryptodome': [
+        "pycryptodome>=3.4.6,<4",
+    ],
+    'pysha3': [
+        "pysha3>=1.0.0,<2.0.0",
+    ],
 }
 
 extras_require['dev'] = (
@@ -45,7 +52,6 @@
     url='https://github.com/ethereum/eth-hash',
     include_package_data=True,
     install_requires=[
-        "pycryptodome>=3.4.6",
     ],
     setup_requires=['setuptools-markdown'],
     extras_require=extras_require,

tests/backends/pycryptodome/conftest.py

@@ -0,0 +1,17 @@
+import pytest
+
+
+@pytest.fixture(params=['auto', 'explicit', 'env'])
+def keccak(monkeypatch, request, keccak_auto):
+    if request.param == 'auto':
+        return keccak_auto
+    elif request.param == 'explicit':
+        from eth_hash.backends import pycryptodome
+        from eth_hash import Keccak256
+        return Keccak256(pycryptodome)
+    elif request.param == 'env':
+        monkeypatch.setenv('ETH_HASH_BACKEND', 'pycryptodome')
+        from eth_hash.auto import keccak
+        return keccak
+    else:
+        raise AssertionError("Unrecognized approach to import keccak: %s" % request.param)

tests/backends/pycryptodome/test_results.py

@@ -0,0 +1 @@
+../test_results.py

tests/backends/pysha3/conftest.py

@@ -0,0 +1,17 @@
+import pytest
+
+
+@pytest.fixture(params=['auto', 'explicit', 'env'])
+def keccak(monkeypatch, request, keccak_auto):
+    if request.param == 'auto':
+        return keccak_auto
+    elif request.param == 'explicit':
+        from eth_hash.backends import pysha3
+        from eth_hash import Keccak256
+        return Keccak256(pysha3)
+    elif request.param == 'env':
+        monkeypatch.setenv('ETH_HASH_BACKEND', 'pysha3')
+        from eth_hash.auto import keccak
+        return keccak
+    else:
+        raise AssertionError("Unrecognized approach to import keccak: %s" % request.param)

tests/backends/pysha3/test_results.py

@@ -0,0 +1 @@
+../test_results.py

tests/core/test_results.py → tests/backends/test_results.py

@@ -2,7 +2,7 @@
 
 
 @pytest.fixture
-def keccak():
+def keccak_auto():
     from eth_hash.auto import keccak
     return keccak