Merge pull request #153 from pipermerriam/piper/reuse-requests-connec…

…tions

Reuses sessions with requests based RPC interactsion

tests/caching-utils/test_generate_cache_key.py

@@ -0,0 +1,52 @@
+import pytest
+from hypothesis import (
+    given,
+    strategies as st,
+)
+
+import random
+
+from web3.utils.functional import (
+    cast_return_to_dict,
+)
+from web3.utils.caching import (
+    generate_cache_key,
+)
+
+
+@cast_return_to_dict
+def shuffle_dict(_dict):
+    keys = list(_dict.keys())
+    random.shuffle(keys)
+    for key in keys:
+        yield key, _dict[key]
+
+
+def extend_fn(children):
+    lists_st = st.lists(children)
+    dicts_st = st.dictionaries(st.text(), children)
+    return lists_st | dicts_st
+
+
+all_st = st.recursive(
+    st.none() | st.integers() | st.booleans() | st.floats() | st.text() | st.binary(),
+    extend_fn,
+)
+
+
+def recursive_shuffle_dict(v):
+    if isinstance(v, dict):
+        return shuffle_dict(v)
+    elif isinstance(v, (list, tuple)):
+        return type(v)((recursive_shuffle_dict(_v) for _v in v))
+    else:
+        return v
+
+
+@given(value=all_st)
+def test_key_generation_is_deterministic(value):
+    left = recursive_shuffle_dict(value)
+    right = recursive_shuffle_dict(value)
+    left_key = generate_cache_key(left)
+    right_key = generate_cache_key(right)
+    assert left_key == right_key

tests/utilities/test_types_utils.py

@@ -46,7 +46,7 @@ def test_is_string(value, expected):
     ("3", False),
     ("0x3", False),
     ({}, True),
-    ({"test": 3}, True)
+    ({"test": 3}, True),
     ]
 )
 def test_is_object(value, expected):
@@ -62,7 +62,7 @@ def test_is_object(value, expected):
     ("3", False),
     ("0x3", False),
     (True, True),
-    (False, True)
+    (False, True),
     ]
 )
 def test_is_boolean(value, expected):
@@ -72,16 +72,18 @@ def test_is_boolean(value, expected):
 @pytest.mark.parametrize(
     "value,expected",
     [
-    (lambda : None, False),
-    (3, False),
-    (None, False),
-    ("3", False),
-    ("0x3", False),
-    ([], True),
-    ([3], True),
-    ([3, 3], True),
-    ([None], True)
+        (lambda : None, False),
+        (3, False),
+        (None, False),
+        ("3", False),
+        ("0x3", False),
+        ([], True),
+        ([3], True),
+        ([3, 3], True),
+        ([None], True),
+        ([tuple()], True),
+        ([(1, 2)], True),
     ]
 )
-def test_isArray(value, expected):
+def test_is_array(value, expected):
     assert is_array(value) == expected

web3/utils/caching.py

@@ -0,0 +1,46 @@
+import hashlib
+
+from .types import (
+    is_boolean,
+    is_null,
+    is_object,
+    is_array,
+    is_number,
+    is_text,
+    is_bytes,
+)
+from .string import (
+    force_bytes,
+)
+from .compat import (
+    Generator,
+)
+
+
+def generate_cache_key(value):
+    """
+    Generates a cache key for the *args and **kwargs
+    """
+    if is_bytes(value):
+        return hashlib.md5(value).hexdigest()
+    elif is_text(value):
+        return generate_cache_key(force_bytes(value))
+    elif is_boolean(value) or is_null(value) or is_number(value):
+        return generate_cache_key(repr(value))
+    elif is_object(value):
+        return generate_cache_key((
+            (key, value[key])
+            for key
+            in sorted(value.keys())
+        ))
+    elif is_array(value) or isinstance(value, Generator):
+        return generate_cache_key("".join((
+            generate_cache_key(item)
+            for item
+            in value
+        )))
+    else:
+        raise TypeError("Cannot generate cache key for value {0} of type {1}".format(
+            value,
+            type(value),
+        ))

web3/utils/compat/__init__.py

@@ -6,11 +6,13 @@
     from .compat_py2 import (
         urlparse,
         urlunparse,
+        Generator,
     )
 else:
     from .compat_py3 import (  # noqa: #401
         urlparse,
         urlunparse,
+        Generator,
     )
 
 

web3/utils/compat/compat_py2.py

@@ -2,3 +2,4 @@
     urlparse,
     urlunparse,
 )
+Generator = type(_ for _ in tuple())

web3/utils/compat/compat_py3.py

@@ -1,4 +1,12 @@
+import collections
+
 from urllib.parse import (  # noqa: F401
     urlparse,
     urlunparse,
 )
+
+try:
+    Generator = collections.Generator
+except AttributeError:
+    # py34
+    Generator = type(_ for _ in tuple())

web3/utils/compat/compat_requests.py

@@ -1,9 +1,24 @@
 import requests
 
+import pylru
+
+from web3.utils.caching import generate_cache_key
+
+
+_session_cache = pylru.lrucache(8)
+
+
+def _get_session(*args, **kwargs):
+    cache_key = generate_cache_key((args, kwargs))
+    if cache_key not in _session_cache:
+        _session_cache[cache_key] = requests.Session()
+    return _session_cache[cache_key]
+
 
 def make_post_request(endpoint_uri, data, *args, **kwargs):
     kwargs.setdefault('timeout', 10)
-    response = requests.post(endpoint_uri, data=data, *args, **kwargs)
+    session = _get_session(endpoint_uri)
+    response = session.post(endpoint_uri, data=data, *args, **kwargs)
     response.raise_for_status()
 
     return response.content

web3/utils/string.py

@@ -12,7 +12,10 @@ def force_bytes(value):
     if is_bytes(value):
         return bytes(value)
     elif is_text(value):
-        return codecs.encode(value, "iso-8859-1")
+        try:
+            return codecs.encode(value, "iso-8859-1")
+        except UnicodeEncodeError:
+            return codecs.encode(value, "utf8")
     else:
         raise TypeError("Unsupported type: {0}".format(type(value)))
 

web3/utils/types.py

@@ -1,4 +1,6 @@
 import sys
+import numbers
+import collections
 
 
 if sys.version_info.major == 2:
@@ -34,12 +36,16 @@ def is_boolean(value):
 
 
 def is_object(obj):
-    return isinstance(obj, dict)
+    return isinstance(obj, collections.Mapping)
 
 
 def is_array(obj):
-    return isinstance(obj, list)
+    return not is_string(obj) and isinstance(obj, collections.Sequence)
 
 
 def is_null(obj):
     return obj is None
+
+
+def is_number(obj):
+    return isinstance(obj, numbers.Number)