Merge pull request #95 from carver/dynamic-arrays-v1

Dynamic array support

docs/index.rst

@@ -18,6 +18,7 @@ Table of Contents
     overview
     encoding
     decoding
+    nested_dynamic_arrays
     eth_abi
     releases
 

docs/nested_dynamic_arrays.rst

@@ -0,0 +1,12 @@
+Nested Dynamic Arrays
+=====================
+
+The ``eth-abi`` library supports the Solidity ABIv2 encoding format for nested
+dynamic arrays.  This means that values for data types such as the following
+are legal and encodable/decodable: ``int[][]``, ``string[]``, ``string[2]``,
+etc.
+
+.. warning::
+
+    Though Solidity's ABIv2 has mostly been finalized, the specification is
+    technically still in development and may change.

eth_abi/base.py

@@ -12,8 +12,8 @@ def parse_type_str(expected_base=None, with_arrlist=False):
     """
     Used by BaseCoder subclasses as a convenience for implementing the
     ``from_type_str`` method required by ``ABIRegistry``.  Useful if normalizing
-    then parsing a basic type string with an expected base is required in that
-    method.
+    then parsing a type string with an (optional) expected base is required in
+    that method.
     """
     def decorator(old_from_type_str):
         @functools.wraps(old_from_type_str)
@@ -28,23 +28,23 @@ def new_from_type_str(cls, type_str, registry):
                     repr(normalized_type_str),
                 )
 
-            if not isinstance(abi_type, BasicType):
-                raise ValueError(
-                    'Cannot create {} for non-basic type {}'.format(
-                        cls.__name__,
-                        type_str_repr,
+            if expected_base is not None:
+                if not isinstance(abi_type, BasicType):
+                    raise ValueError(
+                        'Cannot create {} for non-basic type {}'.format(
+                            cls.__name__,
+                            type_str_repr,
+                        )
                     )
-                )
-
-            if expected_base is not None and abi_type.base != expected_base:
-                raise ValueError(
-                    'Cannot create {} for type {}: expected type with '
-                    "base '{}'".format(
-                        cls.__name__,
-                        type_str_repr,
-                        expected_base,
+                if abi_type.base != expected_base:
+                    raise ValueError(
+                        'Cannot create {} for type {}: expected type with '
+                        "base '{}'".format(
+                            cls.__name__,
+                            type_str_repr,
+                            expected_base,
+                        )
                     )
-                )
 
             if not with_arrlist and abi_type.arrlist is not None:
                 raise ValueError(

eth_abi/decoding.py

@@ -200,6 +200,15 @@ def split_data_and_padding(self, raw_data):
 class BaseArrayDecoder(BaseDecoder):
     item_decoder = None
 
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+
+        # Use a head-tail decoder to decode dynamic elements
+        if self.item_decoder.is_dynamic:
+            self.item_decoder = HeadTailDecoder(
+                tail_decoder=self.item_decoder,
+            )
+
     def validate(self):
         super().validate()
 
@@ -225,20 +234,28 @@ def from_type_str(cls, abi_type, registry):
 class SizedArrayDecoder(BaseArrayDecoder):
     array_size = None
 
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+
+        self.is_dynamic = self.item_decoder.is_dynamic
+
     @to_tuple
     def decode(self, stream):
         for _ in range(self.array_size):
             yield self.item_decoder(stream)
 
 
 class DynamicArrayDecoder(BaseArrayDecoder):
+    # Dynamic arrays are always dynamic, regardless of their elements
     is_dynamic = True
 
     @to_tuple
     def decode(self, stream):
         array_size = decode_uint_256(stream)
+        stream.push_frame(32)
         for _ in range(array_size):
             yield self.item_decoder(stream)
+        stream.pop_frame()
 
 
 class FixedByteSizeDecoder(SingleDecoder):

eth_abi/encoding.py

@@ -598,12 +598,20 @@ def validate_value(self, value):
     def encode_elements(self, value):
         self.validate_value(value)
 
-        encoded_elements = b''.join((
-            self.item_encoder(item)
-            for item in value
-        ))
+        item_encoder = self.item_encoder
+        tail_chunks = tuple(item_encoder(i) for i in value)
 
-        return encoded_elements
+        items_are_dynamic = getattr(item_encoder, 'is_dynamic', False)
+        if not items_are_dynamic:
+            return b''.join(tail_chunks)
+
+        head_length = 32 * len(value)
+        tail_offsets = (0,) + tuple(accumulate(map(len, tail_chunks[:-1])))
+        head_chunks = tuple(
+            encode_uint_256(head_length + offset)
+            for offset in tail_offsets
+        )
+        return b''.join(head_chunks + tail_chunks)
 
     @parse_type_str(with_arrlist=True)
     def from_type_str(cls, abi_type, registry):
@@ -624,6 +632,11 @@ def from_type_str(cls, abi_type, registry):
 class SizedArrayEncoder(BaseArrayEncoder):
     array_size = None
 
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+
+        self.is_dynamic = self.item_encoder.is_dynamic
+
     def validate(self):
         super().validate()
 

eth_abi/grammar.py

@@ -13,7 +13,8 @@
 grammar = parsimonious.Grammar(r"""
 type = tuple_type / basic_type
 
-tuple_type = non_zero_tuple / zero_tuple
+tuple_type = components arrlist?
+components = non_zero_tuple / zero_tuple
 
 non_zero_tuple = "(" type next_type* ")"
 next_type = "," type
@@ -47,7 +48,11 @@ def visit_non_zero_tuple(self, node, visited_children):
         # Ignore left and right parens
         _, first, rest, _ = visited_children
 
-        return TupleType((first,) + rest, node=node)
+        return (first,) + rest
+
+    def visit_tuple_type(self, node, visited_children):
+        components, arrlist = visited_children
+        return TupleType(components, arrlist=arrlist, node=node)
 
     def visit_next_type(self, node, visited_children):
         # Ignore comma
@@ -56,7 +61,7 @@ def visit_next_type(self, node, visited_children):
         return abi_type
 
     def visit_zero_tuple(self, node, visited_children):
-        return TupleType(tuple(), node=node)
+        return tuple()
 
     def visit_basic_type(self, node, visited_children):
         base, sub, arrlist = visited_children
@@ -121,9 +126,12 @@ class ABIType:
     Base class for classes which represent the results of parsing operations on
     abi type strings after post-processing.
     """
-    __slots__ = ('node',)
+    __slots__ = ('arrlist', 'node')
+
+    def __init__(self, arrlist=None, node=None):
+        # Any type might have a list of array dimensions
+        self.arrlist = arrlist
 
-    def __init__(self, *, node=None):
         # The parsimonious `Node` instance associated with this parsed type may
         # be optionally included.  If a type must be validated during a parsing
         # operation, the `Node` instance is required since the `invalidate`
@@ -185,13 +193,34 @@ class TupleType(ABIType):
     """
     __slots__ = ('components',)
 
-    def __init__(self, components, *, node=None):
-        super().__init__(node=node)
+    def __init__(self, components, arrlist=None, *, node=None):
+        super().__init__(arrlist, node)
 
         self.components = components
 
     def __str__(self):
-        return '({})'.format(','.join(str(c) for c in self.components))
+        arrlist = self.arrlist
+        if isinstance(arrlist, tuple):
+            arrlist = ''.join(repr(list(a)) for a in arrlist)
+        else:
+            arrlist = ''
+        return '({}){}'.format(','.join(str(c) for c in self.components), arrlist)
+
+    @property
+    def item_type(self):
+        """
+        If this type is an array type, returns the type of the array's items.
+        """
+        if self.arrlist is None:
+            raise ValueError(
+                "Cannot determine item type for non-array type '{}'".format(self)
+            )
+
+        return type(self)(
+            self.components,
+            self.arrlist[:-1] or None,
+            node=self.node,
+        )
 
     def validate(self):
         # A tuple type is valid if all of its components are valid i.e. if none
@@ -207,14 +236,13 @@ class BasicType(ABIType):
 
     e.g. "uint", "address", "ufixed128x19[][2]"
     """
-    __slots__ = ('base', 'sub', 'arrlist')
+    __slots__ = ('base', 'sub')
 
     def __init__(self, base, sub=None, arrlist=None, *, node=None):
-        super().__init__(node=node)
+        super().__init__(arrlist, node)
 
         self.base = base
         self.sub = sub
-        self.arrlist = arrlist
 
     def __str__(self):
         sub, arrlist = self.sub, self.arrlist

eth_abi/registry.py

@@ -208,29 +208,26 @@ def __str__(self):
 
 def has_arrlist(type_str):
     """
-    A predicate which matches a basic type string with an array dimension list.
+    A predicate which matches a type string with an array dimension list.
     """
     try:
         abi_type = grammar.parse(type_str)
     except exceptions.ParseError:
         return False
 
-    return (
-        isinstance(abi_type, grammar.BasicType)
-        and abi_type.arrlist is not None
-    )
+    return abi_type.arrlist is not None
 
 
-def is_tuple_type(type_str):
+def is_base_tuple(type_str):
     """
-    A predicate which matches a tuple type string.
+    A predicate which matches a tuple type with no array dimension list.
     """
     try:
         abi_type = grammar.parse(type_str)
     except exceptions.ParseError:
         return False
 
-    return isinstance(abi_type, grammar.TupleType)
+    return isinstance(abi_type, grammar.TupleType) and abi_type.arrlist is None
 
 
 def _clear_encoder_cache(old_method):
@@ -403,7 +400,7 @@ def get_decoder(self, type_str):
     label='has_arrlist',
 )
 registry.register(
-    is_tuple_type,
+    is_base_tuple,
     encoding.TupleEncoder, decoding.TupleDecoder,
-    label='is_tuple_type',
+    label='is_base_tuple',
 )

eth_abi/utils/padding.py

@@ -1,4 +1,4 @@
-from cytoolz import (
+from eth_utils.toolz import (
     curry,
 )
 

setup.py

@@ -17,7 +17,8 @@
     url='https://github.com/ethereum/eth-abi',
     include_package_data=True,
     install_requires=[
-        'eth-utils>=1.0.1,<2.0.0',
+        'eth-utils>=1.2.0,<2.0.0',
+        'eth-typing<=2',
         'parsimonious==0.8.0',
     ],
     extras_require={