Merge pull request #11 from ethereum/develop

merge develop

.travis.yml

@@ -1,6 +1,20 @@
 language: python
+
 python:
   - "2.7"
+  - "3.3"
   - "3.4"
-install: python setup.py install
-script: py.test
+  - "pypy"
+  - "pypy3"
+
+# command to install dependencies, e.g. pip install -r requirements.txt --use-mirrors
+install:
+ - pip install -r requirements.txt
+ - pip install coveralls
+
+# command to run tests, e.g. python setup.py test
+script:
+  - coverage run --source pyethapp setup.py test
+
+after_success:
+  - coveralls

LICENSE

@@ -1,6 +1,6 @@
 The MIT License (MIT)
 
-Copyright (c) {{{year}}} {{{fullname}}}
+Copyright (c) 2015 Jnnk, Vitalik Buterin
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

README.md

@@ -1,6 +1,10 @@
 pyrlp
 =====
 
+[![Build Status](https://travis-ci.org/ethereum/pyrlp.svg?branch=develop)](https://travis-ci.org/ethereum/pyrlp)
+[![Coverage Status](https://coveralls.io/repos/ethereum/pyrlp/badge.svg)](https://coveralls.io/r/ethereum/pyrlp)
+[![PyPI version](https://badge.fury.io/py/rlp.svg)](http://badge.fury.io/py/rlp)
+
 A Python implementation of Recursive Length Prefix encoding (RLP). You can find
 the specification of the standard in the
 [Ethereum wiki](https://github.com/ethereum/wiki/wiki/RLP) and the

rlp/__init__.py

@@ -2,5 +2,5 @@
 from .codec import encode, decode, infer_sedes
 from .exceptions import RLPException, EncodingError, DecodingError,  \
                         SerializationError, DeserializationError
-from .lazy import decode_lazy, LazyList
+from .lazy import decode_lazy, peek, LazyList
 from .sedes import Serializable

rlp/codec.py

@@ -1,8 +1,9 @@
 import collections
 import sys
 from .exceptions import EncodingError, DecodingError
-from .utils import (Atomic, str_to_bytes, is_integer, bytes_to_int_array,
-                    ascii_chr)
+
+from .utils import (Atomic, str_to_bytes, is_integer, ascii_chr, safe_ord, big_endian_to_int,
+                    int_to_big_endian)
 from .sedes.binary import Binary as BinaryClass
 from .sedes import big_endian_int, binary
 from .sedes.lists import List, is_sedes
@@ -40,10 +41,18 @@ def encode(obj, sedes=None, infer_serializer=True):
     return encode_raw(item)
 
 
+class RLPData(str):
+
+    "wraper to mark already rlp serialized data"
+    pass
+
+
 def encode_raw(item):
     """RLP encode (a nested sequence of) :class:`Atomic`s."""
-    if isinstance(item, Atomic):
-        if len(item) == 1 and bytes_to_int_array(item)[0] < 128:
+    if isinstance(item, RLPData):
+        return item
+    elif isinstance(item, Atomic):
+        if len(item) == 1 and safe_ord(item[0]) < 128:
             return str_to_bytes(item)
         payload = str_to_bytes(item)
         prefix_offset = 128  # string
@@ -72,7 +81,7 @@ def length_prefix(length, offset):
     if length < 56:
         return ascii_chr(offset + length)
     elif length < 256**8:
-        length_string = big_endian_int.serialize(length)
+        length_string = int_to_big_endian(length)
         return ascii_chr(offset + 56 - 1 + len(length_string)) + length_string
     else:
         raise ValueError('Length greater than 256**8')
@@ -88,23 +97,26 @@ def consume_length_prefix(rlp, start):
               ``length`` is the length of the payload in bytes, and ``end`` is
               the position of the first payload byte in the rlp string
     """
-    if isinstance(rlp, str):
-        rlp = str_to_bytes(rlp)
-
-    b0 = bytes_to_int_array(rlp)[start]
+    b0 = safe_ord(rlp[start])
     if b0 < 128:  # single byte
         return (str, 1, start)
     elif b0 < 128 + 56:  # short string
+        if b0 - 128 == 1 and safe_ord(rlp[start + 1]) < 128:
+            raise DecodingError('Encoded as short string although single byte was possible', rlp)
         return (str, b0 - 128, start + 1)
     elif b0 < 192:  # long string
         ll = b0 - 128 - 56 + 1
-        l = big_endian_int.deserialize(rlp[start + 1:start + 1 + ll])
+        if rlp[start + 1:start + 2] == b'\x00':
+            raise DecodingError('Length starts with zero bytes', rlp)
+        l = big_endian_to_int(rlp[start + 1:start + 1 + ll])
         return (str, l, start + 1 + ll)
     elif b0 < 192 + 56:  # short list
         return (list, b0 - 192, start + 1)
     else:  # long list
         ll = b0 - 192 - 56 + 1
-        l = big_endian_int.deserialize(rlp[start + 1:start + 1 + ll])
+        if rlp[start + 1:start + 2] == b'\x00':
+            raise DecodingError('Length starts with zero bytes', rlp)
+        l = big_endian_to_int(rlp[start + 1:start + 1 + ll])
         if l < 56:
             raise DecodingError('Long list prefix used for short list', rlp)
         return (list, l, start + 1 + ll)
@@ -149,24 +161,27 @@ def consume_item(rlp, start):
     return consume_payload(rlp, s, t, l)
 
 
-def decode(rlp, sedes=None, **kwargs):
+def decode(rlp, sedes=None, strict=True, **kwargs):
     """Decode an RLP encoded object.
 
     :param sedes: an object implementing a function ``deserialize(code)`` which
                   will be applied after decoding, or ``None`` if no
                   deserialization should be performed
     :param \*\*kwargs: additional keyword arguments that will be passed to the
                      deserializer
+    :param strict: if false inputs that are longer than necessary don't cause
+                   an exception
     :returns: the decoded and maybe deserialized Python object
     :raises: :exc:`rlp.DecodingError` if the input string does not end after
-             the root item
+             the root item and `strict` is true
     :raises: :exc:`rlp.DeserializationError` if the deserialization fails
     """
+    rlp = str_to_bytes(rlp)
     try:
         item, end = consume_item(rlp, 0)
     except IndexError:
         raise DecodingError('RLP string to short', rlp)
-    if end != len(rlp):
+    if end != len(rlp) and strict:
         msg = 'RLP string ends with {} superfluous bytes'.format(len(rlp) - end)
         raise DecodingError(msg, rlp)
     if sedes:
@@ -179,8 +194,8 @@ def infer_sedes(obj):
     """Try to find a sedes objects suitable for a given Python object.
 
     The sedes objects considered are `obj`'s class, `big_endian_int` and
-    `binary`. If `obj` is a sequence, a :class:`ListSedes` will be constructed
-    recursively.
+    `binary`. If `obj` is a sequence, a :class:`rlp.sedes.List` will be
+    constructed recursively.
 
     :param obj: the python object for which to find a sedes object
     :raises: :exc:`TypeError` if no appropriate sedes could be found

rlp/lazy.py

@@ -1,6 +1,7 @@
-from collections import Sequence
-from .exceptions import DecodingError
+from collections import Iterable, Sequence
 from .codec import consume_length_prefix, consume_payload
+from .exceptions import DecodingError
+from .utils import Atomic
 
 
 def decode_lazy(rlp, sedes=None, **sedes_kwargs):
@@ -18,9 +19,9 @@ def decode_lazy(rlp, sedes=None, **sedes_kwargs):
     "vertical lazyness" can be preserved.
 
     :param rlp: the RLP string to decode
-    :param sedes: an object implementing a method ``deserialize(code)``
-                          which is used as described above, or ``None`` if no
-                          deserialization should be performed
+    :param sedes: an object implementing a method ``deserialize(code)`` which
+                  is used as described above, or ``None`` if no
+                  deserialization should be performed
     :param \*\*sedes_kwargs: additional keyword arguments that will be passed
                              to the deserializers
     :returns: either the already decoded and deserialized object (if encoded as
@@ -81,37 +82,72 @@ def __init__(self, rlp, start, end, sedes=None, **sedes_kwargs):
         self.start = start
         self.end = end
         self.index = start
-        self.elements_ = []
+        self._elements = []
         self.len_ = None
         self.sedes = sedes
         self.sedes_kwargs = sedes_kwargs
 
     def next(self):
         if self.index == self.end:
-            self.len_ = len(self.elements_)
+            self.len_ = len(self._elements)
             raise StopIteration
         assert self.index < self.end
         item, end = consume_item_lazy(self.rlp, self.index)
         self.index = end
         if self.sedes:
             item = self.sedes.deserialize(item, **self.sedes_kwargs)
-        self.elements_.append(item)
+        self._elements.append(item)
         return item
 
     def __getitem__(self, i):
         try:
-            while len(self.elements_) <= i:
+            while len(self._elements) <= i:
                 self.next()
         except StopIteration:
             assert self.index == self.end
             raise IndexError('Index %d out of range' % i)
-        return self.elements_[i]
+        return self._elements[i]
 
     def __len__(self):
         if not self.len_:
             try:
                 while True:
                     self.next()
             except StopIteration:
-                self.len_ = len(self.elements_)
-        return self.len_
+                self._len = len(self._elements)
+        return self._len
+
+
+def peek(rlp, index, sedes=None):
+    """Get a specific element from an rlp encoded nested list.
+
+    This function uses :func:`rlp.decode_lazy` and, thus, decodes only the
+    necessary parts of the string.
+
+    Usage example::
+
+        >>> rlpdata = rlp.encode([1, 2, [3, [4, 5]]])
+        >>> rlp.peek(rlpdata, 0, rlp.sedes.big_endian_int)
+        1
+        >>> rlp.peek(rlpdata, [2, 0], rlp.sedes.big_endian_int)
+        3
+
+    :param rlp: the rlp string
+    :param index: the index of the element to peek at (can be a list for
+                  nested data)
+    :param sedes: a sedes used to deserialize the peeked at object, or `None`
+                  if no deserialization should be performed
+    :raises: :exc:`IndexError` if `index` is invalid (out of range or too many
+             levels)
+    """
+    ll = decode_lazy(rlp)
+    if not isinstance(index, Iterable):
+        index = [index]
+    for i in index:
+        if isinstance(ll, Atomic):
+            raise IndexError('Too many indices given')
+        ll = ll[i]
+    if sedes:
+        return sedes.deserialize(ll)
+    else:
+        return ll

rlp/sedes/big_endian_int.py

@@ -1,5 +1,5 @@
 from ..exceptions import DeserializationError, SerializationError
-from ..utils import int_to_big_endian, is_integer, encode_hex
+from ..utils import int_to_big_endian, big_endian_to_int, is_integer, ascii_chr
 
 
 class BigEndianInt(object):
@@ -35,11 +35,11 @@ def deserialize(self, serial):
         if self.l is not None and len(serial) != self.l:
             raise DeserializationError('Invalid serialization (wrong size)',
                                        serial)
-        if self.l is None and len(serial) > 1 and serial[0] == 0:
+        if self.l is None and len(serial) > 0 and serial[0:1] == ascii_chr(0):
             raise DeserializationError('Invalid serialization (not minimal '
                                        'length)', serial)
 
         serial = serial or b'\x00'
-        return int(encode_hex(serial), 16)
+        return big_endian_to_int(serial)
 
 big_endian_int = BigEndianInt()

rlp/sedes/binary.py

@@ -53,7 +53,7 @@ def deserialize(self, serial):
             raise DeserializationError(m.format(type(serial).__name__), serial)
 
         if self.is_valid_length(len(serial)):
-            return bytes_to_str(serial)
+            return serial
         else:
             raise DeserializationError('{} has invalid length'.format(type(serial)), serial)
 

rlp/sedes/lists.py

@@ -14,7 +14,8 @@ def is_sedes(obj):
     A sedes object is characterized by having the methods `serialize(obj)` and
     `deserialize(serial)`.
     """
-    return all(hasattr(obj, m) for m in ('serialize', 'deserialize'))
+    # return all(hasattr(obj, m) for m in ('serialize', 'deserialize'))
+    return hasattr(obj, 'serialize') and hasattr(obj, 'deserialize')
 
 
 def is_sequence(obj):
@@ -23,10 +24,18 @@ def is_sequence(obj):
 
 
 class List(list):
-    """A sedes for lists, implemented as a list of other sedes objects."""
 
-    def __init__(self, elements=[]):
+    """A sedes for lists, implemented as a list of other sedes objects.
+
+    :param strict: If true (de)serializing lists that have a length not
+                   matching the sedes length will result in an error. If false
+                   (de)serialization will stop as soon as either one of the
+                   lists runs out of elements.
+    """
+
+    def __init__(self, elements=[], strict=True):
         super(List, self).__init__()
+        self.strict = strict
         for e in elements:
             if is_sedes(e):
                 self.append(e)
@@ -39,7 +48,7 @@ def __init__(self, elements=[]):
     def serialize(self, obj):
         if not is_sequence(obj):
             raise SerializationError('Can only serialize sequences', obj)
-        if len(self) != len(obj):
+        if self.strict and len(self) != len(obj) or len(self) < len(obj):
             raise SerializationError('List has wrong length', obj)
         return [sedes.serialize(element)
                 for element, sedes in zip(obj, self)]
@@ -48,13 +57,14 @@ def deserialize(self, serial):
         if not is_sequence(serial):
             raise DeserializationError('Can only deserialize sequences',
                                        serial)
-        if len(serial) != len(self):
+        if len(serial) > len(self) or self.strict and len(serial) != len(self):
             raise DeserializationError('List has wrong length', serial)
         return [sedes.deserialize(element)
                 for element, sedes in zip(serial, self)]
 
 
 class CountableList(object):
+
     """A sedes for lists of arbitrary length.
 
     :param element_sedes: when (de-)serializing a list, this sedes will be
@@ -77,6 +87,7 @@ def deserialize(self, serial):
 
 
 class Serializable(object):
+
     """Base class for objects which can be serialized into RLP lists.
 
     :attr:`fields` defines which attributes are serialized and how this is
@@ -134,25 +145,25 @@ def get_sedes(cls):
     @classmethod
     def serialize(cls, obj):
         if not hasattr(obj, 'fields'):
-            raise SerializationError('Cannot serialize this object', obj)
+            raise SerializationError('Cannot serialize this object (no fields)', obj)
         try:
             field_values = [getattr(obj, field) for field, _ in cls.fields]
         except AttributeError:
-            raise SerializationError('Cannot serialize this object', obj)
+            raise SerializationError('Cannot serialize this object (missing attribute)', obj)
         return cls.get_sedes().serialize(field_values)
 
     @classmethod
     def deserialize(cls, serial, **kwargs):
         values = cls.get_sedes().deserialize(serial)
         params = {field: value for (field, _), value
-                               in zip(cls.fields, values)}
+                  in zip(cls.fields, values)}
         return cls(**dict(list(params.items()) + list(kwargs.items())))
 
     @classmethod
     def exclude(cls, excluded_fields):
         """Create a new sedes considering only a reduced set of fields."""
         class SerializableExcluded(cls):
             fields = [(field, sedes) for field, sedes in cls.fields
-                                     if field not in excluded_fields]
+                      if field not in excluded_fields]
             _sedes = None
         return SerializableExcluded