[revised, tests added] added support for buffer / memoryview data

src/genpy/generate_struct.py

@@ -186,3 +186,17 @@ def unpack3(var, struct_var, buff):
     :param buff: buffer that the unpack reads from, ``StringIO``
     """
     return '%s = %s.unpack(%s)' % (var, struct_var, buff)
+
+
+def memoryview_len(view):
+    """
+    Compute the size (in bytes) of a ``memoryview`` object.
+
+    This is the same as memoryview.nbytes, but compatible with Python 2.
+
+    :param view: The ``memoryview`` object.
+    """
+    length = view.itemsize
+    for s in view.shape:
+        length *= s
+    return length

src/genpy/generator.py

@@ -77,6 +77,7 @@
 from . generate_struct import pack
 from . generate_struct import pack2
 from . generate_struct import reduce_pattern
+from . generate_struct import serialize as serializeit
 from . generate_struct import unpack
 from . generate_struct import unpack2
 from . generate_struct import unpack3
@@ -469,17 +470,37 @@ def string_serializer_generator(package, type_, name, serialize):  # noqa: D401
         # check to see if its a uint8/byte type, in which case we need to convert to string before serializing
         base_type, is_array, array_len = genmsg.msgs.parse_type(type_)
         if base_type in ['uint8', 'char']:
-            yield '# - if encoded as a list instead, serialize as bytes instead of string'
+            yield '# check for buffer protocol support'
             if array_len is None:
-                yield 'if type(%s) in [list, tuple]:' % var
-                yield INDENT+pack2("'<I%sB'%length", 'length, *%s' % var)
-                yield 'else:'
-                yield INDENT+pack2("'<I%ss'%length", 'length, %s' % var)
+                yield 'try:'
+                yield INDENT+'tmp = memoryview(%s)' % var
+                yield INDENT+'from genpy.generate_struct import memoryview_len'
+                yield INDENT+pack2("'<I'", "memoryview_len(tmp)")
+                yield INDENT+serializeit('tmp')
+                yield 'except TypeError:'
+                yield INDENT+'# - if encoded as a list instead, serialize as bytes instead of string'
+                yield INDENT+'if type(%s) in [list, tuple]:' % var
+                yield INDENT+INDENT+pack2("'<I%sB'%length", 'length, *%s' % var)
+                yield INDENT+'else:'
+                yield INDENT+INDENT+pack2("'<I%ss'%length", 'length, %s' % var)
             else:
-                yield 'if type(%s) in [list, tuple]:' % var
-                yield INDENT+pack('%sB' % array_len, '*%s' % var)
+                yield 'try:'
+                yield INDENT+'# we want to process str and bytes by the except clause'
+                yield INDENT+'if isinstance(%s, bytes) or isinstance(%s, str):' % (var, var)
+                yield INDENT+INDENT+'raise TypeError()'
+                yield INDENT+'tmp = memoryview(%s)' % var
+                yield 'except TypeError:'
+                yield INDENT+'# - if encoded as a list instead, serialize as bytes instead of string'
+                yield INDENT+'if type(%s) in [list, tuple]:' % var
+                yield INDENT+INDENT+pack('%sB' % array_len, '*%s' % var)
+                yield INDENT+'else:'
+                yield INDENT+INDENT+pack('%ss' % array_len, var)
                 yield 'else:'
-                yield INDENT+pack('%ss' % array_len, var)
+                yield INDENT+'from genpy.generate_struct import memoryview_len'
+                yield INDENT+'tmp_len = memoryview_len(tmp)'
+                yield INDENT+'if tmp_len != %i:' % array_len
+                yield INDENT+INDENT+'raise TypeError("expected %i bytes, got %%i" %% (tmp_len,))' % array_len
+                yield INDENT+serializeit('tmp[:%i]' % array_len)
         else:
             # FIXME: for py3k, this needs to be w/ encode(), but this interferes with actual byte data
             yield 'if python3 or type(%s) == unicode:' % (var)

src/genpy/message.py

@@ -48,6 +48,7 @@
 import yaml
 
 from .base import is_simple
+from .generate_struct import memoryview_len
 from .rostime import Duration
 from .rostime import TVal
 from .rostime import Time
@@ -299,14 +300,25 @@ def check_type(field_name, field_type, field_val):
         base_type = field_type[:field_type.index('[')]
 
         if type(field_val) in (bytes, str):
-            if base_type not in ['char', 'uint8']:
+            if base_type not in ('char', 'uint8'):
                 raise SerializationError('field %s must be a list or tuple type. Only uint8[] can be a string' % field_name)
             else:
                 # It's a string so its already in byte format and we
                 # don't need to check the individual bytes in the
                 # string.
                 return
-
+        if base_type in ('char', 'uint8'):
+            # check buffer protocol support
+            try:
+                length = memoryview_len(memoryview(field_val))
+                _, _, array_len = genmsg.msgs.parse_type(field_type)
+                if array_len is not None and length != array_len:
+                    raise SerializationError('field %s must receive %i bytes, but %i given' % (
+                        field_name, array_len, length))
+                # if the value supports buffer protocol, pass it directly
+                return
+            except TypeError:
+                pass  # does not support buffer protocol
         if not type(field_val) in [list, tuple]:
             raise SerializationError('field %s must be a list or tuple type' % field_name)
         for v in field_val:

test/files/array/uint8_fixed_ser.txt

@@ -1,5 +1,18 @@
-# - if encoded as a list instead, serialize as bytes instead of string
-if type(data) in [list, tuple]:
-  buff.write(_get_struct_8B().pack(*data))
+# check for buffer protocol support
+try:
+  # we want to process str and bytes by the except clause
+  if isinstance(data, bytes) or isinstance(data, str):
+    raise TypeError()
+  tmp = memoryview(data)
+except TypeError:
+  # - if encoded as a list instead, serialize as bytes instead of string
+  if type(data) in [list, tuple]:
+    buff.write(_get_struct_8B().pack(*data))
+  else:
+    buff.write(_get_struct_8s().pack(data))
 else:
-  buff.write(_get_struct_8s().pack(data))
+  from genpy.generate_struct import memoryview_len
+  tmp_len = memoryview_len(tmp)
+  if tmp_len != 8:
+    raise TypeError("expected 8 bytes, got %i" % (tmp_len,))
+  buff.write(tmp[:8])

test/files/array/uint8_fixed_ser_np.txt

@@ -1,5 +1,18 @@
-# - if encoded as a list instead, serialize as bytes instead of string
-if type(data) in [list, tuple]:
-  buff.write(_get_struct_8B().pack(*data))
+# check for buffer protocol support
+try:
+  # we want to process str and bytes by the except clause
+  if isinstance(data, bytes) or isinstance(data, str):
+    raise TypeError()
+  tmp = memoryview(data)
+except TypeError:
+  # - if encoded as a list instead, serialize as bytes instead of string
+  if type(data) in [list, tuple]:
+    buff.write(_get_struct_8B().pack(*data))
+  else:
+    buff.write(_get_struct_8s().pack(data))
 else:
-  buff.write(_get_struct_8s().pack(data))
+  from genpy.generate_struct import memoryview_len
+  tmp_len = memoryview_len(tmp)
+  if tmp_len != 8:
+    raise TypeError("expected 8 bytes, got %i" % (tmp_len,))
+  buff.write(tmp[:8])

test/files/array/uint8_varlen_ser.txt

@@ -1,6 +1,13 @@
 length = len(data)
-# - if encoded as a list instead, serialize as bytes instead of string
-if type(data) in [list, tuple]:
-  buff.write(struct.Struct('<I%sB'%length).pack(length, *data))
-else:
-  buff.write(struct.Struct('<I%ss'%length).pack(length, data))
+# check for buffer protocol support
+try:
+  tmp = memoryview(data)
+  from genpy.generate_struct import memoryview_len
+  buff.write(struct.Struct('<I').pack(memoryview_len(tmp)))
+  buff.write(tmp)
+except TypeError:
+  # - if encoded as a list instead, serialize as bytes instead of string
+  if type(data) in [list, tuple]:
+    buff.write(struct.Struct('<I%sB'%length).pack(length, *data))
+  else:
+    buff.write(struct.Struct('<I%ss'%length).pack(length, data))

test/files/array/uint8_varlen_ser_np.txt

@@ -1,6 +1,13 @@
 length = len(data)
-# - if encoded as a list instead, serialize as bytes instead of string
-if type(data) in [list, tuple]:
-  buff.write(struct.Struct('<I%sB'%length).pack(length, *data))
-else:
-  buff.write(struct.Struct('<I%ss'%length).pack(length, data))
+# check for buffer protocol support
+try:
+  tmp = memoryview(data)
+  from genpy.generate_struct import memoryview_len
+  buff.write(struct.Struct('<I').pack(memoryview_len(tmp)))
+  buff.write(tmp)
+except TypeError:
+  # - if encoded as a list instead, serialize as bytes instead of string
+  if type(data) in [list, tuple]:
+    buff.write(struct.Struct('<I%sB'%length).pack(length, *data))
+  else:
+    buff.write(struct.Struct('<I%ss'%length).pack(length, data))

test/msg/TestBinary.msg

@@ -0,0 +1,2 @@
+uint8[] data_var
+uint8[4] data_fixed

test/test_genpy_generator.py

@@ -318,11 +318,18 @@ def test_string_serializer_generator():
     for t in ['uint8[]', 'byte[]', 'uint8[10]', 'byte[20]']:
         g = genpy.generator.string_serializer_generator('foo', 'uint8[]', 'b_name', True)
         assert """length = len(b_name)
-# - if encoded as a list instead, serialize as bytes instead of string
-if type(b_name) in [list, tuple]:
-  buff.write(struct.Struct('<I%sB'%length).pack(length, *b_name))
-else:
-  buff.write(struct.Struct('<I%ss'%length).pack(length, b_name))""" == '\n'.join(g)
+# check for buffer protocol support
+try:
+  tmp = memoryview(b_name)
+  from genpy.generate_struct import memoryview_len
+  buff.write(struct.Struct('<I').pack(memoryview_len(tmp)))
+  buff.write(tmp)
+except TypeError:
+  # - if encoded as a list instead, serialize as bytes instead of string
+  if type(b_name) in [list, tuple]:
+    buff.write(struct.Struct('<I%sB'%length).pack(length, *b_name))
+  else:
+    buff.write(struct.Struct('<I%ss'%length).pack(length, b_name))""" == '\n'.join(g)
 
     # Test Deserializers
     val = """start = end