Merge pull request #41 from mhkline/expose-subprocess-kwargs

expose subprocess kwargs through parse_file call

appveyor.yml

@@ -1,4 +1,4 @@
-version: 0.11.1.{build}
+version: 0.12.0.{build}
 
 environment:
   PYTHONUNBUFFERED: 1

pycstruct/cparser.py

@@ -29,11 +29,15 @@
 
 
 def _run_castxml(
-    input_files, xml_filename, castxml_cmd="castxml", castxml_extra_args=None
+    input_files,
+    xml_filename,
+    castxml_cmd="castxml",
+    castxml_extra_args=None,
+    **subprocess_kwargs,
 ):
     """Run castcml as a 'shell command'"""
     if shutil.which(castxml_cmd) is None:
-        raise Exception(
+        raise RuntimeError(
             f'Executable "{castxml_cmd}" not found.\n'
             + "External software castxml is not installed.\n"
             + "You need to install it and put it in your PATH."
@@ -47,18 +51,23 @@ def _run_castxml(
     args.append("-o")
     args.append(xml_filename)
 
+    # to preserve behavior before subprocess_kwargs were allowed to be passed
+    # in, default to redirecting stderr to STDOUT.
+    if "stderr" not in subprocess_kwargs:
+        subprocess_kwargs["stderr"] = subprocess.STDOUT
+
     try:
-        output = subprocess.check_output(args, stderr=subprocess.STDOUT)
+        output = subprocess.check_output(args, **subprocess_kwargs)
     except subprocess.CalledProcessError as exception:
-        raise Exception(
+        raise RuntimeError(
             "Unable to run:\n"
             + f"{' '.join(args)}\n\n"
             + "Output:\n"
             + exception.output.decode()
         ) from exception
 
     if not os.path.isfile(xml_filename):
-        raise Exception(
+        raise RuntimeError(
             "castxml did not report any error but "
             + f"{xml_filename} was never produced.\n\n"
             + f"castxml output was:\n{output.decode()}"
@@ -305,13 +314,13 @@ def _get_attrib(self, elem, attrib, default):
     def _get_elem_with_id(self, typeid):
         elem = self.root.find(f"*[@id='{typeid}']")
         if elem is None:
-            raise Exception(f"No XML element with id attribute {typeid} identified")
+            raise RuntimeError(f"No XML element with id attribute {typeid} identified")
         return elem
 
     def _get_elem_with_attrib(self, tag, attrib, value):
         elem = self.root.find(f"{tag}[@{attrib}='{value}']")
         if elem is None:
-            raise Exception(
+            raise RuntimeError(
                 f"No {tag} XML element with {attrib} attribute {value} identified"
             )
         return elem
@@ -403,7 +412,7 @@ def _get_type(self, type_id, is_array=False):
             member_type["type_name"] = "enum"
             member_type["reference"] = elem.attrib["id"]
         else:
-            raise Exception(f"Member type {elem.tag} is not supported.")
+            raise RuntimeError(f"Member type {elem.tag} is not supported.")
 
         return member_type
 
@@ -468,7 +477,7 @@ def _to_instance(self, name):
                 if "reference" in member:
                     other_instance = self._to_instance(member["reference"])
                     if other_instance is None:
-                        raise Exception(
+                        raise RuntimeError(
                             f"Member {member['name']} is of type {member['type']} "
                             f"{member['reference']} that is not supported"
                         )
@@ -482,7 +491,6 @@ def _to_instance(self, name):
                         same_level=same_level,
                     )
                 else:
-
                     instance.add(member["type"], member["name"], shape=shape)
 
         # Enum
@@ -531,6 +539,7 @@ def parse_file(
     castxml_extra_args=None,
     cache_path="",
     use_cached=False,
+    **subprocess_kwargs,
 ):
     """Parse one or more C source files (C or C++) and generate pycstruct
     instances as a result.
@@ -576,6 +585,11 @@ def parse_file(
                        castxml (since it could be time consuming).
                        Default is False.
     :type use_cached: boolean, optional
+    :param subprocess_kwargs: keyword arguments that will be passed down to the
+                              `subprocess.check_outputs()` call used to run
+                              castxml. By default, stderr will be redirected to
+                              stdout. To get the subprocess default behavior,
+                              pass `stderr=None`.
     :return: A dictionary keyed on names of the structs, unions
              etc. The values are the actual pycstruct instances.
     :rtype: dict
@@ -596,7 +610,9 @@ def parse_file(
 
     # Generate XML
     if not use_cached or not os.path.isfile(xml_path):
-        _run_castxml(input_files, xml_path, castxml_cmd, castxml_extra_args)
+        _run_castxml(
+            input_files, xml_path, castxml_cmd, castxml_extra_args, **subprocess_kwargs
+        )
 
     # Parse XML
     castxml_parser = _CastXmlParser(xml_path)

pycstruct/instance.py

@@ -40,7 +40,7 @@ class Instance:
 
     def __init__(self, datatype, buffer=None, buffer_offset=0):
         if not isinstance(datatype, (pycstruct.StructDef, pycstruct.BitfieldDef)):
-            raise Exception("top_class needs to be of type StructDef or BitfieldDef")
+            raise RuntimeError("top_class needs to be of type StructDef or BitfieldDef")
 
         if buffer is None:
             buffer = bytearray(datatype.size())

pycstruct/pycstruct.py

@@ -138,7 +138,7 @@ def __getitem__(self, length):
 
     def dtype(self):
         """Returns the numpy dtype of this definition"""
-        raise Exception(f"dtype not implemented for {type(self)}")
+        raise RuntimeError(f"dtype not implemented for {type(self)}")
 
 
 ###############################################################################
@@ -217,11 +217,11 @@ def serialize(self, data, buffer=None, offset=0):
             assert len(buffer) >= offset + self.size(), "Specified buffer too small"
 
         if not isinstance(data, str):
-            raise Exception(f"Not a valid string: {data}")
+            raise RuntimeError(f"Not a valid string: {data}")
 
         utf8_bytes = data.encode("utf-8")
         if len(utf8_bytes) > self.length:
-            raise Exception(
+            raise RuntimeError(
                 f"String overflow. Produced size {len(utf8_bytes)} but max is {self.length}"
             )
 
@@ -268,9 +268,9 @@ def __init__(self, element_type, length):
     def serialize(self, data, buffer=None, offset=0):
         """Serialize a python type into a binary type following this array type"""
         if not isinstance(data, collections.abc.Iterable):
-            raise Exception("Data shall be a list")
+            raise RuntimeError("Data shall be a list")
         if len(data) > self.length:
-            raise Exception(f"List is larger than {self.length}")
+            raise RuntimeError(f"List is larger than {self.length}")
 
         if buffer is None:
             assert offset == 0, "When buffer is None, offset have to be unset"
@@ -400,7 +400,7 @@ class StructDef(_BaseDef):
     def __init__(self, default_byteorder="native", alignment=1, union=False):
         """Constructor method"""
         if default_byteorder not in _BYTEORDER:
-            raise Exception(f"Invalid byteorder: {default_byteorder}.")
+            raise RuntimeError(f"Invalid byteorder: {default_byteorder}.")
         self.__default_byteorder = default_byteorder
         self.__alignment = alignment
         self.__union = union
@@ -529,15 +529,17 @@ def add(self, datatype, name, length=1, byteorder="", same_level=False, shape=No
         # Sanity checks
         shape = self._normalize_shape(length, shape)
         if name in self.__fields:
-            raise Exception(f"Field name already exist: {name}.")
+            raise RuntimeError(f"Field name already exist: {name}.")
         if byteorder == "":
             byteorder = self.__default_byteorder
         elif byteorder not in _BYTEORDER:
-            raise Exception(f"Invalid byteorder: {byteorder}.")
+            raise RuntimeError(f"Invalid byteorder: {byteorder}.")
         if same_level and len(shape) != 0:
-            raise Exception("same_level not allowed in combination with arrays")
+            raise RuntimeError("same_level not allowed in combination with arrays")
         if same_level and not isinstance(datatype, BitfieldDef):
-            raise Exception("same_level only allowed in combination with BitfieldDef")
+            raise RuntimeError(
+                "same_level only allowed in combination with BitfieldDef"
+            )
 
         # Invalidate the dtype cache
         self.__dtype = None
@@ -552,7 +554,7 @@ def add(self, datatype, name, length=1, byteorder="", same_level=False, shape=No
         elif datatype in _TYPE:
             datatype = BasicTypeDef(datatype, byteorder)
         elif not isinstance(datatype, _BaseDef):
-            raise Exception(f"Invalid datatype: {datatype}.")
+            raise RuntimeError(f"Invalid datatype: {datatype}.")
 
         if len(shape) > 0:
             for dim in reversed(shape):
@@ -641,7 +643,7 @@ def deserialize(self, buffer, offset=0):
         result = {}
 
         if len(buffer) < self.size() + offset:
-            raise Exception(
+            raise RuntimeError(
                 f"Invalid buffer size: {len(buffer)}. Expected: {self.size()}"
             )
 
@@ -682,7 +684,7 @@ def _deserialize_element(self, name, buffer, buffer_offset=0):
         try:
             value = datatype.deserialize(buffer, buffer_offset + offset)
         except Exception as exception:
-            raise Exception(
+            raise RuntimeError(
                 f"Unable to deserialize {datatype._type_name()} {name}. "
                 f"Reason:\n{exception.args[0]}"
             ) from exception
@@ -747,7 +749,7 @@ def _serialize_element(self, name, value, buffer, buffer_offset=0):
         try:
             datatype.serialize(value, buffer, next_offset)
         except Exception as exception:
-            raise Exception(
+            raise RuntimeError(
                 f"Unable to serialize {datatype._type_name()} {name}. Reason:\n{exception.args[0]}"
             ) from exception
 
@@ -816,7 +818,6 @@ def _type_name(self):
         return "struct"
 
     def remove_from(self, name):
-
         """Remove all elements from a specific element
 
         This function is useful to create a sub-set of a struct.
@@ -827,7 +828,6 @@ def remove_from(self, name):
         self._remove_from_or_to(name, to_criteria=False)
 
     def remove_to(self, name):
-
         """Remove all elements from beginning to a specific element
 
         This function is useful to create a sub-set of a struct.
@@ -839,7 +839,7 @@ def remove_to(self, name):
 
     def _remove_from_or_to(self, name, to_criteria=True):
         if name not in self.__fields:
-            raise Exception(f"Element {name} does not exist")
+            raise RuntimeError(f"Element {name} does not exist")
 
         # Invalidate the dtype cache
         self.__dtype = None
@@ -897,7 +897,7 @@ def _element_offset(self, name):
         """
         if name in self.__fields:
             return self.__fields[name]["offset"]
-        raise Exception(f"Invalid element {name}")
+        raise RuntimeError(f"Invalid element {name}")
 
     def get_field_type(self, name):
         """Returns the type of a field of this struct.
@@ -977,7 +977,7 @@ class BitfieldDef(_BaseDef):
 
     def __init__(self, byteorder="native", size=-1):
         if byteorder not in _BYTEORDER:
-            raise Exception(f"Invalid byteorder: {byteorder}.")
+            raise RuntimeError(f"Invalid byteorder: {byteorder}.")
         if byteorder == "native":
             byteorder = sys.byteorder
         self.__byteorder = byteorder
@@ -999,13 +999,13 @@ def add(self, name, nbr_of_bits=1, signed=False):
         :type signed: bool, optional"""
         # Check for same bitfield name
         if name in self.__fields:
-            raise Exception(f"Field with name {name} already exists.")
+            raise RuntimeError(f"Field with name {name} already exists.")
 
         # Check that new size is not too large
         assigned_bits = self.assigned_bits()
         total_nbr_of_bits = assigned_bits + nbr_of_bits
         if total_nbr_of_bits > self._max_bits():
-            raise Exception(
+            raise RuntimeError(
                 f"Maximum number of bits ({self._max_bits()}) exceeded: {total_nbr_of_bits}."
             )
 
@@ -1027,7 +1027,7 @@ def deserialize(self, buffer, offset=0):
         """
         result = {}
         if len(buffer) < self.size() + offset:
-            raise Exception(
+            raise RuntimeError(
                 f"Invalid buffer size: {len(buffer)}. Expected at least: {self.size()}"
             )
 
@@ -1198,12 +1198,12 @@ def _set_subvalue(self, value, subvalue, nbr_of_bits, start_bit, signed):
             signed_str = "Signed"
 
         if subvalue > max_value:
-            raise Exception(
+            raise RuntimeError(
                 f"{signed_str} value {subvalue} is too large to fit in "
                 f"{nbr_of_bits} bits. Max value is {max_value}."
             )
         if subvalue < min_value:
-            raise Exception(
+            raise RuntimeError(
                 f"{signed_str} value {subvalue} is too small to fit in "
                 f"{nbr_of_bits} bits. Min value is {min_value}."
             )
@@ -1283,7 +1283,7 @@ class EnumDef(_BaseDef):
 
     def __init__(self, byteorder="native", size=-1, signed=False):
         if byteorder not in _BYTEORDER:
-            raise Exception(f"Invalid byteorder: {byteorder}.")
+            raise RuntimeError(f"Invalid byteorder: {byteorder}.")
         if byteorder == "native":
             byteorder = sys.byteorder
         self.__byteorder = byteorder
@@ -1306,7 +1306,7 @@ def add(self, name, value=None):
         # pylint: disable=bare-except
         # Check for same bitfield name
         if name in self.__constants:
-            raise Exception(f"Constant with name {name} already exists.")
+            raise RuntimeError(f"Constant with name {name} already exists.")
 
         # Automatically assigned to next available value
         index = 0
@@ -1319,13 +1319,13 @@ def add(self, name, value=None):
 
         # Secure that no negative number are added to signed enum
         if not self.__signed and value < 0:
-            raise Exception(
+            raise RuntimeError(
                 f"Negative value, {value}, not supported in unsigned enums."
             )
 
         # Check that new size is not too large
         if self._bit_length(value) > self._max_bits():
-            raise Exception(
+            raise RuntimeError(
                 f"Maximum number of bits ({self._max_bits()}) exceeded: {self._bit_length(value)}."
             )
 
@@ -1347,7 +1347,7 @@ def deserialize(self, buffer, offset=0):
         """
         # pylint: disable=bare-except
         if len(buffer) < self.size() + offset:
-            raise Exception(
+            raise RuntimeError(
                 f"Invalid buffer size: {len(buffer)}. Expected: {self.size()}"
             )
 
@@ -1423,7 +1423,7 @@ def get_name(self, value):
         for constant, item_value in self.__constants.items():
             if value == item_value:
                 return constant
-        raise Exception(f"Value {value} is not a valid value for this enum.")
+        raise RuntimeError(f"Value {value} is not a valid value for this enum.")
 
     def get_value(self, name):
         """Get the value representation of the name
@@ -1432,7 +1432,7 @@ def get_value(self, name):
         :rtype: int
         """
         if name not in self.__constants:
-            raise Exception(f"{name} is not a valid name in this enum.")
+            raise RuntimeError(f"{name} is not a valid name in this enum.")
         return self.__constants[name]
 
     def _type_name(self):

setup.py

@@ -8,7 +8,7 @@
 
 setup(
     name="pycstruct",
-    version="0.11.1",
+    version="0.12.0",
     description="Binary data handling in Python using dictionaries",
     long_description=long_description,
     long_description_content_type="text/markdown",

tests/test_instance.py

@@ -4,7 +4,6 @@
 
 class TestInstance(unittest.TestCase):
     def test_instance(self):
-
         # First define a complex definition structure
         car_type = pycstruct.EnumDef(size=4)
         car_type.add("Sedan", 0)