diff --git a/.github/actions/spelling/expect.txt b/.github/actions/spelling/expect.txt index 91944cb4..e3449fd0 100644 --- a/.github/actions/spelling/expect.txt +++ b/.github/actions/spelling/expect.txt @@ -12,6 +12,7 @@ autoapi autocoded autocode Autocoders +autocoders autocoding autodoc autoescape @@ -118,6 +119,7 @@ isdir isfile isinstance isnumeric +issubclass iterdir itertools itle @@ -136,6 +138,7 @@ lestarch LGTM lgtm Linux +lld locs lstrip lxml @@ -200,6 +203,7 @@ SCLK scm sdd Serializables +serializables setuptools shutil someotherpath diff --git a/src/fprime/common/models/serialize/array_type.py b/src/fprime/common/models/serialize/array_type.py index 19f98a21..901c41ef 100644 --- a/src/fprime/common/models/serialize/array_type.py +++ b/src/fprime/common/models/serialize/array_type.py @@ -3,9 +3,7 @@ Created on May 29, 2020 @author: jishii """ -import copy - -from .type_base import ValueType +from .type_base import DictionaryType from .type_exceptions import ( ArrayLengthException, NotInitializedException, @@ -16,39 +14,38 @@ from fprime.util.string_util import format_string_template -class ArrayType(ValueType): +class ArrayType(DictionaryType): """Generic fixed-size array type representation. Represents a custom named type of a fixed number of like members, each of which are other types in the system. """ - def __init__(self, typename, config_info, val=None): - """Constructs a new array type. + @classmethod + def construct_type(cls, name, member_type, length, format): + """Constructs a sub-array type + + Constructs a new sub-type of array to represent an array of the given name, member type, length, and format + string. Args: - typename: name of this array type - config_info: (type, size, format string) information for array - val: (optional) list of values to assign to array + name: name of the array subtype + member_type: type of the members of the array subtype + length: length of the array subtype + format: format string for members of the array subtype """ - super().__init__() - if not isinstance(typename, str): - raise TypeMismatchException(str, type(typename)) - self.__val = None - self.__typename = typename - self.__arr_type, self.__arr_size, self.__arr_format = config_info - # Set value only if it is a valid, non-empty list - if not val: - return - self.val = val - - def validate(self, val): + return DictionaryType.construct_type( + cls, name, MEMBER_TYPE=member_type, LENGTH=length, FORMAT=format + ) + + @classmethod + def validate(cls, val): """Validates the values of the array""" - size = self.__arr_size - if len(val) != size: - raise ArrayLengthException(self.__arr_type, size, len(val)) - for i in range(self.__arr_size): - if not isinstance(val[i], type(self.__arr_type)): - raise TypeMismatchException(type(self.__arr_type), type(val[i])) + if not isinstance(val, (tuple, list)): + raise TypeMismatchException(list, type(val)) + elif len(val) != cls.LENGTH: + raise ArrayLengthException(cls.MEMBER_TYPE, cls.LENGTH, len(val)) + for i in range(cls.LENGTH): + cls.MEMBER_TYPE.validate(val[i]) @property def val(self) -> list: @@ -58,7 +55,9 @@ def val(self) -> list: :return dictionary of member names to python values of member keys """ - return [item.val for item in self.__val] + if self._val is None: + return None + return [item.val for item in self._val] @property def formatted_val(self) -> list: @@ -69,11 +68,11 @@ def formatted_val(self) -> list: :return a formatted array """ result = [] - for item in self.__val: + for item in self._val: if isinstance(item, (serializable_type.SerializableType, ArrayType)): result.append(item.formatted_val) else: - result.append(format_string_template(self.__arr_format, item.val)) + result.append(format_string_template(self.FORMAT, item.val)) return result @val.setter @@ -85,25 +84,22 @@ def val(self, val: list): :param val: dictionary containing python types to key names. This """ - items = [] - for item in val: - cloned = copy.deepcopy(self.arr_type) - cloned.val = item - items.append(cloned) - self.__val = items + self.validate(val) + items = [self.MEMBER_TYPE(item) for item in val] + self._val = items def to_jsonable(self): """ JSONable type """ members = { - "name": self.__typename, - "type": self.__typename, - "size": self.__arr_size, - "format": self.__arr_format, + "name": self.__class__.__name__, + "type": self.__class__.__name__, + "size": self.LENGTH, + "format": self.FORMAT, "values": None - if self.__val is None - else [member.to_jsonable() for member in self.__val], + if self._val is None + else [member.to_jsonable() for member in self._val], } return members @@ -111,32 +107,18 @@ def serialize(self): """Serialize the array by serializing the elements one by one""" if self.val is None: raise NotInitializedException(type(self)) - return b"".join([item.serialize() for item in self.val]) + return b"".join([item.serialize() for item in self._val]) def deserialize(self, data, offset): """Deserialize the members of the array""" values = [] - for i in range(self.__arr_size): - item = copy.deepcopy(self.arr_type) - item.deserialize(data, offset + i * item.getSize()) - values.append(item.val) - self.val = values - - @property - def arr_type(self): - """Property representing the size of the array""" - return self.__arr_type - - @property - def arr_size(self): - """Property representing the number of elements of the array""" - return self.__arr_size - - @property - def arr_format(self): - """Property representing the format string of an item in the array""" - return self.__arr_format + for i in range(self.LENGTH): + item = self.MEMBER_TYPE() + item.deserialize(data, offset) + offset += item.getSize() + values.append(item) + self._val = values def getSize(self): """Return the size of the array""" - return self.arr_type.getSize() * self.arr_size + return sum([item.getSize() for item in self._val]) diff --git a/src/fprime/common/models/serialize/bool_type.py b/src/fprime/common/models/serialize/bool_type.py index 10ee8b21..c9db5954 100644 --- a/src/fprime/common/models/serialize/bool_type.py +++ b/src/fprime/common/models/serialize/bool_type.py @@ -22,16 +22,17 @@ class BoolType(ValueType): TRUE = 0xFF FALSE = 0x00 - def validate(self, val): + @classmethod + def validate(cls, val): """Validate the given class""" if not isinstance(val, bool): raise TypeMismatchException(bool, type(val)) def serialize(self): """Serialize a boolean value""" - if self.val is None: + if self._val is None: raise NotInitializedException(type(self)) - return struct.pack("B", 0xFF if self.val else 0x00) + return struct.pack("B", self.TRUE if self._val else self.FALSE) def deserialize(self, data, offset): """Deserialize boolean value""" @@ -39,7 +40,7 @@ def deserialize(self, data, offset): int_val = struct.unpack_from("B", data, offset)[0] if int_val not in [self.TRUE, self.FALSE]: raise TypeRangeException(int_val) - self.val = int_val == self.TRUE + self._val = int_val == self.TRUE except struct.error: raise DeserializeException("Not enough bytes to deserialize bool.") diff --git a/src/fprime/common/models/serialize/enum_type.py b/src/fprime/common/models/serialize/enum_type.py index ff73ade4..5360e95a 100644 --- a/src/fprime/common/models/serialize/enum_type.py +++ b/src/fprime/common/models/serialize/enum_type.py @@ -4,7 +4,7 @@ """ import struct -from .type_base import ValueType +from .type_base import DictionaryType from .type_exceptions import ( DeserializeException, EnumMismatchException, @@ -14,7 +14,7 @@ ) -class EnumType(ValueType): +class EnumType(DictionaryType): """ Representation of the ENUM type. @@ -23,51 +23,39 @@ class EnumType(ValueType): containing code based on C enum rules """ - def __init__(self, typename="", enum_dict=None, val=None): - """ - Constructor + @classmethod + def construct_type(cls, name, enum_dict): + """Construct the custom enum type - :param typename: name of the enumeration type - :param enum_dict: dictionary of value to integer representation - :param val: value of the enumeration - """ - super().__init__() - if not isinstance(typename, str): - raise TypeMismatchException(str, type(val)) - self.__typename = typename - # Setup the enum dictionary - if enum_dict is None: - enum_dict = {"UNDEFINED": 0} - # Check if the enum dict is an instance of dictionary - self.__enum_dict = enum_dict - # Set val to undefined if not set - if val is None: - val = "UNDEFINED" - self.val = val + Constructs the custom enumeration type, with the supplied enumeration dictionary. - def validate(self, val): - """Validate the value passed into the enumeration""" - if not isinstance(self.enum_dict(), dict): - raise TypeMismatchException(dict, type(self.enum_dict())) - for member in self.keys(): + Args: + name: name of the enumeration type + enum_dict: enumeration: value dictionary defining the enumeration + """ + if not isinstance(enum_dict, dict): + raise TypeMismatchException(dict, type(enum_dict)) + for member in enum_dict.keys(): if not isinstance(member, str): raise TypeMismatchException(str, type(member)) - elif not isinstance(self.enum_dict()[member], int): - raise TypeMismatchException(int, self.enum_dict()[member]) - if val != "UNDEFINED" and val not in self.keys(): - raise EnumMismatchException(self.__typename, val) + elif not isinstance(enum_dict[member], int): + raise TypeMismatchException(int, enum_dict[member]) + return DictionaryType.construct_type(cls, name, ENUM_DICT=enum_dict) - def keys(self): + @classmethod + def validate(cls, val): + """Validate the value passed into the enumeration""" + if not isinstance(val, str): + raise TypeMismatchException(str, type(val)) + if val not in cls.keys(): + raise EnumMismatchException(cls.__class__.__name__, val) + + @classmethod + def keys(cls): """ Return all the enum key values. """ - return list(self.enum_dict().keys()) - - def typename(self): - return self.__typename - - def enum_dict(self): - return self.__enum_dict + return list(cls.ENUM_DICT.keys()) def serialize(self): """ @@ -75,9 +63,11 @@ def serialize(self): """ # for enums, take the string value and convert it to # the numeric equivalent - if self.val is None: + if self._val is None or ( + self._val == "UNDEFINED" and "UNDEFINED" not in self.ENUM_DICT + ): raise NotInitializedException(type(self)) - return struct.pack(">i", self.enum_dict()[self.val]) + return struct.pack(">i", self.ENUM_DICT[self._val]) def deserialize(self, data, offset): """ @@ -85,13 +75,13 @@ def deserialize(self, data, offset): """ try: int_val = struct.unpack_from(">i", data, offset)[0] - except: + except struct.error: raise DeserializeException( f"Could not deserialize enum value. Needed: {self.getSize()} bytes Found: {len(data[offset:])}" ) - for key, val in self.enum_dict().items(): + for key, val in self.ENUM_DICT.items(): if int_val == val: - self.val = key + self._val = key break # Value not found, invalid enumeration value else: diff --git a/src/fprime/common/models/serialize/numerical_types.py b/src/fprime/common/models/serialize/numerical_types.py index 48c3009b..1f12a031 100644 --- a/src/fprime/common/models/serialize/numerical_types.py +++ b/src/fprime/common/models/serialize/numerical_types.py @@ -7,7 +7,6 @@ @author mstarch """ import abc -import re import struct from .type_base import ValueType @@ -18,25 +17,19 @@ TypeRangeException, ) -BITS_RE = re.compile(r"[IUF](\d\d?)") - class NumericalType(ValueType, abc.ABC): """Numerical types that can be serialized using struct and are of some power of 2 byte width""" @classmethod + @abc.abstractmethod def get_bits(cls): """Gets the integer bits of a given type""" - match = BITS_RE.match(cls.__name__) - assert ( - match - ), f"Type {cls} does not follow format I#Type U#Type nor F#Type required of numerical types" - return int(match.group(1)) @classmethod def getSize(cls): """Gets the size of the integer based on the size specified in the class name""" - return int(cls.get_bits() / 8) + return int(cls.get_bits() >> 3) # Divide by 8 quickly @staticmethod @abc.abstractmethod @@ -46,14 +39,14 @@ def get_serialize_format(): def serialize(self): """Serializes this type using struct and the val property""" - if self.val is None: + if self._val is None: raise NotInitializedException(type(self)) - return struct.pack(self.get_serialize_format(), self.val) + return struct.pack(self.get_serialize_format(), self._val) def deserialize(self, data, offset): """Serializes this type using struct and the val property""" try: - self.val = struct.unpack_from(self.get_serialize_format(), data, offset)[0] + self._val = struct.unpack_from(self.get_serialize_format(), data, offset)[0] except struct.error as err: raise DeserializeException(str(err)) @@ -61,23 +54,26 @@ def deserialize(self, data, offset): class IntegerType(NumericalType, abc.ABC): """Base class that represents all integer common functions""" - def validate(self, val): + @classmethod + @abc.abstractmethod + def range(cls): + """Gets signed/unsigned of this type""" + + @classmethod + def validate(cls, val): """Validates the given integer.""" if not isinstance(val, int): raise TypeMismatchException(int, type(val)) - max_val = 1 << ( - self.get_bits() - (1 if self.__class__.__name__.startswith("I") else 0) - ) - min_val = -max_val if self.__class__.__name__.startswith("I") else 0 - # Compare to min and max - if val < min_val or val >= max_val: + min_val, max_val = cls.range() + if val < min_val or val > max_val: raise TypeRangeException(val) class FloatType(NumericalType, abc.ABC): """Base class that represents all float common functions""" - def validate(self, val): + @classmethod + def validate(cls, val): """Validates the given integer.""" if not isinstance(val, (float, int)): raise TypeMismatchException(float, type(val)) @@ -86,6 +82,16 @@ def validate(self, val): class I8Type(IntegerType): """Single byte integer type. Represents C chars""" + @classmethod + def range(cls): + """Gets signed/unsigned of this type""" + return (-128, 127) + + @classmethod + def get_bits(cls): + """Get the bit count of this type""" + return 8 + @staticmethod def get_serialize_format(): """Allows serialization using struct""" @@ -95,6 +101,16 @@ def get_serialize_format(): class I16Type(IntegerType): """Double byte integer type. Represents C shorts""" + @classmethod + def range(cls): + """Gets signed/unsigned of this type""" + return (-32768, 32767) + + @classmethod + def get_bits(cls): + """Get the bit count of this type""" + return 16 + @staticmethod def get_serialize_format(): """Allows serialization using struct""" @@ -104,6 +120,16 @@ def get_serialize_format(): class I32Type(IntegerType): """Four byte integer type. Represents C int32_t,""" + @classmethod + def range(cls): + """Gets signed/unsigned of this type""" + return (-2147483648, 2147483647) + + @classmethod + def get_bits(cls): + """Get the bit count of this type""" + return 32 + @staticmethod def get_serialize_format(): """Allows serialization using struct""" @@ -113,6 +139,16 @@ def get_serialize_format(): class I64Type(IntegerType): """Eight byte integer type. Represents C int64_t,""" + @classmethod + def range(cls): + """Gets signed/unsigned of this type""" + return (-9223372036854775808, 9223372036854775807) + + @classmethod + def get_bits(cls): + """Get the bit count of this type""" + return 64 + @staticmethod def get_serialize_format(): """Allows serialization using struct""" @@ -122,6 +158,16 @@ def get_serialize_format(): class U8Type(IntegerType): """Single byte integer type. Represents C chars""" + @classmethod + def range(cls): + """Gets signed/unsigned of this type""" + return (0, 0xFF) + + @classmethod + def get_bits(cls): + """Get the bit count of this type""" + return 8 + @staticmethod def get_serialize_format(): """Allows serialization using struct""" @@ -131,6 +177,16 @@ def get_serialize_format(): class U16Type(IntegerType): """Double byte integer type. Represents C shorts""" + @classmethod + def range(cls): + """Gets signed/unsigned of this type""" + return (0, 0xFFFF) + + @classmethod + def get_bits(cls): + """Get the bit count of this type""" + return 16 + @staticmethod def get_serialize_format(): """Allows serialization using struct""" @@ -140,6 +196,16 @@ def get_serialize_format(): class U32Type(IntegerType): """Four byte integer type. Represents C unt32_t,""" + @classmethod + def range(cls): + """Gets signed/unsigned of this type""" + return (0, 0xFFFFFFFF) + + @classmethod + def get_bits(cls): + """Get the bit count of this type""" + return 32 + @staticmethod def get_serialize_format(): """Allows serialization using struct""" @@ -149,6 +215,16 @@ def get_serialize_format(): class U64Type(IntegerType): """Eight byte integer type. Represents C unt64_t,""" + @classmethod + def range(cls): + """Gets signed/unsigned of this type""" + return (0, 0xFFFFFFFFFFFFFFFF) + + @classmethod + def get_bits(cls): + """Get the bit count of this type""" + return 64 + @staticmethod def get_serialize_format(): """Allows serialization using struct""" @@ -158,6 +234,11 @@ def get_serialize_format(): class F32Type(FloatType): """Eight byte integer type. Represents C unt64_t,""" + @classmethod + def get_bits(cls): + """Get the bit count of this type""" + return 32 + @staticmethod def get_serialize_format(): """Allows serialization using struct""" @@ -167,6 +248,11 @@ def get_serialize_format(): class F64Type(FloatType): """Eight byte integer type. Represents C unt64_t,""" + @classmethod + def get_bits(cls): + """Get the bit count of this type""" + return 64 + @staticmethod def get_serialize_format(): """Allows serialization using struct""" diff --git a/src/fprime/common/models/serialize/serializable_type.py b/src/fprime/common/models/serialize/serializable_type.py index 86146bae..39a0fa01 100644 --- a/src/fprime/common/models/serialize/serializable_type.py +++ b/src/fprime/common/models/serialize/serializable_type.py @@ -4,16 +4,19 @@ @author: tcanham """ -import copy - -from .type_base import BaseType, ValueType -from .type_exceptions import NotInitializedException, TypeMismatchException +from .type_base import BaseType, DictionaryType +from .type_exceptions import ( + IncorrectMembersException, + MissingMemberException, + NotInitializedException, + TypeMismatchException, +) from . import array_type from fprime.util.string_util import format_string_template -class SerializableType(ValueType): +class SerializableType(DictionaryType): """ Representation of the Serializable type (comparable to the ANY type) @@ -27,83 +30,49 @@ class SerializableType(ValueType): The member descriptions can be None """ - def __init__(self, typename, mem_list=None): - """ - Constructor - """ - super().__init__() - if not isinstance(typename, str): - raise TypeMismatchException(str, type(typename)) - self.__typename = typename - - new_mem_list = [] - # If the member list is defined, stamp in None for any missing descriptions - if mem_list: - new_mem_list = [ - entry if len(entry) == 4 else (entry[0], entry[1], entry[2], None) - for entry in mem_list - ] - # Set the member list then set the value - self.mem_list = new_mem_list + @classmethod + def construct_type(cls, name, member_list): + """Construct a new serializable sub-type - def validate(self, val=None): - """Validate this object including member list and values""" - # Blank member list does not validate - if not self.mem_list: - return - elif not isinstance(self.mem_list, list): - raise TypeMismatchException(list, self.mem_list) - for member_name, member_val, format_string, description in self.mem_list: + Constructs a new serializable subtype from the supplied member list and name. Member list may optionally exclude + description keys, which will be filled with None. + + Args: + name: name of the new sub-type + member_list: list of member definitions in form list of tuples (name, type, format string, description) + """ + # Check that we are dealing with a list + if not isinstance(member_list, list): + raise TypeMismatchException(list, type(member_list)) + member_list = [list(item) + ([None] * (4 - len(item))) for item in member_list] + # Check the validity of the member list + for member_name, member_type, format_string, description in member_list: # Check each of these members for correct types if not isinstance(member_name, str): raise TypeMismatchException(str, type(member_name)) - elif not isinstance(member_val, BaseType): - raise TypeMismatchException(BaseType, type(member_val)) - elif not isinstance(format_string, str): + elif not issubclass(member_type, BaseType): + raise TypeMismatchException(BaseType, member_type) + elif format_string is not None and not isinstance(format_string, str): raise TypeMismatchException(str, type(format_string)) elif description is not None and not isinstance(description, str): raise TypeMismatchException(str, type(description)) - # When a value is set and is not empty we need to set the member properties - if not val: - return - # If a value is supplied then check each value against the member list - for val_member, list_entry in zip(val, self.mem_list): - _, member_list_val, _, _ = list_entry - member_list_val.validate( - val_member - ) # Insure that the the val_member is consistent with the existing member + return DictionaryType.construct_type(cls, name, MEMBER_LIST=member_list) - @property - def mem_list(self): - """Gets the member list""" - return self.__mem_list - - @mem_list.setter - def mem_list(self, mem_list): - """ - Sets the member list and validates against the current value. - """ - self.__mem_list = mem_list - if mem_list is None: - self.validate(self.mem_list) - - def serialize(self): - """Serializes the members of the serializable""" - if self.mem_list is None: - raise NotInitializedException(type(self)) - return b"".join( - [member_val.serialize() for _, member_val, _, _ in self.mem_list] - ) - - def deserialize(self, data, offset): - """Deserialize the values of each of the members""" - new_member_list = [] - for entry1, member_val, entry3, entry4 in self.mem_list: - cloned = copy.copy(member_val) - cloned.deserialize(data, offset) - new_member_list.append((entry1, cloned, entry3, entry4)) - offset += member_val.getSize() - self.mem_list = new_member_list + @classmethod + def validate(cls, val): + """Validate this object including member list and values""" + # Ensure that the supplied value is a dictionary + if not isinstance(val, dict): + raise TypeMismatchException(dict, type(val)) + elif len(val) != len(cls.MEMBER_LIST): + raise IncorrectMembersException([name for name, _, _, _ in cls.MEMBER_LIST]) + # Now validate each field as defined via the value + for member_name, member_type, _, _ in cls.MEMBER_LIST: + try: + member_val = val[member_name] + except KeyError: + raise MissingMemberException(member_name) + member_type.validate(member_val) @property def val(self) -> dict: @@ -113,9 +82,26 @@ def val(self) -> dict: :return dictionary of member names to python values of member keys """ + if self._val is None: + return None return { - member_name: member_val.val - for member_name, member_val, _, _ in self.mem_list + member_name: self._val.get(member_name).val + for member_name, _, _, _ in self.MEMBER_LIST + } + + @val.setter + def val(self, val: dict): + """ + The .val property typically returns the python-native type. This the python native type closes to a serializable + without generating full classes would be a dictionary (anonymous object). This takes such an object and sets the + member val list from it. + + :param val: dictionary containing python types to key names. This + """ + self.validate(val) + self._val = { + member_name: member_type(val.get(member_name)) + for member_name, member_type, _, _, in self.MEMBER_LIST } @property @@ -126,38 +112,53 @@ def formatted_val(self) -> dict: Note 2: If a member is an array will call array formatted_val :return a formatted dict """ - return { - member_name: member_val.formatted_val - if isinstance(member_val, (array_type.ArrayType, SerializableType)) - else format_string_template(member_format, member_val.val) - for member_name, member_val, member_format, _ in self.mem_list - } + result = dict() + for member_name, _, member_format, _ in self.MEMBER_LIST: + value_object = self._val[member_name] + if isinstance(value_object, (array_type.ArrayType, SerializableType)): + result[member_name] = value_object.formatted_val + else: + result[member_name] = format_string_template( + member_format, value_object.val + ) + return result - @val.setter - def val(self, val: dict): - """ - The .val property typically returns the python-native type. This the python native type closes to a serializable - without generating full classes would be a dictionary (anonymous object). This takes such an object and sets the - member val list from it. + def serialize(self): + """Serializes the members of the serializable""" + if self._val is None: + raise NotInitializedException(type(self)) + return b"".join( + [ + self._val.get(member_name).serialize() + for member_name, _, _, _ in self.MEMBER_LIST + ] + ) - :param val: dictionary containing python types to key names. This - """ - values_list = [val[name] for name, _, _, _ in self.mem_list] - # Member list is the explicit store for storing these values - for val_member, list_entry in zip(values_list, self.mem_list): - _, member_list_val, _, _ = list_entry - member_list_val.val = val_member + def deserialize(self, data, offset): + """Deserialize the values of each of the members""" + new_value = {} + for member_name, member_type, _, _ in self.MEMBER_LIST: + new_member = member_type() + new_member.deserialize(data, offset) + new_value[member_name] = new_member + offset += new_member.getSize() + self._val = new_value def getSize(self): """The size of a struct is the size of all the members""" - return sum(mem_type.getSize() for _, mem_type, _, _ in self.mem_list) + return sum(self._val.get(name).getSize() for name, _, _, _ in self.MEMBER_LIST) def to_jsonable(self): """ JSONable type for a serializable """ members = {} - for member_name, member_value, member_format, member_desc in self.mem_list: + for member_name, member_value, member_format, member_desc in self.MEMBER_LIST: + value = ( + {"value": None} + if self._val is None + else self._val.get(member_name).to_jsonable() + ) members[member_name] = {"format": member_format, "description": member_desc} - members[member_name].update(member_value.to_jsonable()) + members[member_name].update(value) return members diff --git a/src/fprime/common/models/serialize/string_type.py b/src/fprime/common/models/serialize/string_type.py index 1bd48a0b..7e8d10ad 100644 --- a/src/fprime/common/models/serialize/string_type.py +++ b/src/fprime/common/models/serialize/string_type.py @@ -17,27 +17,30 @@ ) -class StringType(type_base.ValueType): +class StringType(type_base.DictionaryType): """ - String type representation for F prime. This is a value type that stores a half-word first for representing the - length of this given string. + String type representation for F prime. This is a type that stores a half-word first for representing the length of + this given string. Each sub string class defines the sub-type property max length that represents the maximum + length of any string value stored to it. + + All string types follow this implementation, but have some specific type-based properties: MAX_LENGTH. """ - def __init__(self, val=None, max_string_len=None): - """ - Constructor to build a string - @param val: value form which to create a string. Default: None. - @param max_string: maximum length of the string. Default: None, not used. - """ - self.__max_string_len = max_string_len - super().__init__(val) + @classmethod + def construct_type(cls, name, max_length=None): + """Constructs a new string type with given name and maximum length""" + temporary = type_base.DictionaryType.construct_type( + cls, name, MAX_LENGTH=max_length + ) + return temporary - def validate(self, val): + @classmethod + def validate(cls, val): """Validates that this is a string""" if not isinstance(val, str): raise TypeMismatchException(str, type(val)) - elif self.__max_string_len is not None and len(val) > self.__max_string_len: - raise StringSizeException(len(val), self.__max_string_len) + elif cls.MAX_LENGTH is not None and len(val) > cls.MAX_LENGTH: + raise StringSizeException(len(val), cls.MAX_LENGTH) def serialize(self): """ @@ -47,10 +50,8 @@ def serialize(self): if self.val is None: raise NotInitializedException(type(self)) # Check string size before serializing - elif ( - self.__max_string_len is not None and len(self.val) > self.__max_string_len - ): - raise StringSizeException(len(self.val), self.__max_string_len) + elif self.MAX_LENGTH is not None and len(self.val) > self.MAX_LENGTH: + raise StringSizeException(len(self.val), self.MAX_LENGTH) # Pack the string size first then return the encoded data buff = struct.pack(">H", len(self.val)) + self.val.encode(DATA_ENCODING) return buff @@ -67,8 +68,8 @@ def deserialize(self, data, offset): f"Not enough data to deserialize string data. Needed: {val_size} Left: {len(data[offset + 2 :])}" ) # Deal with a string that is larger than max string - elif self.__max_string_len is not None and val_size > self.__max_string_len: - raise StringSizeException(val_size, self.__max_string_len) + elif self.MAX_LENGTH is not None and val_size > self.MAX_LENGTH: + raise StringSizeException(val_size, self.MAX_LENGTH) self.val = data[offset + 2 : offset + 2 + val_size].decode(DATA_ENCODING) except struct.error: raise DeserializeException("Not enough bytes to deserialize string length.") diff --git a/src/fprime/common/models/serialize/type_base.py b/src/fprime/common/models/serialize/type_base.py index 3122fac2..8ad03f21 100644 --- a/src/fprime/common/models/serialize/type_base.py +++ b/src/fprime/common/models/serialize/type_base.py @@ -5,7 +5,6 @@ Replaced type base class with decorators """ import abc -import struct from .type_exceptions import AbstractMethodException @@ -56,52 +55,91 @@ class ValueType(BaseType): def __init__(self, val=None): """Defines the single value""" - self.__val = None + self._val = None # Run full setter if val is not None: self.val = val + @classmethod @abc.abstractmethod - def validate(self, val): + def validate(cls, val): """ Checks the val for validity with respect to the current type. This will raise TypeMismatchException when the - validation fails of the val's type fails. It will raise TypeRangeException when val is out of range. + validation fails of the val's type fails. It will raise TypeRangeException when val is out of range. Concrete + implementations will raise other exceptions based on type. For example, serializable types raise exceptions for + missing members. :param val: value to validate :raises TypeMismatchException: value has incorrect type, TypeRangeException: val is out of range """ + def __eq__(self, other): + """Check equality between types""" + if type(other) != type(self): + return False + return self._val == other._val + @property def val(self): """Getter for .val""" - return self.__val + return self._val @val.setter def val(self, val): """Setter for .val calls validate internally""" self.validate(val) - self.__val = val + self._val = val def to_jsonable(self): """ Converts this type to a JSON serializable object """ - return {"value": self.val, "type": str(self)} + return {"value": self.val, "type": str(self.__class__)} -# -# -def showBytes(byteBuffer): - """ - Routine to show bytes in buffer for testing. +class DictionaryType(ValueType, abc.ABC): + """Type whose specification is defined in the dictionary + + Certain types in fprime (strings, serializables, enums) are defined in the dictionary. Where all projects have + access to primitive types (U8, F32, etc) and the definitions of theses types is global, other types complete + specification comes from the dictionary itself. String set max-lengths per project, serializable fields are defined, + and enumeration values are enumerated. This class is designed to take base complex types (StringType, etc) and build + dynamic subclasses for the given dictionary defined type. """ - print("Byte buffer size: %d" % len(byteBuffer)) - for entry in range(0, len(byteBuffer)): - print( - "Byte %d: 0x%02X (%c)" - % ( - entry, - struct.unpack("B", bytes([byteBuffer[entry]]))[0], - struct.unpack("B", bytes([byteBuffer[entry]]))[0], - ) + + _CONSTRUCTS = {} + + @classmethod + def construct_type(cls, parent_class, name, **class_properties): + """Construct a new dictionary type + + Construct a new dynamic subtype of the given base type. This type will be named with the name parameter, define + the supplied class properties, and will be a subtype of the parent class. This function registers these new + types by name in the DictionaryType._CONSTRUCTS dictionary. When a type is defined a second or more times, the + originally constructed sub type is used and the newly supplied class properties are validated for equality + against the original definition. It is an error to define the type multiple times with inconsistent class + properties. + + Args: + cls: DictionaryType, used to access the class property _CONSTRUCTS + parent_class: class used as parent to the new sub type + name: name of the new sub type + **class_properties: properties to define on the subtype (e.g. max length for strings) + """ + assert ( + parent_class != DictionaryType + ), "Cannot build dictionary type from dictionary type directly" + construct, original_properties = cls._CONSTRUCTS.get( + name, (type(name, (parent_class,), class_properties), class_properties) ) + # Validate both new properties against original properties and against what is set on original class + assert ( + original_properties == class_properties + ), "Different class properties specified" + for attribute, value in class_properties.items(): + previous_value = getattr(construct, attribute, None) + assert ( + previous_value == value + ), f"Class {name} differs by attribute {attribute}. {previous_value} vs {value}" + cls._CONSTRUCTS[name] = (construct, class_properties) + return construct diff --git a/src/fprime/common/models/serialize/type_exceptions.py b/src/fprime/common/models/serialize/type_exceptions.py index 0b62482c..6afa3f15 100644 --- a/src/fprime/common/models/serialize/type_exceptions.py +++ b/src/fprime/common/models/serialize/type_exceptions.py @@ -65,6 +65,20 @@ def __init__(self, enum, bad_member): super().__init__(f"Invalid enum member {bad_member} set in {enum} enum!") +class MissingMemberException(TypeException): + """Member was not defined on type""" + + def __init__(self, field): + super().__init__(f"Value does not define required field: {field}") + + +class IncorrectMembersException(TypeException): + """Members incorrectly defined on type (too many, too few)""" + + def __init__(self, fields): + super().__init__(f"Value does not define required fields: {fields}") + + class DeserializeException(TypeException): """Exception during deserialization""" diff --git a/src/fprime/fbuild/gcovr.py b/src/fprime/fbuild/gcovr.py index 2e32df3d..18049bfd 100644 --- a/src/fprime/fbuild/gcovr.py +++ b/src/fprime/fbuild/gcovr.py @@ -7,7 +7,6 @@ from typing import Dict, List, Tuple, Union from .target import ExecutableAction, Target, TargetScope, CompositeTarget -from .types import MissingBuildCachePath TEMPORARY_DIRECTORY = "{{AUTOCODE}}" diff --git a/src/fprime/util/build_helper.py b/src/fprime/util/build_helper.py index 8ad3ff14..2d56ed49 100644 --- a/src/fprime/util/build_helper.py +++ b/src/fprime/util/build_helper.py @@ -183,8 +183,6 @@ def utility_entry(args): parsed, cmake_args, make_args, parser, runners = parse_args(args) try: - cwd = Path(parsed.path) - try: target = get_target(parsed) build_type = target.build_type diff --git a/test/fprime/common/models/serialize/test_types.py b/test/fprime/common/models/serialize/test_types.py index f34e850d..8c24ef53 100644 --- a/test/fprime/common/models/serialize/test_types.py +++ b/test/fprime/common/models/serialize/test_types.py @@ -27,12 +27,17 @@ from fprime.common.models.serialize.serializable_type import SerializableType from fprime.common.models.serialize.string_type import StringType from fprime.common.models.serialize.time_type import TimeBase, TimeType -from fprime.common.models.serialize.type_base import BaseType, ValueType +from fprime.common.models.serialize.type_base import BaseType, DictionaryType, ValueType from fprime.common.models.serialize.type_exceptions import ( AbstractMethodException, + ArrayLengthException, DeserializeException, + EnumMismatchException, + IncorrectMembersException, + MissingMemberException, NotInitializedException, + StringSizeException, TypeMismatchException, TypeRangeException, ) @@ -58,39 +63,50 @@ ] -def valid_values_test(type_input, valid_values, sizes, extras=None): +def valid_values_test(type_input, valid_values, sizes): """Tests to be run on all types""" if not isinstance(sizes, Iterable): sizes = [sizes] * len(valid_values) - # Should be able to instantiate a blank type, but not serialize it until a value has been supplied - if not extras: - extras = [[]] * len(valid_values) - instantiation = type_input(*extras[0]) + + instantiation = type_input() with pytest.raises(NotInitializedException): instantiation.serialize() + assert instantiation.val is None + # Setting to None is invalid + with pytest.raises(TypeMismatchException): + instantiation.val = None # Should be able to get a JSONable object that is dumpable to a JSON string jsonable = instantiation.to_jsonable() json.loads(json.dumps(jsonable)) # Run on valid values - for value, size, extra in zip(valid_values, sizes, extras): - instantiation = type_input(*extra, val=value) + for value, size in zip(valid_values, sizes): + instantiation = type_input(val=value) assert instantiation.val == value assert instantiation.getSize() == size # Check assignment by value - by_value = type_input(*extra) + by_value = type_input() by_value.val = value assert by_value.val == instantiation.val, "Assignment by value has failed" assert by_value.getSize() == size + # Check that the value returned by .val is also assignable to .val + by_value.val = by_value.val + # Check serialization and deserialization serialized = instantiation.serialize() for offset in [0, 10, 50]: - deserializer = type_input(*extra) + deserializer = type_input() deserializer.deserialize((b" " * offset) + serialized, offset) assert instantiation.val == deserializer.val, "Deserialization has failed" assert deserializer.getSize() == size + # Check another get/set pair and serialization of the post-deserialized object + deserializer.val = deserializer.val + new_serialized_bytes = deserializer.serialize() + assert ( + serialized == new_serialized_bytes + ), "Repeated serialization has failed" def invalid_values_test( @@ -215,85 +231,187 @@ def test_float_types_off_nominal(): ) -def test_enum_type(): +def test_enum_nominal(): """ Tests the EnumType serialization and deserialization """ members = {"MEMB1": 0, "MEMB2": 6, "MEMB3": 9} - val1 = EnumType("SomeEnum", members, "MEMB3") - buff = val1.serialize() - val2 = EnumType("SomeEnum", members) - val2.deserialize(buff, 0) - assert val1.val == val2.val + enum_class = EnumType.construct_type("SomeEnum", members) + valid = ["MEMB1", "MEMB2", "MEMB3"] + valid_values_test(enum_class, valid, [4] * len(valid)) + + +def test_enum_off_nominal(): + """ + Tests the EnumType serialization and deserialization + """ + members = {"MEMB1": 0, "MEMB2": 6, "MEMB3": 9} + enum_class = EnumType.construct_type("SomeEnum", members) + invalid = ["MEMB12", "MEMB22", "MEMB23"] + invalid_values_test(enum_class, invalid, EnumMismatchException) + invalid_values_test( + enum_class, + filter(lambda item: not isinstance(item, str), PYTHON_TESTABLE_TYPES), + ) + + +def test_string_nominal(): + """Tests named string types""" + py_string = "ABC123DEF456" + string_type = StringType.construct_type("MyFancyString", max_length=10) + valid_values_test( + string_type, [py_string[:10], py_string[:4], py_string[:7]], [12, 6, 9] + ) + +def test_string_off_nominal(): + """Tests named string types""" + py_string = "ABC123DEF456" + string_type = StringType.construct_type("MyFancyString", max_length=10) + invalid_values_test(string_type, [py_string], StringSizeException) + invalid_values_test( + string_type, + filter(lambda item: not isinstance(item, str), PYTHON_TESTABLE_TYPES), + ) + + +def test_serializable_basic(): + """Serializable type with basic member types""" + member_list = [ + [ + ("member1", U32Type, "%d"), + ("member2", U32Type, "%lu"), + ("member3", I64Type, "%lld"), + ], + [ + ( + "member4", + StringType.construct_type("StringMember1", max_length=10), + "%s", + ), + ("member5", StringType.construct_type("StringMember2", max_length=4), "%s"), + ("member6", I64Type, "%lld"), + ], + ] + valid_values = [ + ({"member1": 123, "member2": 456, "member3": -234}, 4 + 4 + 8), + ({"member4": "345", "member5": "abc1", "member6": 213}, 5 + 6 + 8), + ] + + for index, (members, (valid, size)) in enumerate(zip(member_list, valid_values)): + serializable_type = SerializableType.construct_type( + f"BasicSerializable{index}", members + ) + valid_values_test(serializable_type, [valid], [size]) -def check_cloned_member_list(members1, members2): - """Check member list knowing direct compares don't work""" - for tuple1, tuple2 in zip(members1, members2): - assert tuple1[0] == tuple2[0], "Names do not match" - assert tuple1[2] == tuple2[2], "Format strings do not match" - assert tuple1[3] == tuple2[3], "Descriptions do not match" - assert tuple1[1].val == tuple2[1].val, "Values don't match" +def test_serializable_basic_off_nominal(): + """Serializable type with basic member types""" + member_list = [ + ("member1", U32Type, "%d"), + ("member2", StringType.construct_type("StringMember1", max_length=10), "%s"), + ] + serializable_type = SerializableType.construct_type( + f"BasicInvalidSerializable", member_list + ) + + invalid_values = [ + ({"member5": 123, "member6": 456}, MissingMemberException), + ({"member1": "345", "member2": 123}, TypeMismatchException), + ( + {"member1": 345, "member2": "234", "member3": "Something"}, + IncorrectMembersException, + ), + ] + + for valid, exception_class in invalid_values: + invalid_values_test(serializable_type, [valid], exception_class) + invalid_values_test( + serializable_type, + filter(lambda item: not isinstance(item, dict), PYTHON_TESTABLE_TYPES), + ) -def test_serializable_type(): + +def test_serializable_advanced(): """ Tests the SerializableType serialization and deserialization """ - u32Mem = U32Type(1000000) - stringMem = StringType("something to say") - members = {"MEMB1": 0, "MEMB2": 6, "MEMB3": 9} - enumMem = EnumType("SomeEnum", members, "MEMB3") - memList = [ - ("mem1", u32Mem, ">i"), - ("mem2", stringMem, ">H"), - ("mem3", enumMem, ">i"), + + # First setup some classes to represent various member types ensuring that the serializable can handle them + string_member_class = StringType.construct_type("StringMember", max_length=3) + enum_member_class = EnumType.construct_type( + "EnumMember", {"Option1": 0, "Option2": 6, "Option3": 9} + ) + array_member_class = ArrayType.construct_type( + "ArrayMember", string_member_class, 3, "%s" + ) + sub_serializable_class = SerializableType.construct_type( + "AdvancedSubSerializable", + [("subfield1", U32Type), ("subfield2", array_member_class)], + ) + + field_data = [ + ("field1", string_member_class), + ("field2", U32Type), + ("field3", enum_member_class), + ("field4", array_member_class), + ("field5", sub_serializable_class), ] - serType1 = SerializableType("ASerType", memList) - buff = serType1.serialize() - serType2 = SerializableType("ASerType", memList) - serType2.deserialize(buff, 0) - check_cloned_member_list(serType1.mem_list, serType2.mem_list) - - assert serType1.val == serType2.val - - i32Mem = I32Type(-1000000) - stringMem = StringType("something else to say") - members = {"MEMB1": 4, "MEMB2": 2, "MEMB3": 0} - enumMem = EnumType("SomeEnum", members, "MEMB3") - memList = [ - ("mem1", i32Mem, ">i"), - ("mem2", stringMem, ">H"), - ("mem3", enumMem, ">i"), + serializable_class = SerializableType.construct_type( + "AdvancedSerializable", field_data + ) + + serializable1 = { + "field1": "abc", + "field2": 123, + "field3": "Option2", + "field4": ["", "123", "6"], + "field5": {"subfield1": 3234, "subfield2": ["abc", "def", "abc"]}, + } + valid_values_test( + serializable_class, + [serializable1], + [5 + 4 + 4 + (2 + 5 + 3) + (4 + (5 + 5 + 5))], + ) + + +def test_array_type(): + """ + Tests the ArrayType serialization and deserialization + """ + extra_ctor_args = [ + ("TestArray", I32Type, 2, "%d"), + ("TestArray2", U8Type, 4, "%d"), + ( + "TestArray3", + StringType.construct_type("TestArrayString", max_length=18), + 3, + "%s", + ), ] - serType1 = SerializableType("ASerType", memList) - buff = serType1.serialize() - serType2 = SerializableType("ASerType", memList) - serType2.deserialize(buff, 0) - check_cloned_member_list(serType1.mem_list, serType2.mem_list) - - value_dict = {"mem1": 3, "mem2": "abc 123", "mem3": "MEMB1"} - serType1.val = value_dict - assert serType1.val == value_dict - mem_list = serType1.mem_list - memList = [(a, b, c, None) for a, b, c in memList] - check_cloned_member_list(mem_list, memList) - - serTypeEmpty = SerializableType("ASerType", []) - assert serTypeEmpty.val == {} - assert serTypeEmpty.mem_list == [] - - -# def test_array_type(): -# """ -# Tests the ArrayType serialization and deserialization -# """ -# extra_ctor_args = [("TestArray", (I32Type, 2, "I DON'T KNOW")), ("TestArray2", (U8Type, 4, "I DON'T KNOW")), -# ("TestArray3", (StringType, 1, "I DON'T KNOW"))] -# values = [[32, 1], [0, 1, 2, 3], ["one"]] -# sizes = [8, 4, 3] -# -# valid_values_test(ArrayType, values, sizes, extra_ctor_args) + values = [[32, 1], [0, 1, 2, 3], ["one", "1234", "1"]] + sizes = [8, 4, 14] + for ctor_args, values, size in zip(extra_ctor_args, values, sizes): + type_input = ArrayType.construct_type(*ctor_args) + valid_values_test(type_input, [values], [size]) + + +def test_array_type_off_nominal(): + """ + Test the array type for invalid values etc + """ + type_input = ArrayType.construct_type("TestArrayPicky", I32Type, 4, "%d") + invalid_inputs = [ + ([1, 2, 3], ArrayLengthException), + ([1, 2, 3, 4, 5, 6], ArrayLengthException), + (["one", "two", "three", "four"], TypeMismatchException), + ] + for invalid, exception_class in invalid_inputs: + invalid_values_test(type_input, [invalid], exception_class) + invalid_values_test( + type_input, + filter(lambda item: not isinstance(item, (list, tuple)), PYTHON_TESTABLE_TYPES), + ) def test_time_type(): @@ -364,3 +482,31 @@ def test_base_type(): # raw abstract method, which is the only way to # raise an `AbstractMethodException`. d.deserialize("a", 0) + + +def test_dictionary_type_errors(): + """Ensure the dictionary type is preventing errors""" + # Check no raw calls passing in DictionaryType + with pytest.raises(AssertionError): + DictionaryType.construct_type( + DictionaryType, "MyNewString", PROPERTY="one", PROPERTY2="two" + ) + + # Check consistent field definitions: field values + DictionaryType.construct_type(str, "MyNewString1", PROPERTY="one", PROPERTY2="two") + with pytest.raises(AssertionError): + DictionaryType.construct_type( + str, "MyNewString1", PROPERTY="three", PROPERTY2="four" + ) + + # Check consistent field definitions: field names + DictionaryType.construct_type(str, "MyNewString2", PROPERTY1="one", PROPERTY2="two") + with pytest.raises(AssertionError): + DictionaryType.construct_type( + str, "MyNewString2", PROPERTY="one", PROPERTY3="two" + ) + + # Check consistent field definitions: missing field + DictionaryType.construct_type(str, "MyNewString3", PROPERTY1="one", PROPERTY2="two") + with pytest.raises(AssertionError): + DictionaryType.construct_type(str, "MyNewString3", PROPERTY1="one")