From 8f17a5b944604b4af59aa922724c2edc9c71c38f Mon Sep 17 00:00:00 2001 From: Michael D Starch Date: Thu, 31 Mar 2022 10:12:22 -0700 Subject: [PATCH 01/15] lestarch: refactoring data model architecture to avoid deepcopies --- .../common/models/serialize/array_type.py | 80 +++------ .../common/models/serialize/enum_type.py | 68 ++++---- .../models/serialize/numerical_types.py | 152 ++++++++++++++-- .../models/serialize/serializable_type.py | 165 +++++++----------- .../common/models/serialize/string_type.py | 36 ++-- .../common/models/serialize/type_base.py | 31 ++++ .../models/serialize/type_exceptions.py | 6 + 7 files changed, 320 insertions(+), 218 deletions(-) diff --git a/src/fprime/common/models/serialize/array_type.py b/src/fprime/common/models/serialize/array_type.py index 19f98a21..b2e96de3 100644 --- a/src/fprime/common/models/serialize/array_type.py +++ b/src/fprime/common/models/serialize/array_type.py @@ -5,7 +5,7 @@ """ import copy -from .type_base import ValueType +from .type_base import DictionaryType from .type_exceptions import ( ArrayLengthException, NotInitializedException, @@ -16,39 +16,35 @@ from fprime.util.string_util import format_string_template -class ArrayType(ValueType): +class ArrayType(DictionaryType): """Generic fixed-size array type representation. Represents a custom named type of a fixed number of like members, each of which are other types in the system. """ - def __init__(self, typename, config_info, val=None): - """Constructs a new array type. + @classmethod + def construct_type(cls, name, member_type, length, format): + """ Constructs a sub-array type + + Constructs a new sub-type of array to represent an array of the given name, member type, length, and format + string. Args: - typename: name of this array type - config_info: (type, size, format string) information for array - val: (optional) list of values to assign to array + name: name of the array subtype + member_type: type of the members of the array subtype + length: length of the array subtype + format: format string for members of the array subtype """ - super().__init__() - if not isinstance(typename, str): - raise TypeMismatchException(str, type(typename)) - self.__val = None - self.__typename = typename - self.__arr_type, self.__arr_size, self.__arr_format = config_info - # Set value only if it is a valid, non-empty list - if not val: - return - self.val = val + return DictionaryType.construct_type(cls, name, MEMBER_TYPE=member_type, LENGTH=length, FORMAT=format) def validate(self, val): """Validates the values of the array""" - size = self.__arr_size - if len(val) != size: - raise ArrayLengthException(self.__arr_type, size, len(val)) - for i in range(self.__arr_size): - if not isinstance(val[i], type(self.__arr_type)): - raise TypeMismatchException(type(self.__arr_type), type(val[i])) + if len(val) != self.LENGTH: + raise ArrayLengthException(self.MEMBER_TYPE, self.LENGTH, len(val)) + for i in range(self.LENGTH): + if not isinstance(val[i], self.MEMBER_TYPE): + raise TypeMismatchException(self.MEMBER_TYPE, type(val[i])) + val[i].validate(val) @property def val(self) -> list: @@ -73,7 +69,7 @@ def formatted_val(self) -> list: if isinstance(item, (serializable_type.SerializableType, ArrayType)): result.append(item.formatted_val) else: - result.append(format_string_template(self.__arr_format, item.val)) + result.append(format_string_template(self.FORMAT, item.val)) return result @val.setter @@ -85,11 +81,8 @@ def val(self, val: list): :param val: dictionary containing python types to key names. This """ - items = [] - for item in val: - cloned = copy.deepcopy(self.arr_type) - cloned.val = item - items.append(cloned) + items = [self.MEMBER_TYPE(val) for item in val] + self.validate(val) self.__val = items def to_jsonable(self): @@ -97,10 +90,10 @@ def to_jsonable(self): JSONable type """ members = { - "name": self.__typename, - "type": self.__typename, - "size": self.__arr_size, - "format": self.__arr_format, + "name": self.__class__.__name__, + "type": self.__class__.__name__, + "size": self.LENGTH, + "format": self.FORMAT, "values": None if self.__val is None else [member.to_jsonable() for member in self.__val], @@ -116,27 +109,12 @@ def serialize(self): def deserialize(self, data, offset): """Deserialize the members of the array""" values = [] - for i in range(self.__arr_size): - item = copy.deepcopy(self.arr_type) + for i in range(self.LENGTH): + item = self.MEMBER_TYPE() item.deserialize(data, offset + i * item.getSize()) values.append(item.val) self.val = values - @property - def arr_type(self): - """Property representing the size of the array""" - return self.__arr_type - - @property - def arr_size(self): - """Property representing the number of elements of the array""" - return self.__arr_size - - @property - def arr_format(self): - """Property representing the format string of an item in the array""" - return self.__arr_format - def getSize(self): """Return the size of the array""" - return self.arr_type.getSize() * self.arr_size + return sum([item.getSize() for item in self.__val]) diff --git a/src/fprime/common/models/serialize/enum_type.py b/src/fprime/common/models/serialize/enum_type.py index ff73ade4..ea0aba64 100644 --- a/src/fprime/common/models/serialize/enum_type.py +++ b/src/fprime/common/models/serialize/enum_type.py @@ -4,7 +4,7 @@ """ import struct -from .type_base import ValueType +from .type_base import DictionaryType from .type_exceptions import ( DeserializeException, EnumMismatchException, @@ -14,7 +14,7 @@ ) -class EnumType(ValueType): +class EnumType(DictionaryType): """ Representation of the ENUM type. @@ -23,51 +23,45 @@ class EnumType(ValueType): containing code based on C enum rules """ - def __init__(self, typename="", enum_dict=None, val=None): + def __init__(self, val="UNDEFINED"): + """ Construct the enumeration value, called through sub-type constructor + + Args: + val: (optional) value this instance of enumeration is set to. Default: "UNDEFINED" """ - Constructor + super().__init__(val) + + + @classmethod + def construct_type(cls, name, enum_dict=None): + """ Construct the custom enum type - :param typename: name of the enumeration type - :param enum_dict: dictionary of value to integer representation - :param val: value of the enumeration + Constructs the custom enumeration type, with the supplied enumeration dictionary. + + Args: + name: name of the enumeration type + enum_dict: enumeration: value dictionary defining the enumeration """ - super().__init__() - if not isinstance(typename, str): - raise TypeMismatchException(str, type(val)) - self.__typename = typename - # Setup the enum dictionary - if enum_dict is None: - enum_dict = {"UNDEFINED": 0} - # Check if the enum dict is an instance of dictionary - self.__enum_dict = enum_dict - # Set val to undefined if not set - if val is None: - val = "UNDEFINED" - self.val = val + enum_dict = enum_dict if enum_dict is not None else {"UNDEFINED": 0} + if not isinstance(enum_dict, dict): + raise TypeMismatchException(dict, type(enum_dict)) + for member in enum_dict.keys(): + if not isinstance(member, str): + raise TypeMismatchException(str, type(member)) + elif not isinstance(enum_dict[member], int): + raise TypeMismatchException(int, enum_dict[member]) + return DictionaryType.construct_type(cls, name, ENUM_DICT=enum_dict) def validate(self, val): """Validate the value passed into the enumeration""" - if not isinstance(self.enum_dict(), dict): - raise TypeMismatchException(dict, type(self.enum_dict())) - for member in self.keys(): - if not isinstance(member, str): - raise TypeMismatchException(str, type(member)) - elif not isinstance(self.enum_dict()[member], int): - raise TypeMismatchException(int, self.enum_dict()[member]) if val != "UNDEFINED" and val not in self.keys(): - raise EnumMismatchException(self.__typename, val) + raise EnumMismatchException(self.__class__.__name__, val) def keys(self): """ Return all the enum key values. """ - return list(self.enum_dict().keys()) - - def typename(self): - return self.__typename - - def enum_dict(self): - return self.__enum_dict + return list(self.ENUM_DICT.keys()) def serialize(self): """ @@ -77,7 +71,7 @@ def serialize(self): # the numeric equivalent if self.val is None: raise NotInitializedException(type(self)) - return struct.pack(">i", self.enum_dict()[self.val]) + return struct.pack(">i", self.ENUM_DICT[self.val]) def deserialize(self, data, offset): """ @@ -89,7 +83,7 @@ def deserialize(self, data, offset): raise DeserializeException( f"Could not deserialize enum value. Needed: {self.getSize()} bytes Found: {len(data[offset:])}" ) - for key, val in self.enum_dict().items(): + for key, val in self.ENUM_DICT.items(): if int_val == val: self.val = key break diff --git a/src/fprime/common/models/serialize/numerical_types.py b/src/fprime/common/models/serialize/numerical_types.py index 48c3009b..081a4082 100644 --- a/src/fprime/common/models/serialize/numerical_types.py +++ b/src/fprime/common/models/serialize/numerical_types.py @@ -18,25 +18,21 @@ TypeRangeException, ) -BITS_RE = re.compile(r"[IUF](\d\d?)") +#BITS_RE = re.compile(r"[IUF](\d\d?)") class NumericalType(ValueType, abc.ABC): """Numerical types that can be serialized using struct and are of some power of 2 byte width""" @classmethod + @abc.abstractmethod def get_bits(cls): """Gets the integer bits of a given type""" - match = BITS_RE.match(cls.__name__) - assert ( - match - ), f"Type {cls} does not follow format I#Type U#Type nor F#Type required of numerical types" - return int(match.group(1)) @classmethod def getSize(cls): """Gets the size of the integer based on the size specified in the class name""" - return int(cls.get_bits() / 8) + return int(cls.get_bits() >> 3) # Divide by 8 quickly @staticmethod @abc.abstractmethod @@ -61,16 +57,17 @@ def deserialize(self, data, offset): class IntegerType(NumericalType, abc.ABC): """Base class that represents all integer common functions""" + @classmethod + @abc.abstractmethod + def range(cls): + """Gets signed/unsigned of this type""" + def validate(self, val): """Validates the given integer.""" if not isinstance(val, int): raise TypeMismatchException(int, type(val)) - max_val = 1 << ( - self.get_bits() - (1 if self.__class__.__name__.startswith("I") else 0) - ) - min_val = -max_val if self.__class__.__name__.startswith("I") else 0 - # Compare to min and max - if val < min_val or val >= max_val: + min_val, max_val = self.range() + if val < min_val or val > max_val: raise TypeRangeException(val) @@ -86,6 +83,16 @@ def validate(self, val): class I8Type(IntegerType): """Single byte integer type. Represents C chars""" + @classmethod + def range(cls): + """Gets signed/unsigned of this type""" + return (-128, 127) + + @classmethod + def get_bits(cls): + """ Get the bit count of this type """ + return 8 + @staticmethod def get_serialize_format(): """Allows serialization using struct""" @@ -95,6 +102,22 @@ def get_serialize_format(): class I16Type(IntegerType): """Double byte integer type. Represents C shorts""" + @classmethod + def range(cls): + """Gets signed/unsigned of this type""" + return (-32768, 32767) + + @classmethod + def is_signed(cls): + """Gets signed/unsigned of this type""" + return True + + @classmethod + def get_bits(cls): + """ Get the bit count of this type """ + return 16 + + @staticmethod def get_serialize_format(): """Allows serialization using struct""" @@ -104,6 +127,21 @@ def get_serialize_format(): class I32Type(IntegerType): """Four byte integer type. Represents C int32_t,""" + @classmethod + def range(cls): + """Gets signed/unsigned of this type""" + return (-2147483648, 2147483647) + + @classmethod + def is_signed(cls): + """Gets signed/unsigned of this type""" + return True + + @classmethod + def get_bits(cls): + """ Get the bit count of this type """ + return 32 + @staticmethod def get_serialize_format(): """Allows serialization using struct""" @@ -113,6 +151,22 @@ def get_serialize_format(): class I64Type(IntegerType): """Eight byte integer type. Represents C int64_t,""" + @classmethod + def range(cls): + """Gets signed/unsigned of this type""" + return (-9223372036854775808, 9223372036854775807) + + @classmethod + def is_signed(cls): + """Gets signed/unsigned of this type""" + return True + + @classmethod + def get_bits(cls): + """ Get the bit count of this type """ + return 64 + + @staticmethod def get_serialize_format(): """Allows serialization using struct""" @@ -122,6 +176,22 @@ def get_serialize_format(): class U8Type(IntegerType): """Single byte integer type. Represents C chars""" + @classmethod + def range(cls): + """Gets signed/unsigned of this type""" + return (0, 0xFF) + + @classmethod + def is_signed(cls): + """Gets signed/unsigned of this type""" + return False + + @classmethod + def get_bits(cls): + """ Get the bit count of this type """ + return 8 + + @staticmethod def get_serialize_format(): """Allows serialization using struct""" @@ -131,6 +201,21 @@ def get_serialize_format(): class U16Type(IntegerType): """Double byte integer type. Represents C shorts""" + @classmethod + def range(cls): + """Gets signed/unsigned of this type""" + return (0, 0xFFFF) + + @classmethod + def is_signed(cls): + """Gets signed/unsigned of this type""" + return False + + @classmethod + def get_bits(cls): + """ Get the bit count of this type """ + return 16 + @staticmethod def get_serialize_format(): """Allows serialization using struct""" @@ -140,6 +225,21 @@ def get_serialize_format(): class U32Type(IntegerType): """Four byte integer type. Represents C unt32_t,""" + @classmethod + def range(cls): + """Gets signed/unsigned of this type""" + return (0, 0xFFFFFFFF) + + @classmethod + def is_signed(cls): + """Gets signed/unsigned of this type""" + return False + + @classmethod + def get_bits(cls): + """ Get the bit count of this type """ + return 32 + @staticmethod def get_serialize_format(): """Allows serialization using struct""" @@ -149,6 +249,22 @@ def get_serialize_format(): class U64Type(IntegerType): """Eight byte integer type. Represents C unt64_t,""" + @classmethod + def range(cls): + """Gets signed/unsigned of this type""" + return (0, 0xFFFFFFFFFFFFFFFF) + + @classmethod + def is_signed(cls): + """Gets signed/unsigned of this type""" + return False + + @classmethod + def get_bits(cls): + """ Get the bit count of this type """ + return 32 + + @staticmethod def get_serialize_format(): """Allows serialization using struct""" @@ -158,6 +274,11 @@ def get_serialize_format(): class F32Type(FloatType): """Eight byte integer type. Represents C unt64_t,""" + @classmethod + def get_bits(cls): + """ Get the bit count of this type """ + return 32 + @staticmethod def get_serialize_format(): """Allows serialization using struct""" @@ -167,6 +288,11 @@ def get_serialize_format(): class F64Type(FloatType): """Eight byte integer type. Represents C unt64_t,""" + @classmethod + def get_bits(cls): + """ Get the bit count of this type """ + return 64 + @staticmethod def get_serialize_format(): """Allows serialization using struct""" diff --git a/src/fprime/common/models/serialize/serializable_type.py b/src/fprime/common/models/serialize/serializable_type.py index 86146bae..7c9e27b1 100644 --- a/src/fprime/common/models/serialize/serializable_type.py +++ b/src/fprime/common/models/serialize/serializable_type.py @@ -6,14 +6,14 @@ """ import copy -from .type_base import BaseType, ValueType -from .type_exceptions import NotInitializedException, TypeMismatchException +from .type_base import BaseType, DictionaryType +from .type_exceptions import MissingMemberException, NotInitializedException, TypeMismatchException from . import array_type from fprime.util.string_util import format_string_template -class SerializableType(ValueType): +class SerializableType(DictionaryType): """ Representation of the Serializable type (comparable to the ANY type) @@ -27,83 +27,50 @@ class SerializableType(ValueType): The member descriptions can be None """ - def __init__(self, typename, mem_list=None): - """ - Constructor + @classmethod + def construct_type(cls, name, member_list=None): + """ Consturct a new serializable sub-type + + Constructs a new serializable subtype from the supplied member list and name. Member list may optionaly excluede + description keys, which will be filled with None. + + Args: + name: name of the new sub-type + member_list: list of member definitions in form list of tuples (name, type, format string, description) """ - super().__init__() - if not isinstance(typename, str): - raise TypeMismatchException(str, type(typename)) - self.__typename = typename - - new_mem_list = [] - # If the member list is defined, stamp in None for any missing descriptions - if mem_list: - new_mem_list = [ - entry if len(entry) == 4 else (entry[0], entry[1], entry[2], None) - for entry in mem_list - ] - # Set the member list then set the value - self.mem_list = new_mem_list + if member_list: + member_list = [item if len(item) == 4 else (item[0], item[1], item[2], None) for item in member_list] + # Check that we are dealing with a list + if not isinstance(member_list, list): + raise TypeMismatchException(list, type(self.mem_list)) + # Check the validity of the member list + for member_name, member_type, format_string, description in member_list: + # Check each of these members for correct types + if not isinstance(member_name, str): + raise TypeMismatchException(str, type(member_name)) + elif not issubclass(member_type, BaseType): + raise TypeMismatchException(BaseType, member_type) + elif not isinstance(format_string, str): + raise TypeMismatchException(str, type(format_string)) + elif not isinstance(description, (type(None), str)): + raise TypeMismatchException(str, type(description)) + return DictionaryType.construct_type(cls, name, MEMBER_LIST=member_list) def validate(self, val=None): """Validate this object including member list and values""" - # Blank member list does not validate - if not self.mem_list: + if not self.MEMBER_LIST or not val: return - elif not isinstance(self.mem_list, list): - raise TypeMismatchException(list, self.mem_list) - for member_name, member_val, format_string, description in self.mem_list: - # Check each of these members for correct types - if not isinstance(member_name, str): - raise TypeMismatchException(str, type(member_name)) - elif not isinstance(member_val, BaseType): - raise TypeMismatchException(BaseType, type(member_val)) - elif not isinstance(format_string, str): - raise TypeMismatchException(str, type(format_string)) - elif description is not None and not isinstance(description, str): - raise TypeMismatchException(str, type(description)) - # When a value is set and is not empty we need to set the member properties - if not val: - return - # If a value is supplied then check each value against the member list - for val_member, list_entry in zip(val, self.mem_list): - _, member_list_val, _, _ = list_entry - member_list_val.validate( - val_member - ) # Insure that the the val_member is consistent with the existing member - - @property - def mem_list(self): - """Gets the member list""" - return self.__mem_list - - @mem_list.setter - def mem_list(self, mem_list): - """ - Sets the member list and validates against the current value. - """ - self.__mem_list = mem_list - if mem_list is None: - self.validate(self.mem_list) - - def serialize(self): - """Serializes the members of the serializable""" - if self.mem_list is None: - raise NotInitializedException(type(self)) - return b"".join( - [member_val.serialize() for _, member_val, _, _ in self.mem_list] - ) - - def deserialize(self, data, offset): - """Deserialize the values of each of the members""" - new_member_list = [] - for entry1, member_val, entry3, entry4 in self.mem_list: - cloned = copy.copy(member_val) - cloned.deserialize(data, offset) - new_member_list.append((entry1, cloned, entry3, entry4)) - offset += member_val.getSize() - self.mem_list = new_member_list + # Ensure that the supplied value is a dictionary + if not isinstance(val, dict): + raise TypeMismatchException(dict, type(val)) + # Now validate each field as defined via the value + for member_name, member_type, _, _ in self.MEMBER_LIST: + member_val = val.get(member_name, None) + if not member_val: + raise MissingMemberException(member_name) + elif not isinstance(member_val, member_type): + raise TypeMismatchException(type(member_val), member_type) + member_val.validate(member_val.val) @property def val(self) -> dict: @@ -114,23 +81,8 @@ def val(self) -> dict: :return dictionary of member names to python values of member keys """ return { - member_name: member_val.val - for member_name, member_val, _, _ in self.mem_list - } - - @property - def formatted_val(self) -> dict: - """ - Format all the members of dict according to the member_format. - Note 1: All elements will be cast to str - Note 2: If a member is an array will call array formatted_val - :return a formatted dict - """ - return { - member_name: member_val.formatted_val - if isinstance(member_val, (array_type.ArrayType, SerializableType)) - else format_string_template(member_format, member_val.val) - for member_name, member_val, member_format, _ in self.mem_list + member_name: self.__val.get(member_name).val + for member_name, _, _, _ in self.MEMBER_LIST } @val.setter @@ -142,15 +94,30 @@ def val(self, val: dict): :param val: dictionary containing python types to key names. This """ - values_list = [val[name] for name, _, _, _ in self.mem_list] - # Member list is the explicit store for storing these values - for val_member, list_entry in zip(values_list, self.mem_list): - _, member_list_val, _, _ = list_entry - member_list_val.val = val_member + self.validate(val) + self.__val = {member_name: member_type(val.get(member_name)) for memer_name, member_type, _, _, in self.MEMBER_LIST} + + def serialize(self): + """Serializes the members of the serializable""" + if self.MEMBER_LIST is None: + raise NotInitializedException(type(self)) + return b"".join( + [self.__val.get(member_name).serialize() for member_name, _, _, _ in self.MEMBER_LIST] + ) + + def deserialize(self, data, offset): + """Deserialize the values of each of the members""" + new_value = {} + for member_name, member_type, _, _ in self.MEMBER_LIST: + new_member = member_type() + member_type.deserialize(new_member, offset) + new_value[member_name] = new_member + offset += new_member.getSize() + self.__val = new_value def getSize(self): """The size of a struct is the size of all the members""" - return sum(mem_type.getSize() for _, mem_type, _, _ in self.mem_list) + return sum(self.__val.get(name).getSize() for name, _, _, _ in self.MEMBER_LIST) def to_jsonable(self): """ @@ -159,5 +126,5 @@ def to_jsonable(self): members = {} for member_name, member_value, member_format, member_desc in self.mem_list: members[member_name] = {"format": member_format, "description": member_desc} - members[member_name].update(member_value.to_jsonable()) + members[member_name].update(self.__val.get(member_name).to_jsonable()) return members diff --git a/src/fprime/common/models/serialize/string_type.py b/src/fprime/common/models/serialize/string_type.py index 1bd48a0b..4cb2e617 100644 --- a/src/fprime/common/models/serialize/string_type.py +++ b/src/fprime/common/models/serialize/string_type.py @@ -16,28 +16,28 @@ TypeMismatchException, ) - -class StringType(type_base.ValueType): +class StringType(type_base.DictionaryType): """ - String type representation for F prime. This is a value type that stores a half-word first for representing the - length of this given string. + String type representation for F prime. This is a type that stores a half-word first for representing the length of + this given string. Each sub stiring class defines the sub-type property max length that represents the maximum + length of any string value stored to it. + + All string types follow this implementation, but have some specific type-based properties: MAX_LENGTH. """ - def __init__(self, val=None, max_string_len=None): - """ - Constructor to build a string - @param val: value form which to create a string. Default: None. - @param max_string: maximum length of the string. Default: None, not used. - """ - self.__max_string_len = max_string_len - super().__init__(val) + @classmethod + def construct_type(cls, name, max_length=None): + """ Constructs a new string type with given name and maximum length """ + tmp = type_base.DictionaryType.construct_type(cls, name, MAX_LENGTH=max_length) + return tmp + def validate(self, val): """Validates that this is a string""" if not isinstance(val, str): raise TypeMismatchException(str, type(val)) - elif self.__max_string_len is not None and len(val) > self.__max_string_len: - raise StringSizeException(len(val), self.__max_string_len) + elif self.MAX_LENGTH is not None and len(val) > self.MAX_LENGTH: + raise StringSizeException(len(val), self.MAX_LENGTH) def serialize(self): """ @@ -48,9 +48,9 @@ def serialize(self): raise NotInitializedException(type(self)) # Check string size before serializing elif ( - self.__max_string_len is not None and len(self.val) > self.__max_string_len + self.MAX_LENGTH is not None and len(self.val) > self.MAX_LENGTH ): - raise StringSizeException(len(self.val), self.__max_string_len) + raise StringSizeException(len(self.val), self.MAX_LENGTH) # Pack the string size first then return the encoded data buff = struct.pack(">H", len(self.val)) + self.val.encode(DATA_ENCODING) return buff @@ -67,8 +67,8 @@ def deserialize(self, data, offset): f"Not enough data to deserialize string data. Needed: {val_size} Left: {len(data[offset + 2 :])}" ) # Deal with a string that is larger than max string - elif self.__max_string_len is not None and val_size > self.__max_string_len: - raise StringSizeException(val_size, self.__max_string_len) + elif self.MAX_LENGTH is not None and val_size > self.MAX_LENGTH: + raise StringSizeException(val_size, self.MAX_LENGTH) self.val = data[offset + 2 : offset + 2 + val_size].decode(DATA_ENCODING) except struct.error: raise DeserializeException("Not enough bytes to deserialize string length.") diff --git a/src/fprime/common/models/serialize/type_base.py b/src/fprime/common/models/serialize/type_base.py index 3122fac2..c61dc968 100644 --- a/src/fprime/common/models/serialize/type_base.py +++ b/src/fprime/common/models/serialize/type_base.py @@ -89,6 +89,37 @@ def to_jsonable(self): return {"value": self.val, "type": str(self)} +class DictionaryType(ValueType, abc.ABC): + """ Type whose specification is defined in the dictionary + + Certain types in fprime (strings, serializables, enums) are defined in the dictionary. Where all projects have + access to primitave types (U8, F32, etc) and the definitions of theses types is global, other types complete + specification comes from the dictionary itself. String set max-lengths per project, serializable fields are defined, + and enumeration values are enumerated. This class is designed to take a baes framework (StringType, etc) and build + a dynamic subclass for the given dictionary defined type. + """ + _CONSTRUCTS = {} + + @classmethod + def construct_type(this_cls, cls, name, **class_properties): + """ Construct a new dictionary type + + Construct a new dynamic subtype of the given base type. This type will be named with the name parameter, define + the supplied class properties, and will be a subtype of the class. + + Args: + name: name of the new sub type + **class_properties: properties to define on the subtype (e.g. max lenght for strings) + """ + assert cls != DictionaryType, "Cannot build dictionary type from dictionary type directly" + construct = this_cls._CONSTRUCTS.get(name, type(name, (cls,), class_properties)) + for attr, value in class_properties.items(): + previous_value = getattr(construct, attr, None) + assert previous_value == value, f"Class {name} differs by attribute {attr}. {previous_value} vs {value}" + this_cls._CONSTRUCTS[name] = construct + return construct + + # # def showBytes(byteBuffer): diff --git a/src/fprime/common/models/serialize/type_exceptions.py b/src/fprime/common/models/serialize/type_exceptions.py index 0b62482c..5f2d6790 100644 --- a/src/fprime/common/models/serialize/type_exceptions.py +++ b/src/fprime/common/models/serialize/type_exceptions.py @@ -65,6 +65,12 @@ def __init__(self, enum, bad_member): super().__init__(f"Invalid enum member {bad_member} set in {enum} enum!") +class MissingMemberException(TypeException): + """ Member was not defined on type """ + def __init__(self, field): + super().__init__(f"Value does not define required field: {field}") + + class DeserializeException(TypeException): """Exception during deserialization""" From c1272d399c6afb1f015f3332c7a682fa1c714db3 Mon Sep 17 00:00:00 2001 From: Michael D Starch Date: Mon, 4 Apr 2022 13:07:33 -0700 Subject: [PATCH 02/15] mstarch: bug fixes for refactored type setup --- .../models/serialize/serializable_type.py | 25 ++++++++++++++++--- 1 file changed, 22 insertions(+), 3 deletions(-) diff --git a/src/fprime/common/models/serialize/serializable_type.py b/src/fprime/common/models/serialize/serializable_type.py index 7c9e27b1..f4a2a2bb 100644 --- a/src/fprime/common/models/serialize/serializable_type.py +++ b/src/fprime/common/models/serialize/serializable_type.py @@ -42,7 +42,7 @@ def construct_type(cls, name, member_list=None): member_list = [item if len(item) == 4 else (item[0], item[1], item[2], None) for item in member_list] # Check that we are dealing with a list if not isinstance(member_list, list): - raise TypeMismatchException(list, type(self.mem_list)) + raise TypeMismatchException(list, type(member_list)) # Check the validity of the member list for member_name, member_type, format_string, description in member_list: # Check each of these members for correct types @@ -95,7 +95,26 @@ def val(self, val: dict): :param val: dictionary containing python types to key names. This """ self.validate(val) - self.__val = {member_name: member_type(val.get(member_name)) for memer_name, member_type, _, _, in self.MEMBER_LIST} + self.__val = {member_name: member_type(val.get(member_name)) for member_name, member_type, _, _, in self.MEMBER_LIST} + + @property + def formatted_val(self) -> dict: + """ + Format all the members of dict according to the member_format. + Note 1: All elements will be cast to str + Note 2: If a member is an array will call array formatted_val + :return a formatted dict + """ + result = dict() + for member_name, _, member_format, _ in self.MEMBER_LIST: + value_object = self.__val[member_name] + if isinstance(value_object, (array_type.ArrayType, SerializableType)): + result[member_name] = value_object.formatted_val + else: + result[member_name] = format_string_template( + member_format, value_object.val + ) + return result def serialize(self): """Serializes the members of the serializable""" @@ -110,7 +129,7 @@ def deserialize(self, data, offset): new_value = {} for member_name, member_type, _, _ in self.MEMBER_LIST: new_member = member_type() - member_type.deserialize(new_member, offset) + new_member.deserialize(data, offset) new_value[member_name] = new_member offset += new_member.getSize() self.__val = new_value From 659dd26ffea0992fba6c679dde910f42cd9c6abe Mon Sep 17 00:00:00 2001 From: Michael D Starch Date: Tue, 3 May 2022 14:17:46 -0700 Subject: [PATCH 03/15] lestarch: formatted new data model architecture --- .../common/models/serialize/array_type.py | 6 ++-- .../common/models/serialize/enum_type.py | 5 ++-- .../models/serialize/numerical_types.py | 28 ++++++++----------- .../models/serialize/serializable_type.py | 23 +++++++++++---- .../common/models/serialize/string_type.py | 8 ++---- .../common/models/serialize/type_base.py | 13 ++++++--- .../models/serialize/type_exceptions.py | 3 +- 7 files changed, 50 insertions(+), 36 deletions(-) diff --git a/src/fprime/common/models/serialize/array_type.py b/src/fprime/common/models/serialize/array_type.py index b2e96de3..9c492e6a 100644 --- a/src/fprime/common/models/serialize/array_type.py +++ b/src/fprime/common/models/serialize/array_type.py @@ -24,7 +24,7 @@ class ArrayType(DictionaryType): @classmethod def construct_type(cls, name, member_type, length, format): - """ Constructs a sub-array type + """Constructs a sub-array type Constructs a new sub-type of array to represent an array of the given name, member type, length, and format string. @@ -35,7 +35,9 @@ def construct_type(cls, name, member_type, length, format): length: length of the array subtype format: format string for members of the array subtype """ - return DictionaryType.construct_type(cls, name, MEMBER_TYPE=member_type, LENGTH=length, FORMAT=format) + return DictionaryType.construct_type( + cls, name, MEMBER_TYPE=member_type, LENGTH=length, FORMAT=format + ) def validate(self, val): """Validates the values of the array""" diff --git a/src/fprime/common/models/serialize/enum_type.py b/src/fprime/common/models/serialize/enum_type.py index ea0aba64..9ccff5cb 100644 --- a/src/fprime/common/models/serialize/enum_type.py +++ b/src/fprime/common/models/serialize/enum_type.py @@ -24,17 +24,16 @@ class EnumType(DictionaryType): """ def __init__(self, val="UNDEFINED"): - """ Construct the enumeration value, called through sub-type constructor + """Construct the enumeration value, called through sub-type constructor Args: val: (optional) value this instance of enumeration is set to. Default: "UNDEFINED" """ super().__init__(val) - @classmethod def construct_type(cls, name, enum_dict=None): - """ Construct the custom enum type + """Construct the custom enum type Constructs the custom enumeration type, with the supplied enumeration dictionary. diff --git a/src/fprime/common/models/serialize/numerical_types.py b/src/fprime/common/models/serialize/numerical_types.py index 081a4082..8e19d143 100644 --- a/src/fprime/common/models/serialize/numerical_types.py +++ b/src/fprime/common/models/serialize/numerical_types.py @@ -18,7 +18,7 @@ TypeRangeException, ) -#BITS_RE = re.compile(r"[IUF](\d\d?)") +# BITS_RE = re.compile(r"[IUF](\d\d?)") class NumericalType(ValueType, abc.ABC): @@ -32,7 +32,7 @@ def get_bits(cls): @classmethod def getSize(cls): """Gets the size of the integer based on the size specified in the class name""" - return int(cls.get_bits() >> 3) # Divide by 8 quickly + return int(cls.get_bits() >> 3) # Divide by 8 quickly @staticmethod @abc.abstractmethod @@ -90,7 +90,7 @@ def range(cls): @classmethod def get_bits(cls): - """ Get the bit count of this type """ + """Get the bit count of this type""" return 8 @staticmethod @@ -114,10 +114,9 @@ def is_signed(cls): @classmethod def get_bits(cls): - """ Get the bit count of this type """ + """Get the bit count of this type""" return 16 - @staticmethod def get_serialize_format(): """Allows serialization using struct""" @@ -139,7 +138,7 @@ def is_signed(cls): @classmethod def get_bits(cls): - """ Get the bit count of this type """ + """Get the bit count of this type""" return 32 @staticmethod @@ -163,10 +162,9 @@ def is_signed(cls): @classmethod def get_bits(cls): - """ Get the bit count of this type """ + """Get the bit count of this type""" return 64 - @staticmethod def get_serialize_format(): """Allows serialization using struct""" @@ -188,10 +186,9 @@ def is_signed(cls): @classmethod def get_bits(cls): - """ Get the bit count of this type """ + """Get the bit count of this type""" return 8 - @staticmethod def get_serialize_format(): """Allows serialization using struct""" @@ -213,7 +210,7 @@ def is_signed(cls): @classmethod def get_bits(cls): - """ Get the bit count of this type """ + """Get the bit count of this type""" return 16 @staticmethod @@ -237,7 +234,7 @@ def is_signed(cls): @classmethod def get_bits(cls): - """ Get the bit count of this type """ + """Get the bit count of this type""" return 32 @staticmethod @@ -261,10 +258,9 @@ def is_signed(cls): @classmethod def get_bits(cls): - """ Get the bit count of this type """ + """Get the bit count of this type""" return 32 - @staticmethod def get_serialize_format(): """Allows serialization using struct""" @@ -276,7 +272,7 @@ class F32Type(FloatType): @classmethod def get_bits(cls): - """ Get the bit count of this type """ + """Get the bit count of this type""" return 32 @staticmethod @@ -290,7 +286,7 @@ class F64Type(FloatType): @classmethod def get_bits(cls): - """ Get the bit count of this type """ + """Get the bit count of this type""" return 64 @staticmethod diff --git a/src/fprime/common/models/serialize/serializable_type.py b/src/fprime/common/models/serialize/serializable_type.py index f4a2a2bb..275a7ed6 100644 --- a/src/fprime/common/models/serialize/serializable_type.py +++ b/src/fprime/common/models/serialize/serializable_type.py @@ -7,7 +7,11 @@ import copy from .type_base import BaseType, DictionaryType -from .type_exceptions import MissingMemberException, NotInitializedException, TypeMismatchException +from .type_exceptions import ( + MissingMemberException, + NotInitializedException, + TypeMismatchException, +) from . import array_type from fprime.util.string_util import format_string_template @@ -29,7 +33,7 @@ class SerializableType(DictionaryType): @classmethod def construct_type(cls, name, member_list=None): - """ Consturct a new serializable sub-type + """Consturct a new serializable sub-type Constructs a new serializable subtype from the supplied member list and name. Member list may optionaly excluede description keys, which will be filled with None. @@ -39,7 +43,10 @@ def construct_type(cls, name, member_list=None): member_list: list of member definitions in form list of tuples (name, type, format string, description) """ if member_list: - member_list = [item if len(item) == 4 else (item[0], item[1], item[2], None) for item in member_list] + member_list = [ + item if len(item) == 4 else (item[0], item[1], item[2], None) + for item in member_list + ] # Check that we are dealing with a list if not isinstance(member_list, list): raise TypeMismatchException(list, type(member_list)) @@ -95,7 +102,10 @@ def val(self, val: dict): :param val: dictionary containing python types to key names. This """ self.validate(val) - self.__val = {member_name: member_type(val.get(member_name)) for member_name, member_type, _, _, in self.MEMBER_LIST} + self.__val = { + member_name: member_type(val.get(member_name)) + for member_name, member_type, _, _, in self.MEMBER_LIST + } @property def formatted_val(self) -> dict: @@ -121,7 +131,10 @@ def serialize(self): if self.MEMBER_LIST is None: raise NotInitializedException(type(self)) return b"".join( - [self.__val.get(member_name).serialize() for member_name, _, _, _ in self.MEMBER_LIST] + [ + self.__val.get(member_name).serialize() + for member_name, _, _, _ in self.MEMBER_LIST + ] ) def deserialize(self, data, offset): diff --git a/src/fprime/common/models/serialize/string_type.py b/src/fprime/common/models/serialize/string_type.py index 4cb2e617..bfa1b832 100644 --- a/src/fprime/common/models/serialize/string_type.py +++ b/src/fprime/common/models/serialize/string_type.py @@ -16,6 +16,7 @@ TypeMismatchException, ) + class StringType(type_base.DictionaryType): """ String type representation for F prime. This is a type that stores a half-word first for representing the length of @@ -27,11 +28,10 @@ class StringType(type_base.DictionaryType): @classmethod def construct_type(cls, name, max_length=None): - """ Constructs a new string type with given name and maximum length """ + """Constructs a new string type with given name and maximum length""" tmp = type_base.DictionaryType.construct_type(cls, name, MAX_LENGTH=max_length) return tmp - def validate(self, val): """Validates that this is a string""" if not isinstance(val, str): @@ -47,9 +47,7 @@ def serialize(self): if self.val is None: raise NotInitializedException(type(self)) # Check string size before serializing - elif ( - self.MAX_LENGTH is not None and len(self.val) > self.MAX_LENGTH - ): + elif self.MAX_LENGTH is not None and len(self.val) > self.MAX_LENGTH: raise StringSizeException(len(self.val), self.MAX_LENGTH) # Pack the string size first then return the encoded data buff = struct.pack(">H", len(self.val)) + self.val.encode(DATA_ENCODING) diff --git a/src/fprime/common/models/serialize/type_base.py b/src/fprime/common/models/serialize/type_base.py index c61dc968..49c087a1 100644 --- a/src/fprime/common/models/serialize/type_base.py +++ b/src/fprime/common/models/serialize/type_base.py @@ -90,7 +90,7 @@ def to_jsonable(self): class DictionaryType(ValueType, abc.ABC): - """ Type whose specification is defined in the dictionary + """Type whose specification is defined in the dictionary Certain types in fprime (strings, serializables, enums) are defined in the dictionary. Where all projects have access to primitave types (U8, F32, etc) and the definitions of theses types is global, other types complete @@ -98,11 +98,12 @@ class DictionaryType(ValueType, abc.ABC): and enumeration values are enumerated. This class is designed to take a baes framework (StringType, etc) and build a dynamic subclass for the given dictionary defined type. """ + _CONSTRUCTS = {} @classmethod def construct_type(this_cls, cls, name, **class_properties): - """ Construct a new dictionary type + """Construct a new dictionary type Construct a new dynamic subtype of the given base type. This type will be named with the name parameter, define the supplied class properties, and will be a subtype of the class. @@ -111,11 +112,15 @@ def construct_type(this_cls, cls, name, **class_properties): name: name of the new sub type **class_properties: properties to define on the subtype (e.g. max lenght for strings) """ - assert cls != DictionaryType, "Cannot build dictionary type from dictionary type directly" + assert ( + cls != DictionaryType + ), "Cannot build dictionary type from dictionary type directly" construct = this_cls._CONSTRUCTS.get(name, type(name, (cls,), class_properties)) for attr, value in class_properties.items(): previous_value = getattr(construct, attr, None) - assert previous_value == value, f"Class {name} differs by attribute {attr}. {previous_value} vs {value}" + assert ( + previous_value == value + ), f"Class {name} differs by attribute {attr}. {previous_value} vs {value}" this_cls._CONSTRUCTS[name] = construct return construct diff --git a/src/fprime/common/models/serialize/type_exceptions.py b/src/fprime/common/models/serialize/type_exceptions.py index 5f2d6790..1af8b79e 100644 --- a/src/fprime/common/models/serialize/type_exceptions.py +++ b/src/fprime/common/models/serialize/type_exceptions.py @@ -66,7 +66,8 @@ def __init__(self, enum, bad_member): class MissingMemberException(TypeException): - """ Member was not defined on type """ + """Member was not defined on type""" + def __init__(self, field): super().__init__(f"Value does not define required field: {field}") From a97738e29921539ccf1bdede91e0ff55cf74087e Mon Sep 17 00:00:00 2001 From: Michael D Starch Date: Tue, 3 May 2022 14:28:55 -0700 Subject: [PATCH 04/15] lestarch: sp --- .github/actions/spelling/expect.txt | 3 +++ .../common/models/serialize/serializable_type.py | 4 ++-- src/fprime/common/models/serialize/string_type.py | 8 +++++--- src/fprime/common/models/serialize/type_base.py | 14 +++++++------- 4 files changed, 17 insertions(+), 12 deletions(-) diff --git a/.github/actions/spelling/expect.txt b/.github/actions/spelling/expect.txt index 91944cb4..d4af5e38 100644 --- a/.github/actions/spelling/expect.txt +++ b/.github/actions/spelling/expect.txt @@ -12,6 +12,7 @@ autoapi autocoded autocode Autocoders +autocoders autocoding autodoc autoescape @@ -118,6 +119,7 @@ isdir isfile isinstance isnumeric +issubclass iterdir itertools itle @@ -200,6 +202,7 @@ SCLK scm sdd Serializables +serializables setuptools shutil someotherpath diff --git a/src/fprime/common/models/serialize/serializable_type.py b/src/fprime/common/models/serialize/serializable_type.py index 275a7ed6..15b9b085 100644 --- a/src/fprime/common/models/serialize/serializable_type.py +++ b/src/fprime/common/models/serialize/serializable_type.py @@ -33,9 +33,9 @@ class SerializableType(DictionaryType): @classmethod def construct_type(cls, name, member_list=None): - """Consturct a new serializable sub-type + """Construct a new serializable sub-type - Constructs a new serializable subtype from the supplied member list and name. Member list may optionaly excluede + Constructs a new serializable subtype from the supplied member list and name. Member list may optionally exclude description keys, which will be filled with None. Args: diff --git a/src/fprime/common/models/serialize/string_type.py b/src/fprime/common/models/serialize/string_type.py index bfa1b832..aecaa8f6 100644 --- a/src/fprime/common/models/serialize/string_type.py +++ b/src/fprime/common/models/serialize/string_type.py @@ -20,7 +20,7 @@ class StringType(type_base.DictionaryType): """ String type representation for F prime. This is a type that stores a half-word first for representing the length of - this given string. Each sub stiring class defines the sub-type property max length that represents the maximum + this given string. Each sub string class defines the sub-type property max length that represents the maximum length of any string value stored to it. All string types follow this implementation, but have some specific type-based properties: MAX_LENGTH. @@ -29,8 +29,10 @@ class StringType(type_base.DictionaryType): @classmethod def construct_type(cls, name, max_length=None): """Constructs a new string type with given name and maximum length""" - tmp = type_base.DictionaryType.construct_type(cls, name, MAX_LENGTH=max_length) - return tmp + temporary = type_base.DictionaryType.construct_type( + cls, name, MAX_LENGTH=max_length + ) + return temporary def validate(self, val): """Validates that this is a string""" diff --git a/src/fprime/common/models/serialize/type_base.py b/src/fprime/common/models/serialize/type_base.py index 49c087a1..181fc658 100644 --- a/src/fprime/common/models/serialize/type_base.py +++ b/src/fprime/common/models/serialize/type_base.py @@ -93,10 +93,10 @@ class DictionaryType(ValueType, abc.ABC): """Type whose specification is defined in the dictionary Certain types in fprime (strings, serializables, enums) are defined in the dictionary. Where all projects have - access to primitave types (U8, F32, etc) and the definitions of theses types is global, other types complete + access to primitive types (U8, F32, etc) and the definitions of theses types is global, other types complete specification comes from the dictionary itself. String set max-lengths per project, serializable fields are defined, - and enumeration values are enumerated. This class is designed to take a baes framework (StringType, etc) and build - a dynamic subclass for the given dictionary defined type. + and enumeration values are enumerated. This class is designed to take base complex types (StringType, etc) and build + dynamic subclasses for the given dictionary defined type. """ _CONSTRUCTS = {} @@ -110,17 +110,17 @@ def construct_type(this_cls, cls, name, **class_properties): Args: name: name of the new sub type - **class_properties: properties to define on the subtype (e.g. max lenght for strings) + **class_properties: properties to define on the subtype (e.g. max length for strings) """ assert ( cls != DictionaryType ), "Cannot build dictionary type from dictionary type directly" construct = this_cls._CONSTRUCTS.get(name, type(name, (cls,), class_properties)) - for attr, value in class_properties.items(): - previous_value = getattr(construct, attr, None) + for attribute, value in class_properties.items(): + previous_value = getattr(construct, attribute, None) assert ( previous_value == value - ), f"Class {name} differs by attribute {attr}. {previous_value} vs {value}" + ), f"Class {name} differs by attribute {attribute}. {previous_value} vs {value}" this_cls._CONSTRUCTS[name] = construct return construct From f737b8e75ae7d3028392e7e2ef31ac918025b143 Mon Sep 17 00:00:00 2001 From: Michael D Starch Date: Thu, 5 May 2022 13:44:52 -0700 Subject: [PATCH 05/15] lestarch: fixing UTs, models consistency --- .../common/models/serialize/array_type.py | 17 ++-- .../common/models/serialize/bool_type.py | 5 +- .../common/models/serialize/enum_type.py | 12 ++- .../models/serialize/numerical_types.py | 10 +- .../models/serialize/serializable_type.py | 20 ++-- .../common/models/serialize/string_type.py | 7 +- .../common/models/serialize/type_base.py | 13 ++- .../common/models/serialize/test_types.py | 92 ++++++++++--------- 8 files changed, 94 insertions(+), 82 deletions(-) diff --git a/src/fprime/common/models/serialize/array_type.py b/src/fprime/common/models/serialize/array_type.py index 9c492e6a..4bc8455e 100644 --- a/src/fprime/common/models/serialize/array_type.py +++ b/src/fprime/common/models/serialize/array_type.py @@ -39,14 +39,13 @@ def construct_type(cls, name, member_type, length, format): cls, name, MEMBER_TYPE=member_type, LENGTH=length, FORMAT=format ) - def validate(self, val): + @classmethod + def validate(cls, val): """Validates the values of the array""" - if len(val) != self.LENGTH: - raise ArrayLengthException(self.MEMBER_TYPE, self.LENGTH, len(val)) - for i in range(self.LENGTH): - if not isinstance(val[i], self.MEMBER_TYPE): - raise TypeMismatchException(self.MEMBER_TYPE, type(val[i])) - val[i].validate(val) + if len(val) != cls.LENGTH: + raise ArrayLengthException(cls.MEMBER_TYPE, cls.LENGTH, len(val)) + for i in range(cls.LENGTH): + cls.MEMBER_TYPE.validate(val[i]) @property def val(self) -> list: @@ -83,8 +82,8 @@ def val(self, val: list): :param val: dictionary containing python types to key names. This """ - items = [self.MEMBER_TYPE(val) for item in val] self.validate(val) + items = [self.MEMBER_TYPE(item) for item in val] self.__val = items def to_jsonable(self): @@ -106,7 +105,7 @@ def serialize(self): """Serialize the array by serializing the elements one by one""" if self.val is None: raise NotInitializedException(type(self)) - return b"".join([item.serialize() for item in self.val]) + return b"".join([item.serialize() for item in self.__val]) def deserialize(self, data, offset): """Deserialize the members of the array""" diff --git a/src/fprime/common/models/serialize/bool_type.py b/src/fprime/common/models/serialize/bool_type.py index 10ee8b21..20eaf7a4 100644 --- a/src/fprime/common/models/serialize/bool_type.py +++ b/src/fprime/common/models/serialize/bool_type.py @@ -22,7 +22,8 @@ class BoolType(ValueType): TRUE = 0xFF FALSE = 0x00 - def validate(self, val): + @classmethod + def validate(cls, val): """Validate the given class""" if not isinstance(val, bool): raise TypeMismatchException(bool, type(val)) @@ -31,7 +32,7 @@ def serialize(self): """Serialize a boolean value""" if self.val is None: raise NotInitializedException(type(self)) - return struct.pack("B", 0xFF if self.val else 0x00) + return struct.pack("B", self.TRUE if self.val else self.FALSE) def deserialize(self, data, offset): """Deserialize boolean value""" diff --git a/src/fprime/common/models/serialize/enum_type.py b/src/fprime/common/models/serialize/enum_type.py index 9ccff5cb..c59b5e81 100644 --- a/src/fprime/common/models/serialize/enum_type.py +++ b/src/fprime/common/models/serialize/enum_type.py @@ -51,16 +51,18 @@ def construct_type(cls, name, enum_dict=None): raise TypeMismatchException(int, enum_dict[member]) return DictionaryType.construct_type(cls, name, ENUM_DICT=enum_dict) - def validate(self, val): + @classmethod + def validate(cls, val): """Validate the value passed into the enumeration""" - if val != "UNDEFINED" and val not in self.keys(): - raise EnumMismatchException(self.__class__.__name__, val) + if val != "UNDEFINED" and val not in cls.keys(): + raise EnumMismatchException(cls.__class__.__name__, val) - def keys(self): + @classmethod + def keys(cls): """ Return all the enum key values. """ - return list(self.ENUM_DICT.keys()) + return list(cls.ENUM_DICT.keys()) def serialize(self): """ diff --git a/src/fprime/common/models/serialize/numerical_types.py b/src/fprime/common/models/serialize/numerical_types.py index 8e19d143..e3f4760c 100644 --- a/src/fprime/common/models/serialize/numerical_types.py +++ b/src/fprime/common/models/serialize/numerical_types.py @@ -62,11 +62,12 @@ class IntegerType(NumericalType, abc.ABC): def range(cls): """Gets signed/unsigned of this type""" - def validate(self, val): + @classmethod + def validate(cls, val): """Validates the given integer.""" if not isinstance(val, int): raise TypeMismatchException(int, type(val)) - min_val, max_val = self.range() + min_val, max_val = cls.range() if val < min_val or val > max_val: raise TypeRangeException(val) @@ -74,7 +75,8 @@ def validate(self, val): class FloatType(NumericalType, abc.ABC): """Base class that represents all float common functions""" - def validate(self, val): + @classmethod + def validate(cls, val): """Validates the given integer.""" if not isinstance(val, (float, int)): raise TypeMismatchException(float, type(val)) @@ -259,7 +261,7 @@ def is_signed(cls): @classmethod def get_bits(cls): """Get the bit count of this type""" - return 32 + return 64 @staticmethod def get_serialize_format(): diff --git a/src/fprime/common/models/serialize/serializable_type.py b/src/fprime/common/models/serialize/serializable_type.py index 15b9b085..92cabdf5 100644 --- a/src/fprime/common/models/serialize/serializable_type.py +++ b/src/fprime/common/models/serialize/serializable_type.py @@ -43,10 +43,7 @@ def construct_type(cls, name, member_list=None): member_list: list of member definitions in form list of tuples (name, type, format string, description) """ if member_list: - member_list = [ - item if len(item) == 4 else (item[0], item[1], item[2], None) - for item in member_list - ] + member_list = [list(item) + ([None] * (4 -len(item))) for item in member_list] # Check that we are dealing with a list if not isinstance(member_list, list): raise TypeMismatchException(list, type(member_list)) @@ -57,27 +54,26 @@ def construct_type(cls, name, member_list=None): raise TypeMismatchException(str, type(member_name)) elif not issubclass(member_type, BaseType): raise TypeMismatchException(BaseType, member_type) - elif not isinstance(format_string, str): + elif format_string is not None and not isinstance(format_string, str): raise TypeMismatchException(str, type(format_string)) - elif not isinstance(description, (type(None), str)): + elif description is not None and not isinstance(description, str): raise TypeMismatchException(str, type(description)) return DictionaryType.construct_type(cls, name, MEMBER_LIST=member_list) - def validate(self, val=None): + @classmethod + def validate(cls, val): """Validate this object including member list and values""" - if not self.MEMBER_LIST or not val: + if not cls.MEMBER_LIST: return # Ensure that the supplied value is a dictionary if not isinstance(val, dict): raise TypeMismatchException(dict, type(val)) # Now validate each field as defined via the value - for member_name, member_type, _, _ in self.MEMBER_LIST: + for member_name, member_type, _, _ in cls.MEMBER_LIST: member_val = val.get(member_name, None) if not member_val: raise MissingMemberException(member_name) - elif not isinstance(member_val, member_type): - raise TypeMismatchException(type(member_val), member_type) - member_val.validate(member_val.val) + member_type.validate(member_val) @property def val(self) -> dict: diff --git a/src/fprime/common/models/serialize/string_type.py b/src/fprime/common/models/serialize/string_type.py index aecaa8f6..7e8d10ad 100644 --- a/src/fprime/common/models/serialize/string_type.py +++ b/src/fprime/common/models/serialize/string_type.py @@ -34,12 +34,13 @@ def construct_type(cls, name, max_length=None): ) return temporary - def validate(self, val): + @classmethod + def validate(cls, val): """Validates that this is a string""" if not isinstance(val, str): raise TypeMismatchException(str, type(val)) - elif self.MAX_LENGTH is not None and len(val) > self.MAX_LENGTH: - raise StringSizeException(len(val), self.MAX_LENGTH) + elif cls.MAX_LENGTH is not None and len(val) > cls.MAX_LENGTH: + raise StringSizeException(len(val), cls.MAX_LENGTH) def serialize(self): """ diff --git a/src/fprime/common/models/serialize/type_base.py b/src/fprime/common/models/serialize/type_base.py index 181fc658..b573629c 100644 --- a/src/fprime/common/models/serialize/type_base.py +++ b/src/fprime/common/models/serialize/type_base.py @@ -61,16 +61,25 @@ def __init__(self, val=None): if val is not None: self.val = val + @classmethod @abc.abstractmethod - def validate(self, val): + def validate(cls, val): """ Checks the val for validity with respect to the current type. This will raise TypeMismatchException when the - validation fails of the val's type fails. It will raise TypeRangeException when val is out of range. + validation fails of the val's type fails. It will raise TypeRangeException when val is out of range. Concrete + implementations will raise other exceptions based on type. For example, serializable types raise exceptions for + missing members. :param val: value to validate :raises TypeMismatchException: value has incorrect type, TypeRangeException: val is out of range """ + def __eq__(self, other): + """ Check equality between types """ + if type(other) != type(self): + return False + return self.__val == other.__val + @property def val(self): """Getter for .val""" diff --git a/test/fprime/common/models/serialize/test_types.py b/test/fprime/common/models/serialize/test_types.py index f34e850d..fccf228e 100644 --- a/test/fprime/common/models/serialize/test_types.py +++ b/test/fprime/common/models/serialize/test_types.py @@ -33,6 +33,7 @@ AbstractMethodException, DeserializeException, NotInitializedException, + StringSizeException, TypeMismatchException, TypeRangeException, ) @@ -220,9 +221,10 @@ def test_enum_type(): Tests the EnumType serialization and deserialization """ members = {"MEMB1": 0, "MEMB2": 6, "MEMB3": 9} - val1 = EnumType("SomeEnum", members, "MEMB3") + enum_class = EnumType.construct_type("SomeEnum", members) + val1 = enum_class("MEMB3") buff = val1.serialize() - val2 = EnumType("SomeEnum", members) + val2 = enum_class() val2.deserialize(buff, 0) assert val1.val == val2.val @@ -236,52 +238,52 @@ def check_cloned_member_list(members1, members2): assert tuple1[1].val == tuple2[1].val, "Values don't match" -def test_serializable_type(): +def test_string_type(): + """ Tests named string types """ + py_string = "ABC123DEF456" + string_type = StringType.construct_type("MyFancyString", max_length=10) + + # Test a bad string + with pytest.raises(StringSizeException): + string_val1 = string_type(py_string) + string_val1 = string_type(py_string[:10]) + + string_val2 = string_type() + serialized = string_val1.serialize() + string_val2.deserialize(serialized, 0) + assert string_val2.val == py_string[:10] + + +def test_serializable_basic(): + """ Serializable type with basic member types """ + member_list = [("member1", U32Type, "%d"),("member2", U32Type, "%lu"),("member3", I64Type, "%lld")] + serializable_type = SerializableType.construct_type("BasicSerializable", member_list) + serializable1 = serializable_type({"member1": 123, "member2": 456, "member3": -234}) + bytes1 = serializable1.serialize() + serializable2 = serializable_type() + serializable2.deserialize(bytes1, 0) + assert serializable1 == serializable2, "Serializable not equal" + + + +def test_serializable_advanced(): """ Tests the SerializableType serialization and deserialization """ - u32Mem = U32Type(1000000) - stringMem = StringType("something to say") - members = {"MEMB1": 0, "MEMB2": 6, "MEMB3": 9} - enumMem = EnumType("SomeEnum", members, "MEMB3") - memList = [ - ("mem1", u32Mem, ">i"), - ("mem2", stringMem, ">H"), - ("mem3", enumMem, ">i"), - ] - serType1 = SerializableType("ASerType", memList) - buff = serType1.serialize() - serType2 = SerializableType("ASerType", memList) - serType2.deserialize(buff, 0) - check_cloned_member_list(serType1.mem_list, serType2.mem_list) - - assert serType1.val == serType2.val - - i32Mem = I32Type(-1000000) - stringMem = StringType("something else to say") - members = {"MEMB1": 4, "MEMB2": 2, "MEMB3": 0} - enumMem = EnumType("SomeEnum", members, "MEMB3") - memList = [ - ("mem1", i32Mem, ">i"), - ("mem2", stringMem, ">H"), - ("mem3", enumMem, ">i"), - ] - serType1 = SerializableType("ASerType", memList) - buff = serType1.serialize() - serType2 = SerializableType("ASerType", memList) - serType2.deserialize(buff, 0) - check_cloned_member_list(serType1.mem_list, serType2.mem_list) - - value_dict = {"mem1": 3, "mem2": "abc 123", "mem3": "MEMB1"} - serType1.val = value_dict - assert serType1.val == value_dict - mem_list = serType1.mem_list - memList = [(a, b, c, None) for a, b, c in memList] - check_cloned_member_list(mem_list, memList) - - serTypeEmpty = SerializableType("ASerType", []) - assert serTypeEmpty.val == {} - assert serTypeEmpty.mem_list == [] + + # First setup some classes to represent various member types ensuring that the serializable can handle them + string_member_class = StringType.construct_type("StringMember") + enum_member_class = EnumType.construct_type("EnumMember", {"Option1": 0, "Option2": 6, "Option3": 9}) + array_member_class = ArrayType.construct_type("ArrayMember", string_member_class, 3, "%s") + + field_data = [("field1", string_member_class), ("field2", U32Type), ("field3", enum_member_class), ("field4", array_member_class)] + serializable_class = SerializableType.construct_type("AdvancedSerializable", field_data) + + serializable1 = serializable_class({"field1": "abc", "field2": 123, "field3": "Option2", "field4": ["abc", "123", "456"]}) + bytes1 = serializable1.serialize() + serializable2 = serializable_class() + assert serializable1 == serializable2, "Serializables do not match" + # def test_array_type(): From 1d5e811bc8243ae70422652e0781bdb8a9f43e21 Mon Sep 17 00:00:00 2001 From: Michael D Starch Date: Mon, 9 May 2022 08:49:38 -0700 Subject: [PATCH 06/15] lestarch: better models UTs and fixes there in --- .../common/models/serialize/array_type.py | 19 +++-- .../common/models/serialize/enum_type.py | 2 +- .../models/serialize/serializable_type.py | 14 ++-- .../common/models/serialize/type_base.py | 8 +- .../common/models/serialize/test_types.py | 84 +++++++++---------- 5 files changed, 63 insertions(+), 64 deletions(-) diff --git a/src/fprime/common/models/serialize/array_type.py b/src/fprime/common/models/serialize/array_type.py index 4bc8455e..fa96fc25 100644 --- a/src/fprime/common/models/serialize/array_type.py +++ b/src/fprime/common/models/serialize/array_type.py @@ -55,7 +55,9 @@ def val(self) -> list: :return dictionary of member names to python values of member keys """ - return [item.val for item in self.__val] + if self._val is None: + return None + return [item.val for item in self._val] @property def formatted_val(self) -> list: @@ -66,7 +68,7 @@ def formatted_val(self) -> list: :return a formatted array """ result = [] - for item in self.__val: + for item in self._val: if isinstance(item, (serializable_type.SerializableType, ArrayType)): result.append(item.formatted_val) else: @@ -84,7 +86,7 @@ def val(self, val: list): """ self.validate(val) items = [self.MEMBER_TYPE(item) for item in val] - self.__val = items + self._val = items def to_jsonable(self): """ @@ -96,8 +98,8 @@ def to_jsonable(self): "size": self.LENGTH, "format": self.FORMAT, "values": None - if self.__val is None - else [member.to_jsonable() for member in self.__val], + if self._val is None + else [member.to_jsonable() for member in self._val], } return members @@ -105,17 +107,18 @@ def serialize(self): """Serialize the array by serializing the elements one by one""" if self.val is None: raise NotInitializedException(type(self)) - return b"".join([item.serialize() for item in self.__val]) + return b"".join([item.serialize() for item in self._val]) def deserialize(self, data, offset): """Deserialize the members of the array""" values = [] for i in range(self.LENGTH): item = self.MEMBER_TYPE() - item.deserialize(data, offset + i * item.getSize()) + item.deserialize(data, offset) + offset += item.getSize() values.append(item.val) self.val = values def getSize(self): """Return the size of the array""" - return sum([item.getSize() for item in self.__val]) + return sum([item.getSize() for item in self._val]) diff --git a/src/fprime/common/models/serialize/enum_type.py b/src/fprime/common/models/serialize/enum_type.py index c59b5e81..7975c90c 100644 --- a/src/fprime/common/models/serialize/enum_type.py +++ b/src/fprime/common/models/serialize/enum_type.py @@ -70,7 +70,7 @@ def serialize(self): """ # for enums, take the string value and convert it to # the numeric equivalent - if self.val is None: + if self._val is None or (self._val == "UNDEFINED" and "UNDEFINED" not in self.ENUM_DICT): raise NotInitializedException(type(self)) return struct.pack(">i", self.ENUM_DICT[self.val]) diff --git a/src/fprime/common/models/serialize/serializable_type.py b/src/fprime/common/models/serialize/serializable_type.py index 92cabdf5..1fbaabd6 100644 --- a/src/fprime/common/models/serialize/serializable_type.py +++ b/src/fprime/common/models/serialize/serializable_type.py @@ -84,7 +84,7 @@ def val(self) -> dict: :return dictionary of member names to python values of member keys """ return { - member_name: self.__val.get(member_name).val + member_name: self._val.get(member_name).val for member_name, _, _, _ in self.MEMBER_LIST } @@ -98,7 +98,7 @@ def val(self, val: dict): :param val: dictionary containing python types to key names. This """ self.validate(val) - self.__val = { + self._val = { member_name: member_type(val.get(member_name)) for member_name, member_type, _, _, in self.MEMBER_LIST } @@ -113,7 +113,7 @@ def formatted_val(self) -> dict: """ result = dict() for member_name, _, member_format, _ in self.MEMBER_LIST: - value_object = self.__val[member_name] + value_object = self._val[member_name] if isinstance(value_object, (array_type.ArrayType, SerializableType)): result[member_name] = value_object.formatted_val else: @@ -128,7 +128,7 @@ def serialize(self): raise NotInitializedException(type(self)) return b"".join( [ - self.__val.get(member_name).serialize() + self._val.get(member_name).serialize() for member_name, _, _, _ in self.MEMBER_LIST ] ) @@ -141,11 +141,11 @@ def deserialize(self, data, offset): new_member.deserialize(data, offset) new_value[member_name] = new_member offset += new_member.getSize() - self.__val = new_value + self._val = new_value def getSize(self): """The size of a struct is the size of all the members""" - return sum(self.__val.get(name).getSize() for name, _, _, _ in self.MEMBER_LIST) + return sum(self._val.get(name).getSize() for name, _, _, _ in self.MEMBER_LIST) def to_jsonable(self): """ @@ -154,5 +154,5 @@ def to_jsonable(self): members = {} for member_name, member_value, member_format, member_desc in self.mem_list: members[member_name] = {"format": member_format, "description": member_desc} - members[member_name].update(self.__val.get(member_name).to_jsonable()) + members[member_name].update(self._val.get(member_name).to_jsonable()) return members diff --git a/src/fprime/common/models/serialize/type_base.py b/src/fprime/common/models/serialize/type_base.py index b573629c..b8f8e83e 100644 --- a/src/fprime/common/models/serialize/type_base.py +++ b/src/fprime/common/models/serialize/type_base.py @@ -56,7 +56,7 @@ class ValueType(BaseType): def __init__(self, val=None): """Defines the single value""" - self.__val = None + self._val = None # Run full setter if val is not None: self.val = val @@ -78,18 +78,18 @@ def __eq__(self, other): """ Check equality between types """ if type(other) != type(self): return False - return self.__val == other.__val + return self._val == other._val @property def val(self): """Getter for .val""" - return self.__val + return self._val @val.setter def val(self, val): """Setter for .val calls validate internally""" self.validate(val) - self.__val = val + self._val = val def to_jsonable(self): """ diff --git a/test/fprime/common/models/serialize/test_types.py b/test/fprime/common/models/serialize/test_types.py index fccf228e..c556326a 100644 --- a/test/fprime/common/models/serialize/test_types.py +++ b/test/fprime/common/models/serialize/test_types.py @@ -27,11 +27,12 @@ from fprime.common.models.serialize.serializable_type import SerializableType from fprime.common.models.serialize.string_type import StringType from fprime.common.models.serialize.time_type import TimeBase, TimeType -from fprime.common.models.serialize.type_base import BaseType, ValueType +from fprime.common.models.serialize.type_base import BaseType, ValueType, DictionaryType from fprime.common.models.serialize.type_exceptions import ( AbstractMethodException, DeserializeException, + EnumMismatchException, NotInitializedException, StringSizeException, TypeMismatchException, @@ -59,14 +60,12 @@ ] -def valid_values_test(type_input, valid_values, sizes, extras=None): +def valid_values_test(type_input, valid_values, sizes): """Tests to be run on all types""" if not isinstance(sizes, Iterable): sizes = [sizes] * len(valid_values) - # Should be able to instantiate a blank type, but not serialize it until a value has been supplied - if not extras: - extras = [[]] * len(valid_values) - instantiation = type_input(*extras[0]) + + instantiation = type_input() with pytest.raises(NotInitializedException): instantiation.serialize() # Should be able to get a JSONable object that is dumpable to a JSON string @@ -74,13 +73,13 @@ def valid_values_test(type_input, valid_values, sizes, extras=None): json.loads(json.dumps(jsonable)) # Run on valid values - for value, size, extra in zip(valid_values, sizes, extras): - instantiation = type_input(*extra, val=value) + for value, size in zip(valid_values, sizes): + instantiation = type_input(val=value) assert instantiation.val == value assert instantiation.getSize() == size # Check assignment by value - by_value = type_input(*extra) + by_value = type_input() by_value.val = value assert by_value.val == instantiation.val, "Assignment by value has failed" assert by_value.getSize() == size @@ -88,7 +87,7 @@ def valid_values_test(type_input, valid_values, sizes, extras=None): # Check serialization and deserialization serialized = instantiation.serialize() for offset in [0, 10, 50]: - deserializer = type_input(*extra) + deserializer = type_input() deserializer.deserialize((b" " * offset) + serialized, offset) assert instantiation.val == deserializer.val, "Deserialization has failed" assert deserializer.getSize() == size @@ -216,42 +215,39 @@ def test_float_types_off_nominal(): ) -def test_enum_type(): +def test_enum_nominal(): """ Tests the EnumType serialization and deserialization """ members = {"MEMB1": 0, "MEMB2": 6, "MEMB3": 9} enum_class = EnumType.construct_type("SomeEnum", members) - val1 = enum_class("MEMB3") - buff = val1.serialize() - val2 = enum_class() - val2.deserialize(buff, 0) - assert val1.val == val2.val + valid = ["MEMB1", "MEMB2", "MEMB3"] + valid_values_test(enum_class, valid, [4] * len(valid)) -def check_cloned_member_list(members1, members2): - """Check member list knowing direct compares don't work""" - for tuple1, tuple2 in zip(members1, members2): - assert tuple1[0] == tuple2[0], "Names do not match" - assert tuple1[2] == tuple2[2], "Format strings do not match" - assert tuple1[3] == tuple2[3], "Descriptions do not match" - assert tuple1[1].val == tuple2[1].val, "Values don't match" +def test_enum_off_nominal(): + """ + Tests the EnumType serialization and deserialization + """ + members = {"MEMB1": 0, "MEMB2": 6, "MEMB3": 9} + enum_class = EnumType.construct_type("SomeEnum", members) + valid = ["MEMB12", "MEMB22", "MEMB23"] + invalid_values_test(enum_class, valid, EnumMismatchException) -def test_string_type(): +def test_string_nominal(): """ Tests named string types """ py_string = "ABC123DEF456" string_type = StringType.construct_type("MyFancyString", max_length=10) + valid_values_test(string_type, [py_string[:10], py_string[:4], py_string[:7]], [12, 6, 9]) + - # Test a bad string - with pytest.raises(StringSizeException): - string_val1 = string_type(py_string) - string_val1 = string_type(py_string[:10]) +def test_string_off_nominal(): + """ Tests named string types """ + py_string = "ABC123DEF456" + string_type = StringType.construct_type("MyFancyString", max_length=10) + invalid_values_test(string_type, [py_string], StringSizeException) - string_val2 = string_type() - serialized = string_val1.serialize() - string_val2.deserialize(serialized, 0) - assert string_val2.val == py_string[:10] def test_serializable_basic(): @@ -265,7 +261,6 @@ def test_serializable_basic(): assert serializable1 == serializable2, "Serializable not equal" - def test_serializable_advanced(): """ Tests the SerializableType serialization and deserialization @@ -282,20 +277,21 @@ def test_serializable_advanced(): serializable1 = serializable_class({"field1": "abc", "field2": 123, "field3": "Option2", "field4": ["abc", "123", "456"]}) bytes1 = serializable1.serialize() serializable2 = serializable_class() + serializable2.deserialize(bytes1, 0) assert serializable1 == serializable2, "Serializables do not match" - -# def test_array_type(): -# """ -# Tests the ArrayType serialization and deserialization -# """ -# extra_ctor_args = [("TestArray", (I32Type, 2, "I DON'T KNOW")), ("TestArray2", (U8Type, 4, "I DON'T KNOW")), -# ("TestArray3", (StringType, 1, "I DON'T KNOW"))] -# values = [[32, 1], [0, 1, 2, 3], ["one"]] -# sizes = [8, 4, 3] -# -# valid_values_test(ArrayType, values, sizes, extra_ctor_args) +def test_array_type(): + """ + Tests the ArrayType serialization and deserialization + """ + extra_ctor_args = [("TestArray", I32Type, 2, "%d"), ("TestArray2", U8Type, 4, "%d"), + ("TestArray3", StringType.construct_type("TestArrayString", max_length=3), 1, "%s")] + values = [[32, 1], [0, 1, 2, 3], ["one"]] + sizes = [8, 4, 5] + for ctor_args, values, size in zip(extra_ctor_args, values, sizes): + type_input = ArrayType.construct_type(*ctor_args) + valid_values_test(type_input, [values], [size]) def test_time_type(): From 1c45abf41fe6475d5f2665ef5652918b41ecfea1 Mon Sep 17 00:00:00 2001 From: Michael D Starch Date: Mon, 9 May 2022 08:54:00 -0700 Subject: [PATCH 07/15] lestarch: sp --- .github/actions/spelling/expect.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/actions/spelling/expect.txt b/.github/actions/spelling/expect.txt index d4af5e38..e3449fd0 100644 --- a/.github/actions/spelling/expect.txt +++ b/.github/actions/spelling/expect.txt @@ -138,6 +138,7 @@ lestarch LGTM lgtm Linux +lld locs lstrip lxml From cdb6818557ab4ce5fcf038c3383e891465c1a5d8 Mon Sep 17 00:00:00 2001 From: Michael D Starch Date: Mon, 9 May 2022 09:07:37 -0700 Subject: [PATCH 08/15] lestarch: formatting --- .../common/models/serialize/enum_type.py | 4 +- .../models/serialize/serializable_type.py | 4 +- .../common/models/serialize/type_base.py | 2 +- .../common/models/serialize/test_types.py | 61 ++++++++++++++----- 4 files changed, 54 insertions(+), 17 deletions(-) diff --git a/src/fprime/common/models/serialize/enum_type.py b/src/fprime/common/models/serialize/enum_type.py index 7975c90c..e3377344 100644 --- a/src/fprime/common/models/serialize/enum_type.py +++ b/src/fprime/common/models/serialize/enum_type.py @@ -70,7 +70,9 @@ def serialize(self): """ # for enums, take the string value and convert it to # the numeric equivalent - if self._val is None or (self._val == "UNDEFINED" and "UNDEFINED" not in self.ENUM_DICT): + if self._val is None or ( + self._val == "UNDEFINED" and "UNDEFINED" not in self.ENUM_DICT + ): raise NotInitializedException(type(self)) return struct.pack(">i", self.ENUM_DICT[self.val]) diff --git a/src/fprime/common/models/serialize/serializable_type.py b/src/fprime/common/models/serialize/serializable_type.py index 1fbaabd6..23e1c3d6 100644 --- a/src/fprime/common/models/serialize/serializable_type.py +++ b/src/fprime/common/models/serialize/serializable_type.py @@ -43,7 +43,9 @@ def construct_type(cls, name, member_list=None): member_list: list of member definitions in form list of tuples (name, type, format string, description) """ if member_list: - member_list = [list(item) + ([None] * (4 -len(item))) for item in member_list] + member_list = [ + list(item) + ([None] * (4 - len(item))) for item in member_list + ] # Check that we are dealing with a list if not isinstance(member_list, list): raise TypeMismatchException(list, type(member_list)) diff --git a/src/fprime/common/models/serialize/type_base.py b/src/fprime/common/models/serialize/type_base.py index b8f8e83e..c587ac38 100644 --- a/src/fprime/common/models/serialize/type_base.py +++ b/src/fprime/common/models/serialize/type_base.py @@ -75,7 +75,7 @@ def validate(cls, val): """ def __eq__(self, other): - """ Check equality between types """ + """Check equality between types""" if type(other) != type(self): return False return self._val == other._val diff --git a/test/fprime/common/models/serialize/test_types.py b/test/fprime/common/models/serialize/test_types.py index c556326a..d8b6e969 100644 --- a/test/fprime/common/models/serialize/test_types.py +++ b/test/fprime/common/models/serialize/test_types.py @@ -236,24 +236,31 @@ def test_enum_off_nominal(): def test_string_nominal(): - """ Tests named string types """ + """Tests named string types""" py_string = "ABC123DEF456" string_type = StringType.construct_type("MyFancyString", max_length=10) - valid_values_test(string_type, [py_string[:10], py_string[:4], py_string[:7]], [12, 6, 9]) + valid_values_test( + string_type, [py_string[:10], py_string[:4], py_string[:7]], [12, 6, 9] + ) def test_string_off_nominal(): - """ Tests named string types """ + """Tests named string types""" py_string = "ABC123DEF456" string_type = StringType.construct_type("MyFancyString", max_length=10) invalid_values_test(string_type, [py_string], StringSizeException) - def test_serializable_basic(): - """ Serializable type with basic member types """ - member_list = [("member1", U32Type, "%d"),("member2", U32Type, "%lu"),("member3", I64Type, "%lld")] - serializable_type = SerializableType.construct_type("BasicSerializable", member_list) + """Serializable type with basic member types""" + member_list = [ + ("member1", U32Type, "%d"), + ("member2", U32Type, "%lu"), + ("member3", I64Type, "%lld"), + ] + serializable_type = SerializableType.construct_type( + "BasicSerializable", member_list + ) serializable1 = serializable_type({"member1": 123, "member2": 456, "member3": -234}) bytes1 = serializable1.serialize() serializable2 = serializable_type() @@ -268,13 +275,31 @@ def test_serializable_advanced(): # First setup some classes to represent various member types ensuring that the serializable can handle them string_member_class = StringType.construct_type("StringMember") - enum_member_class = EnumType.construct_type("EnumMember", {"Option1": 0, "Option2": 6, "Option3": 9}) - array_member_class = ArrayType.construct_type("ArrayMember", string_member_class, 3, "%s") + enum_member_class = EnumType.construct_type( + "EnumMember", {"Option1": 0, "Option2": 6, "Option3": 9} + ) + array_member_class = ArrayType.construct_type( + "ArrayMember", string_member_class, 3, "%s" + ) - field_data = [("field1", string_member_class), ("field2", U32Type), ("field3", enum_member_class), ("field4", array_member_class)] - serializable_class = SerializableType.construct_type("AdvancedSerializable", field_data) + field_data = [ + ("field1", string_member_class), + ("field2", U32Type), + ("field3", enum_member_class), + ("field4", array_member_class), + ] + serializable_class = SerializableType.construct_type( + "AdvancedSerializable", field_data + ) - serializable1 = serializable_class({"field1": "abc", "field2": 123, "field3": "Option2", "field4": ["abc", "123", "456"]}) + serializable1 = serializable_class( + { + "field1": "abc", + "field2": 123, + "field3": "Option2", + "field4": ["abc", "123", "456"], + } + ) bytes1 = serializable1.serialize() serializable2 = serializable_class() serializable2.deserialize(bytes1, 0) @@ -285,8 +310,16 @@ def test_array_type(): """ Tests the ArrayType serialization and deserialization """ - extra_ctor_args = [("TestArray", I32Type, 2, "%d"), ("TestArray2", U8Type, 4, "%d"), - ("TestArray3", StringType.construct_type("TestArrayString", max_length=3), 1, "%s")] + extra_ctor_args = [ + ("TestArray", I32Type, 2, "%d"), + ("TestArray2", U8Type, 4, "%d"), + ( + "TestArray3", + StringType.construct_type("TestArrayString", max_length=3), + 1, + "%s", + ), + ] values = [[32, 1], [0, 1, 2, 3], ["one"]] sizes = [8, 4, 5] for ctor_args, values, size in zip(extra_ctor_args, values, sizes): From b3efca4d90ec3d4357eb041c47f67ea40f27ca8a Mon Sep 17 00:00:00 2001 From: Michael D Starch Date: Mon, 9 May 2022 09:41:59 -0700 Subject: [PATCH 09/15] lestarch: fixing static analysis errors --- src/fprime/common/models/serialize/array_type.py | 3 --- src/fprime/common/models/serialize/bool_type.py | 2 +- src/fprime/common/models/serialize/numerical_types.py | 9 +++------ src/fprime/common/models/serialize/serializable_type.py | 2 -- 4 files changed, 4 insertions(+), 12 deletions(-) diff --git a/src/fprime/common/models/serialize/array_type.py b/src/fprime/common/models/serialize/array_type.py index fa96fc25..de5fafd3 100644 --- a/src/fprime/common/models/serialize/array_type.py +++ b/src/fprime/common/models/serialize/array_type.py @@ -3,13 +3,10 @@ Created on May 29, 2020 @author: jishii """ -import copy - from .type_base import DictionaryType from .type_exceptions import ( ArrayLengthException, NotInitializedException, - TypeMismatchException, ) from . import serializable_type diff --git a/src/fprime/common/models/serialize/bool_type.py b/src/fprime/common/models/serialize/bool_type.py index 20eaf7a4..8e9087e7 100644 --- a/src/fprime/common/models/serialize/bool_type.py +++ b/src/fprime/common/models/serialize/bool_type.py @@ -32,7 +32,7 @@ def serialize(self): """Serialize a boolean value""" if self.val is None: raise NotInitializedException(type(self)) - return struct.pack("B", self.TRUE if self.val else self.FALSE) + return struct.pack("B", self.TRUE if self._val else self.FALSE) def deserialize(self, data, offset): """Deserialize boolean value""" diff --git a/src/fprime/common/models/serialize/numerical_types.py b/src/fprime/common/models/serialize/numerical_types.py index e3f4760c..6da2617b 100644 --- a/src/fprime/common/models/serialize/numerical_types.py +++ b/src/fprime/common/models/serialize/numerical_types.py @@ -7,7 +7,6 @@ @author mstarch """ import abc -import re import struct from .type_base import ValueType @@ -18,8 +17,6 @@ TypeRangeException, ) -# BITS_RE = re.compile(r"[IUF](\d\d?)") - class NumericalType(ValueType, abc.ABC): """Numerical types that can be serialized using struct and are of some power of 2 byte width""" @@ -42,14 +39,14 @@ def get_serialize_format(): def serialize(self): """Serializes this type using struct and the val property""" - if self.val is None: + if self._val is None: raise NotInitializedException(type(self)) - return struct.pack(self.get_serialize_format(), self.val) + return struct.pack(self.get_serialize_format(), self._val) def deserialize(self, data, offset): """Serializes this type using struct and the val property""" try: - self.val = struct.unpack_from(self.get_serialize_format(), data, offset)[0] + self._val = struct.unpack_from(self.get_serialize_format(), data, offset)[0] except struct.error as err: raise DeserializeException(str(err)) diff --git a/src/fprime/common/models/serialize/serializable_type.py b/src/fprime/common/models/serialize/serializable_type.py index 23e1c3d6..d5cb3118 100644 --- a/src/fprime/common/models/serialize/serializable_type.py +++ b/src/fprime/common/models/serialize/serializable_type.py @@ -4,8 +4,6 @@ @author: tcanham """ -import copy - from .type_base import BaseType, DictionaryType from .type_exceptions import ( MissingMemberException, From ff3a181a71b73e01125c869e912bf38ee71c0744 Mon Sep 17 00:00:00 2001 From: Michael D Starch Date: Mon, 9 May 2022 10:07:43 -0700 Subject: [PATCH 10/15] lestarch: more static analysis fixes --- src/fprime/common/models/serialize/bool_type.py | 2 +- src/fprime/fbuild/gcovr.py | 1 - src/fprime/util/build_helper.py | 2 -- 3 files changed, 1 insertion(+), 4 deletions(-) diff --git a/src/fprime/common/models/serialize/bool_type.py b/src/fprime/common/models/serialize/bool_type.py index 8e9087e7..3c371037 100644 --- a/src/fprime/common/models/serialize/bool_type.py +++ b/src/fprime/common/models/serialize/bool_type.py @@ -30,7 +30,7 @@ def validate(cls, val): def serialize(self): """Serialize a boolean value""" - if self.val is None: + if self._val is None: raise NotInitializedException(type(self)) return struct.pack("B", self.TRUE if self._val else self.FALSE) diff --git a/src/fprime/fbuild/gcovr.py b/src/fprime/fbuild/gcovr.py index 0c213f25..dbcf6d01 100644 --- a/src/fprime/fbuild/gcovr.py +++ b/src/fprime/fbuild/gcovr.py @@ -7,7 +7,6 @@ from typing import Dict, List, Tuple, Union from .target import ExecutableAction, Target, TargetScope, CompositeTarget -from .types import MissingBuildCachePath TEMPORARY_DIRECTORY = "{{AUTOCODE}}" diff --git a/src/fprime/util/build_helper.py b/src/fprime/util/build_helper.py index 8ad3ff14..2d56ed49 100644 --- a/src/fprime/util/build_helper.py +++ b/src/fprime/util/build_helper.py @@ -183,8 +183,6 @@ def utility_entry(args): parsed, cmake_args, make_args, parser, runners = parse_args(args) try: - cwd = Path(parsed.path) - try: target = get_target(parsed) build_type = target.build_type From 9b8d0d3c58bba77e1da80a89ebe404d2dc570b34 Mon Sep 17 00:00:00 2001 From: Michael D Starch Date: Mon, 9 May 2022 10:44:46 -0700 Subject: [PATCH 11/15] lestarch: no more .val usage in BoolType implementation --- src/fprime/common/models/serialize/bool_type.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/fprime/common/models/serialize/bool_type.py b/src/fprime/common/models/serialize/bool_type.py index 3c371037..c9db5954 100644 --- a/src/fprime/common/models/serialize/bool_type.py +++ b/src/fprime/common/models/serialize/bool_type.py @@ -40,7 +40,7 @@ def deserialize(self, data, offset): int_val = struct.unpack_from("B", data, offset)[0] if int_val not in [self.TRUE, self.FALSE]: raise TypeRangeException(int_val) - self.val = int_val == self.TRUE + self._val = int_val == self.TRUE except struct.error: raise DeserializeException("Not enough bytes to deserialize bool.") From 97eca6b58636f61ff4dc1140581b90488ef23b5a Mon Sep 17 00:00:00 2001 From: Michael D Starch Date: Tue, 10 May 2022 18:32:43 -0700 Subject: [PATCH 12/15] lestarch: fixing review requested items --- .../common/models/serialize/array_type.py | 5 +- .../common/models/serialize/enum_type.py | 21 +-- .../models/serialize/numerical_types.py | 35 ---- .../models/serialize/serializable_type.py | 58 ++++--- .../common/models/serialize/type_base.py | 43 ++--- .../models/serialize/type_exceptions.py | 7 + .../common/models/serialize/test_types.py | 163 +++++++++++++++--- 7 files changed, 204 insertions(+), 128 deletions(-) diff --git a/src/fprime/common/models/serialize/array_type.py b/src/fprime/common/models/serialize/array_type.py index de5fafd3..770aa44d 100644 --- a/src/fprime/common/models/serialize/array_type.py +++ b/src/fprime/common/models/serialize/array_type.py @@ -7,6 +7,7 @@ from .type_exceptions import ( ArrayLengthException, NotInitializedException, + TypeMismatchException, ) from . import serializable_type @@ -39,7 +40,9 @@ def construct_type(cls, name, member_type, length, format): @classmethod def validate(cls, val): """Validates the values of the array""" - if len(val) != cls.LENGTH: + if not isinstance(val, (tuple, list)): + raise TypeMismatchException(list, type(val)) + elif len(val) != cls.LENGTH: raise ArrayLengthException(cls.MEMBER_TYPE, cls.LENGTH, len(val)) for i in range(cls.LENGTH): cls.MEMBER_TYPE.validate(val[i]) diff --git a/src/fprime/common/models/serialize/enum_type.py b/src/fprime/common/models/serialize/enum_type.py index e3377344..5360e95a 100644 --- a/src/fprime/common/models/serialize/enum_type.py +++ b/src/fprime/common/models/serialize/enum_type.py @@ -23,16 +23,8 @@ class EnumType(DictionaryType): containing code based on C enum rules """ - def __init__(self, val="UNDEFINED"): - """Construct the enumeration value, called through sub-type constructor - - Args: - val: (optional) value this instance of enumeration is set to. Default: "UNDEFINED" - """ - super().__init__(val) - @classmethod - def construct_type(cls, name, enum_dict=None): + def construct_type(cls, name, enum_dict): """Construct the custom enum type Constructs the custom enumeration type, with the supplied enumeration dictionary. @@ -41,7 +33,6 @@ def construct_type(cls, name, enum_dict=None): name: name of the enumeration type enum_dict: enumeration: value dictionary defining the enumeration """ - enum_dict = enum_dict if enum_dict is not None else {"UNDEFINED": 0} if not isinstance(enum_dict, dict): raise TypeMismatchException(dict, type(enum_dict)) for member in enum_dict.keys(): @@ -54,7 +45,9 @@ def construct_type(cls, name, enum_dict=None): @classmethod def validate(cls, val): """Validate the value passed into the enumeration""" - if val != "UNDEFINED" and val not in cls.keys(): + if not isinstance(val, str): + raise TypeMismatchException(str, type(val)) + if val not in cls.keys(): raise EnumMismatchException(cls.__class__.__name__, val) @classmethod @@ -74,7 +67,7 @@ def serialize(self): self._val == "UNDEFINED" and "UNDEFINED" not in self.ENUM_DICT ): raise NotInitializedException(type(self)) - return struct.pack(">i", self.ENUM_DICT[self.val]) + return struct.pack(">i", self.ENUM_DICT[self._val]) def deserialize(self, data, offset): """ @@ -82,13 +75,13 @@ def deserialize(self, data, offset): """ try: int_val = struct.unpack_from(">i", data, offset)[0] - except: + except struct.error: raise DeserializeException( f"Could not deserialize enum value. Needed: {self.getSize()} bytes Found: {len(data[offset:])}" ) for key, val in self.ENUM_DICT.items(): if int_val == val: - self.val = key + self._val = key break # Value not found, invalid enumeration value else: diff --git a/src/fprime/common/models/serialize/numerical_types.py b/src/fprime/common/models/serialize/numerical_types.py index 6da2617b..1f12a031 100644 --- a/src/fprime/common/models/serialize/numerical_types.py +++ b/src/fprime/common/models/serialize/numerical_types.py @@ -106,11 +106,6 @@ def range(cls): """Gets signed/unsigned of this type""" return (-32768, 32767) - @classmethod - def is_signed(cls): - """Gets signed/unsigned of this type""" - return True - @classmethod def get_bits(cls): """Get the bit count of this type""" @@ -130,11 +125,6 @@ def range(cls): """Gets signed/unsigned of this type""" return (-2147483648, 2147483647) - @classmethod - def is_signed(cls): - """Gets signed/unsigned of this type""" - return True - @classmethod def get_bits(cls): """Get the bit count of this type""" @@ -154,11 +144,6 @@ def range(cls): """Gets signed/unsigned of this type""" return (-9223372036854775808, 9223372036854775807) - @classmethod - def is_signed(cls): - """Gets signed/unsigned of this type""" - return True - @classmethod def get_bits(cls): """Get the bit count of this type""" @@ -178,11 +163,6 @@ def range(cls): """Gets signed/unsigned of this type""" return (0, 0xFF) - @classmethod - def is_signed(cls): - """Gets signed/unsigned of this type""" - return False - @classmethod def get_bits(cls): """Get the bit count of this type""" @@ -202,11 +182,6 @@ def range(cls): """Gets signed/unsigned of this type""" return (0, 0xFFFF) - @classmethod - def is_signed(cls): - """Gets signed/unsigned of this type""" - return False - @classmethod def get_bits(cls): """Get the bit count of this type""" @@ -226,11 +201,6 @@ def range(cls): """Gets signed/unsigned of this type""" return (0, 0xFFFFFFFF) - @classmethod - def is_signed(cls): - """Gets signed/unsigned of this type""" - return False - @classmethod def get_bits(cls): """Get the bit count of this type""" @@ -250,11 +220,6 @@ def range(cls): """Gets signed/unsigned of this type""" return (0, 0xFFFFFFFFFFFFFFFF) - @classmethod - def is_signed(cls): - """Gets signed/unsigned of this type""" - return False - @classmethod def get_bits(cls): """Get the bit count of this type""" diff --git a/src/fprime/common/models/serialize/serializable_type.py b/src/fprime/common/models/serialize/serializable_type.py index d5cb3118..39a0fa01 100644 --- a/src/fprime/common/models/serialize/serializable_type.py +++ b/src/fprime/common/models/serialize/serializable_type.py @@ -6,6 +6,7 @@ """ from .type_base import BaseType, DictionaryType from .type_exceptions import ( + IncorrectMembersException, MissingMemberException, NotInitializedException, TypeMismatchException, @@ -30,7 +31,7 @@ class SerializableType(DictionaryType): """ @classmethod - def construct_type(cls, name, member_list=None): + def construct_type(cls, name, member_list): """Construct a new serializable sub-type Constructs a new serializable subtype from the supplied member list and name. Member list may optionally exclude @@ -40,38 +41,36 @@ def construct_type(cls, name, member_list=None): name: name of the new sub-type member_list: list of member definitions in form list of tuples (name, type, format string, description) """ - if member_list: - member_list = [ - list(item) + ([None] * (4 - len(item))) for item in member_list - ] - # Check that we are dealing with a list - if not isinstance(member_list, list): - raise TypeMismatchException(list, type(member_list)) - # Check the validity of the member list - for member_name, member_type, format_string, description in member_list: - # Check each of these members for correct types - if not isinstance(member_name, str): - raise TypeMismatchException(str, type(member_name)) - elif not issubclass(member_type, BaseType): - raise TypeMismatchException(BaseType, member_type) - elif format_string is not None and not isinstance(format_string, str): - raise TypeMismatchException(str, type(format_string)) - elif description is not None and not isinstance(description, str): - raise TypeMismatchException(str, type(description)) + # Check that we are dealing with a list + if not isinstance(member_list, list): + raise TypeMismatchException(list, type(member_list)) + member_list = [list(item) + ([None] * (4 - len(item))) for item in member_list] + # Check the validity of the member list + for member_name, member_type, format_string, description in member_list: + # Check each of these members for correct types + if not isinstance(member_name, str): + raise TypeMismatchException(str, type(member_name)) + elif not issubclass(member_type, BaseType): + raise TypeMismatchException(BaseType, member_type) + elif format_string is not None and not isinstance(format_string, str): + raise TypeMismatchException(str, type(format_string)) + elif description is not None and not isinstance(description, str): + raise TypeMismatchException(str, type(description)) return DictionaryType.construct_type(cls, name, MEMBER_LIST=member_list) @classmethod def validate(cls, val): """Validate this object including member list and values""" - if not cls.MEMBER_LIST: - return # Ensure that the supplied value is a dictionary if not isinstance(val, dict): raise TypeMismatchException(dict, type(val)) + elif len(val) != len(cls.MEMBER_LIST): + raise IncorrectMembersException([name for name, _, _, _ in cls.MEMBER_LIST]) # Now validate each field as defined via the value for member_name, member_type, _, _ in cls.MEMBER_LIST: - member_val = val.get(member_name, None) - if not member_val: + try: + member_val = val[member_name] + except KeyError: raise MissingMemberException(member_name) member_type.validate(member_val) @@ -83,6 +82,8 @@ def val(self) -> dict: :return dictionary of member names to python values of member keys """ + if self._val is None: + return None return { member_name: self._val.get(member_name).val for member_name, _, _, _ in self.MEMBER_LIST @@ -124,7 +125,7 @@ def formatted_val(self) -> dict: def serialize(self): """Serializes the members of the serializable""" - if self.MEMBER_LIST is None: + if self._val is None: raise NotInitializedException(type(self)) return b"".join( [ @@ -152,7 +153,12 @@ def to_jsonable(self): JSONable type for a serializable """ members = {} - for member_name, member_value, member_format, member_desc in self.mem_list: + for member_name, member_value, member_format, member_desc in self.MEMBER_LIST: + value = ( + {"value": None} + if self._val is None + else self._val.get(member_name).to_jsonable() + ) members[member_name] = {"format": member_format, "description": member_desc} - members[member_name].update(self._val.get(member_name).to_jsonable()) + members[member_name].update(value) return members diff --git a/src/fprime/common/models/serialize/type_base.py b/src/fprime/common/models/serialize/type_base.py index c587ac38..8ad03f21 100644 --- a/src/fprime/common/models/serialize/type_base.py +++ b/src/fprime/common/models/serialize/type_base.py @@ -5,7 +5,6 @@ Replaced type base class with decorators """ import abc -import struct from .type_exceptions import AbstractMethodException @@ -95,7 +94,7 @@ def to_jsonable(self): """ Converts this type to a JSON serializable object """ - return {"value": self.val, "type": str(self)} + return {"value": self.val, "type": str(self.__class__)} class DictionaryType(ValueType, abc.ABC): @@ -111,42 +110,36 @@ class DictionaryType(ValueType, abc.ABC): _CONSTRUCTS = {} @classmethod - def construct_type(this_cls, cls, name, **class_properties): + def construct_type(cls, parent_class, name, **class_properties): """Construct a new dictionary type Construct a new dynamic subtype of the given base type. This type will be named with the name parameter, define - the supplied class properties, and will be a subtype of the class. + the supplied class properties, and will be a subtype of the parent class. This function registers these new + types by name in the DictionaryType._CONSTRUCTS dictionary. When a type is defined a second or more times, the + originally constructed sub type is used and the newly supplied class properties are validated for equality + against the original definition. It is an error to define the type multiple times with inconsistent class + properties. Args: + cls: DictionaryType, used to access the class property _CONSTRUCTS + parent_class: class used as parent to the new sub type name: name of the new sub type **class_properties: properties to define on the subtype (e.g. max length for strings) """ assert ( - cls != DictionaryType + parent_class != DictionaryType ), "Cannot build dictionary type from dictionary type directly" - construct = this_cls._CONSTRUCTS.get(name, type(name, (cls,), class_properties)) + construct, original_properties = cls._CONSTRUCTS.get( + name, (type(name, (parent_class,), class_properties), class_properties) + ) + # Validate both new properties against original properties and against what is set on original class + assert ( + original_properties == class_properties + ), "Different class properties specified" for attribute, value in class_properties.items(): previous_value = getattr(construct, attribute, None) assert ( previous_value == value ), f"Class {name} differs by attribute {attribute}. {previous_value} vs {value}" - this_cls._CONSTRUCTS[name] = construct + cls._CONSTRUCTS[name] = (construct, class_properties) return construct - - -# -# -def showBytes(byteBuffer): - """ - Routine to show bytes in buffer for testing. - """ - print("Byte buffer size: %d" % len(byteBuffer)) - for entry in range(0, len(byteBuffer)): - print( - "Byte %d: 0x%02X (%c)" - % ( - entry, - struct.unpack("B", bytes([byteBuffer[entry]]))[0], - struct.unpack("B", bytes([byteBuffer[entry]]))[0], - ) - ) diff --git a/src/fprime/common/models/serialize/type_exceptions.py b/src/fprime/common/models/serialize/type_exceptions.py index 1af8b79e..6afa3f15 100644 --- a/src/fprime/common/models/serialize/type_exceptions.py +++ b/src/fprime/common/models/serialize/type_exceptions.py @@ -72,6 +72,13 @@ def __init__(self, field): super().__init__(f"Value does not define required field: {field}") +class IncorrectMembersException(TypeException): + """Members incorrectly defined on type (too many, too few)""" + + def __init__(self, fields): + super().__init__(f"Value does not define required fields: {fields}") + + class DeserializeException(TypeException): """Exception during deserialization""" diff --git a/test/fprime/common/models/serialize/test_types.py b/test/fprime/common/models/serialize/test_types.py index d8b6e969..ab182606 100644 --- a/test/fprime/common/models/serialize/test_types.py +++ b/test/fprime/common/models/serialize/test_types.py @@ -27,12 +27,15 @@ from fprime.common.models.serialize.serializable_type import SerializableType from fprime.common.models.serialize.string_type import StringType from fprime.common.models.serialize.time_type import TimeBase, TimeType -from fprime.common.models.serialize.type_base import BaseType, ValueType, DictionaryType +from fprime.common.models.serialize.type_base import BaseType, DictionaryType, ValueType from fprime.common.models.serialize.type_exceptions import ( AbstractMethodException, + ArrayLengthException, DeserializeException, EnumMismatchException, + IncorrectMembersException, + MissingMemberException, NotInitializedException, StringSizeException, TypeMismatchException, @@ -68,6 +71,10 @@ def valid_values_test(type_input, valid_values, sizes): instantiation = type_input() with pytest.raises(NotInitializedException): instantiation.serialize() + assert instantiation.val is None + # Setting to None is invalid + with pytest.raises(TypeMismatchException): + instantiation.val = None # Should be able to get a JSONable object that is dumpable to a JSON string jsonable = instantiation.to_jsonable() json.loads(json.dumps(jsonable)) @@ -84,6 +91,9 @@ def valid_values_test(type_input, valid_values, sizes): assert by_value.val == instantiation.val, "Assignment by value has failed" assert by_value.getSize() == size + # Check that the value returned by .val is also assignable to .val + by_value.val = by_value.val + # Check serialization and deserialization serialized = instantiation.serialize() for offset in [0, 10, 50]: @@ -231,8 +241,12 @@ def test_enum_off_nominal(): """ members = {"MEMB1": 0, "MEMB2": 6, "MEMB3": 9} enum_class = EnumType.construct_type("SomeEnum", members) - valid = ["MEMB12", "MEMB22", "MEMB23"] - invalid_values_test(enum_class, valid, EnumMismatchException) + invalid = ["MEMB12", "MEMB22", "MEMB23"] + invalid_values_test(enum_class, invalid, EnumMismatchException) + invalid_values_test( + enum_class, + filter(lambda item: not isinstance(item, str), PYTHON_TESTABLE_TYPES), + ) def test_string_nominal(): @@ -249,23 +263,67 @@ def test_string_off_nominal(): py_string = "ABC123DEF456" string_type = StringType.construct_type("MyFancyString", max_length=10) invalid_values_test(string_type, [py_string], StringSizeException) + invalid_values_test( + string_type, + filter(lambda item: not isinstance(item, str), PYTHON_TESTABLE_TYPES), + ) def test_serializable_basic(): + """Serializable type with basic member types""" + member_list = [ + [ + ("member1", U32Type, "%d"), + ("member2", U32Type, "%lu"), + ("member3", I64Type, "%lld"), + ], + [ + ( + "member4", + StringType.construct_type("StringMember1", max_length=10), + "%s", + ), + ("member5", StringType.construct_type("StringMember2", max_length=4), "%s"), + ("member6", I64Type, "%lld"), + ], + ] + valid_values = [ + ({"member1": 123, "member2": 456, "member3": -234}, 4 + 4 + 8), + ({"member4": "345", "member5": "abcd", "member6": 213}, 5 + 6 + 8), + ] + + for index, (members, (valid, size)) in enumerate(zip(member_list, valid_values)): + serializable_type = SerializableType.construct_type( + f"BasicSerializable{index}", members + ) + valid_values_test(serializable_type, [valid], [size]) + + +def test_serializable_basic_off_nominal(): """Serializable type with basic member types""" member_list = [ ("member1", U32Type, "%d"), - ("member2", U32Type, "%lu"), - ("member3", I64Type, "%lld"), + ("member2", StringType.construct_type("StringMember1", max_length=10), "%s"), ] serializable_type = SerializableType.construct_type( - "BasicSerializable", member_list + f"BasicInvalidSerializable", member_list + ) + + invalid_values = [ + ({"member5": 123, "member6": 456}, MissingMemberException), + ({"member1": "345", "member2": 123}, TypeMismatchException), + ( + {"member1": 345, "member2": "234", "member3": "Something"}, + IncorrectMembersException, + ), + ] + + for valid, exception_class in invalid_values: + invalid_values_test(serializable_type, [valid], exception_class) + invalid_values_test( + serializable_type, + filter(lambda item: not isinstance(item, dict), PYTHON_TESTABLE_TYPES), ) - serializable1 = serializable_type({"member1": 123, "member2": 456, "member3": -234}) - bytes1 = serializable1.serialize() - serializable2 = serializable_type() - serializable2.deserialize(bytes1, 0) - assert serializable1 == serializable2, "Serializable not equal" def test_serializable_advanced(): @@ -274,36 +332,41 @@ def test_serializable_advanced(): """ # First setup some classes to represent various member types ensuring that the serializable can handle them - string_member_class = StringType.construct_type("StringMember") + string_member_class = StringType.construct_type("StringMember", max_length=3) enum_member_class = EnumType.construct_type( "EnumMember", {"Option1": 0, "Option2": 6, "Option3": 9} ) array_member_class = ArrayType.construct_type( "ArrayMember", string_member_class, 3, "%s" ) + subserializable_class = SerializableType.construct_type( + "AdvancedSubSerializable", + [("subfield1", U32Type), ("subfield2", array_member_class)], + ) field_data = [ ("field1", string_member_class), ("field2", U32Type), ("field3", enum_member_class), ("field4", array_member_class), + ("field5", subserializable_class), ] serializable_class = SerializableType.construct_type( "AdvancedSerializable", field_data ) - serializable1 = serializable_class( - { - "field1": "abc", - "field2": 123, - "field3": "Option2", - "field4": ["abc", "123", "456"], - } + serializable1 = { + "field1": "abc", + "field2": 123, + "field3": "Option2", + "field4": ["", "123", "6"], + "field5": {"subfield1": 3234, "subfield2": ["abc", "def", "abc"]}, + } + valid_values_test( + serializable_class, + [serializable1], + [5 + 4 + 4 + (2 + 5 + 3) + (4 + (5 + 5 + 5))], ) - bytes1 = serializable1.serialize() - serializable2 = serializable_class() - serializable2.deserialize(bytes1, 0) - assert serializable1 == serializable2, "Serializables do not match" def test_array_type(): @@ -315,18 +378,36 @@ def test_array_type(): ("TestArray2", U8Type, 4, "%d"), ( "TestArray3", - StringType.construct_type("TestArrayString", max_length=3), - 1, + StringType.construct_type("TestArrayString", max_length=18), + 3, "%s", ), ] - values = [[32, 1], [0, 1, 2, 3], ["one"]] - sizes = [8, 4, 5] + values = [[32, 1], [0, 1, 2, 3], ["one", "1234", "1"]] + sizes = [8, 4, 14] for ctor_args, values, size in zip(extra_ctor_args, values, sizes): type_input = ArrayType.construct_type(*ctor_args) valid_values_test(type_input, [values], [size]) +def test_array_type_off_nominal(): + """ + Test the array type for invalid values etc + """ + type_input = ArrayType.construct_type("TestArrayPicky", I32Type, 4, "%d") + invalid_inputs = [ + ([1, 2, 3], ArrayLengthException), + ([1, 2, 3, 4, 5, 6], ArrayLengthException), + (["one", "two", "three", "four"], TypeMismatchException), + ] + for invalid, exception_class in invalid_inputs: + invalid_values_test(type_input, [invalid], exception_class) + invalid_values_test( + type_input, + filter(lambda item: not isinstance(item, (list, tuple)), PYTHON_TESTABLE_TYPES), + ) + + def test_time_type(): """ Tests the TimeType serialization and deserialization @@ -395,3 +476,31 @@ def test_base_type(): # raw abstract method, which is the only way to # raise an `AbstractMethodException`. d.deserialize("a", 0) + + +def test_dictionary_type_errors(): + """Ensure the dictionary type is preventing errors""" + # Check no raw calls passing in DictionayType + with pytest.raises(AssertionError): + DictionaryType.construct_type( + DictionaryType, "MyNewString", PROPERTY="one", PROPERTY2="two" + ) + + # Check consistent field definitions: field values + DictionaryType.construct_type(str, "MyNewString1", PROPERTY="one", PROPERTY2="two") + with pytest.raises(AssertionError): + DictionaryType.construct_type( + str, "MyNewString1", PROPERTY="three", PROPERTY2="four" + ) + + # Check consistent field definitions: field names + DictionaryType.construct_type(str, "MyNewString2", PROPERTY1="one", PROPERTY2="two") + with pytest.raises(AssertionError): + DictionaryType.construct_type( + str, "MyNewString2", PROPERTY="one", PROPERTY3="two" + ) + + # Check consistent field definitions: missing field + DictionaryType.construct_type(str, "MyNewString3", PROPERTY1="one", PROPERTY2="two") + with pytest.raises(AssertionError): + DictionaryType.construct_type(str, "MyNewString3", PROPERTY1="one") From 822149a3263a272ce3811164477ce55c5c26fd73 Mon Sep 17 00:00:00 2001 From: Michael D Starch Date: Tue, 10 May 2022 18:35:57 -0700 Subject: [PATCH 13/15] lestarch: sp --- test/fprime/common/models/serialize/test_types.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/fprime/common/models/serialize/test_types.py b/test/fprime/common/models/serialize/test_types.py index ab182606..73a65e3d 100644 --- a/test/fprime/common/models/serialize/test_types.py +++ b/test/fprime/common/models/serialize/test_types.py @@ -289,7 +289,7 @@ def test_serializable_basic(): ] valid_values = [ ({"member1": 123, "member2": 456, "member3": -234}, 4 + 4 + 8), - ({"member4": "345", "member5": "abcd", "member6": 213}, 5 + 6 + 8), + ({"member4": "345", "member5": "abc1", "member6": 213}, 5 + 6 + 8), ] for index, (members, (valid, size)) in enumerate(zip(member_list, valid_values)): @@ -339,7 +339,7 @@ def test_serializable_advanced(): array_member_class = ArrayType.construct_type( "ArrayMember", string_member_class, 3, "%s" ) - subserializable_class = SerializableType.construct_type( + sub_serializable_class = SerializableType.construct_type( "AdvancedSubSerializable", [("subfield1", U32Type), ("subfield2", array_member_class)], ) @@ -349,7 +349,7 @@ def test_serializable_advanced(): ("field2", U32Type), ("field3", enum_member_class), ("field4", array_member_class), - ("field5", subserializable_class), + ("field5", sub_serializable_class), ] serializable_class = SerializableType.construct_type( "AdvancedSerializable", field_data @@ -480,7 +480,7 @@ def test_base_type(): def test_dictionary_type_errors(): """Ensure the dictionary type is preventing errors""" - # Check no raw calls passing in DictionayType + # Check no raw calls passing in DictionaryType with pytest.raises(AssertionError): DictionaryType.construct_type( DictionaryType, "MyNewString", PROPERTY="one", PROPERTY2="two" From d5736e5b7f4a13d6c13b62c2d54c777d8d00acd3 Mon Sep 17 00:00:00 2001 From: Michael D Starch Date: Thu, 12 May 2022 10:33:08 -0700 Subject: [PATCH 14/15] lestarch: inconsistency after deserialization --- src/fprime/common/models/serialize/array_type.py | 4 ++-- test/fprime/common/models/serialize/test_types.py | 5 ++++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/src/fprime/common/models/serialize/array_type.py b/src/fprime/common/models/serialize/array_type.py index 770aa44d..901c41ef 100644 --- a/src/fprime/common/models/serialize/array_type.py +++ b/src/fprime/common/models/serialize/array_type.py @@ -116,8 +116,8 @@ def deserialize(self, data, offset): item = self.MEMBER_TYPE() item.deserialize(data, offset) offset += item.getSize() - values.append(item.val) - self.val = values + values.append(item) + self._val = values def getSize(self): """Return the size of the array""" diff --git a/test/fprime/common/models/serialize/test_types.py b/test/fprime/common/models/serialize/test_types.py index 73a65e3d..166e111d 100644 --- a/test/fprime/common/models/serialize/test_types.py +++ b/test/fprime/common/models/serialize/test_types.py @@ -101,7 +101,10 @@ def valid_values_test(type_input, valid_values, sizes): deserializer.deserialize((b" " * offset) + serialized, offset) assert instantiation.val == deserializer.val, "Deserialization has failed" assert deserializer.getSize() == size - + # Check another get/set pair and serialization of the post-deserialized object + deserializer.val = deserializer.val + new_serialized_bytes = deserializer.serialize() + assert serialized == new_serialized_bytes, "Repeated serialization has failed" def invalid_values_test( type_input, invalid_values, exception_class=TypeMismatchException From 593a48350cd302b8caa6c7aa1711cacb792148bb Mon Sep 17 00:00:00 2001 From: Michael D Starch Date: Thu, 12 May 2022 11:01:28 -0700 Subject: [PATCH 15/15] lestarch: formatting --- test/fprime/common/models/serialize/test_types.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/test/fprime/common/models/serialize/test_types.py b/test/fprime/common/models/serialize/test_types.py index 166e111d..8c24ef53 100644 --- a/test/fprime/common/models/serialize/test_types.py +++ b/test/fprime/common/models/serialize/test_types.py @@ -104,7 +104,10 @@ def valid_values_test(type_input, valid_values, sizes): # Check another get/set pair and serialization of the post-deserialized object deserializer.val = deserializer.val new_serialized_bytes = deserializer.serialize() - assert serialized == new_serialized_bytes, "Repeated serialization has failed" + assert ( + serialized == new_serialized_bytes + ), "Repeated serialization has failed" + def invalid_values_test( type_input, invalid_values, exception_class=TypeMismatchException