From 5d99780ba05b2580e11a0b10911a5ddbf43da57b Mon Sep 17 00:00:00 2001 From: HENDRIX-ZT2 Date: Wed, 18 Sep 2019 21:38:34 +0200 Subject: [PATCH 01/22] Adapt to new nif.xml Trying to get it not crash. Not functional, but at least it parses the whole xml. --- pyffi/formats/nif/__init__.py | 9 ++++ pyffi/object_models/common.py | 32 +++++++++++++ pyffi/object_models/xml/__init__.py | 74 +++++++++++++++++++++++++++-- 3 files changed, 110 insertions(+), 5 deletions(-) diff --git a/pyffi/formats/nif/__init__.py b/pyffi/formats/nif/__init__.py index 849336c1e..4d8999587 100644 --- a/pyffi/formats/nif/__init__.py +++ b/pyffi/formats/nif/__init__.py @@ -400,6 +400,8 @@ class NifFormat(FileFormat): EPSILON = 0.0001 # basic types + uint64 = pyffi.object_models.common.UInt64 + int64 = pyffi.object_models.common.Int64 ulittle32 = pyffi.object_models.common.ULittle32 int = pyffi.object_models.common.Int uint = pyffi.object_models.common.UInt @@ -407,6 +409,7 @@ class NifFormat(FileFormat): char = pyffi.object_models.common.Char short = pyffi.object_models.common.Short ushort = pyffi.object_models.common.UShort + hfloat = pyffi.object_models.common.HFloat float = pyffi.object_models.common.Float BlockTypeIndex = pyffi.object_models.common.UShort StringIndex = pyffi.object_models.common.UInt @@ -420,6 +423,12 @@ def __init__(self, **kwargs): pyffi.object_models.common.Int.__init__(self, **kwargs) self.set_value(-1) + class NiFixedString(pyffi.object_models.common.Int): + """This is just an integer with -1 as default value.""" + def __init__(self, **kwargs): + pyffi.object_models.common.Int.__init__(self, **kwargs) + self.set_value(-1) + class bool(BasicBase, EditableBoolComboBox): """Basic implementation of a 32-bit (8-bit for versions > 4.0.0.2) boolean type. diff --git a/pyffi/object_models/common.py b/pyffi/object_models/common.py index 9757a4072..ac903ff16 100644 --- a/pyffi/object_models/common.py +++ b/pyffi/object_models/common.py @@ -414,6 +414,38 @@ def get_hash(self, data=None): """ return int(self.get_value()*200) +class HFloat(Float): + def read(self, stream, data): + """Read value from stream. + + :param stream: The stream to read from. + :type stream: file + """ + self._value = struct.unpack(data._byte_order + 'fe', + stream.read(2))[0] + + def write(self, stream, data): + """Write value to stream. + + :param stream: The stream to write to. + :type stream: file + """ + try: + stream.write(struct.pack(data._byte_order + 'e', + self._value)) + except OverflowError: + logger = logging.getLogger("pyffi.object_models") + logger.warn("float value overflow, writing zero") + stream.write(struct.pack(data._byte_order + 'e', + 0)) + + def get_size(self, data=None): + """Return number of bytes this type occupies in a file. + + :return: Number of bytes. + """ + return 2 + class ZString(BasicBase, EditableLineEdit): """String of variable length (null terminated). diff --git a/pyffi/object_models/xml/__init__.py b/pyffi/object_models/xml/__init__.py index bd8fac875..53aefa767 100644 --- a/pyffi/object_models/xml/__init__.py +++ b/pyffi/object_models/xml/__init__.py @@ -308,6 +308,18 @@ class XmlSaxHandler(xml.sax.handler.ContentHandler): tag_struct = 8 tag_attribute = 9 tag_bits = 10 + # new stuff + tag_module = 11 + tag_token = 12 + tag_verexpr = 13 + tag_condexpr = 14 + tag_verset = 15 + tag_default = 16 + tag_range = 17 + tag_global = 18 + tag_operator = 18 + tag_bitfield = 19 + tag_member = 20 tags = { "fileformat": tag_file, @@ -319,7 +331,22 @@ class XmlSaxHandler(xml.sax.handler.ContentHandler): "bitstruct": tag_bit_struct, "struct": tag_struct, "bits": tag_bits, - "add": tag_attribute} + "add": tag_attribute, + #new stuff + "module": tag_module, + "token": tag_token, + "verexpr": tag_verexpr, + "condexpr": tag_condexpr, + "verset": tag_verset, + "default": tag_default, + "range": tag_range, + "global": tag_global, + "operator": tag_operator, + "bitfield": tag_bitfield, + "member": tag_member, + # seems to behave like option used to do + "field": tag_option + } # for compatibility with niftools tags_niftools = { @@ -420,7 +447,8 @@ def startElement(self, name, attrs): try: tag = self.tags_niftools[name] except KeyError: - raise XmlError("error unknown element '%s'" % name) + # raise XmlError("error unknown element '%s'" % name) + print("error unknown element '%s'" % name) # Check the stack, if the stack does not exist then we must be # at the root of the xml file, and the tag must be "fileformat". @@ -462,7 +490,9 @@ def startElement(self, name, attrs): self.version_string) # (class_dict["_games"] is updated when reading the characters) else: - raise XmlError( + # raise XmlError( + # "only add and version tags allowed in struct declaration") + print( "only add and version tags allowed in struct declaration") elif self.current_tag == self.tag_file: self.pushTag(tag) @@ -506,7 +536,10 @@ def startElement(self, name, attrs): # check the class variables is_template = (attrs.get("istemplate") == "1") if self.basic_class._is_template != is_template: - raise XmlError( + # raise XmlError( + # 'class %s should have _is_template = %s' + # % (self.class_name, is_template)) + print( 'class %s should have _is_template = %s' % (self.class_name, is_template)) @@ -565,6 +598,21 @@ def startElement(self, name, attrs): self.cls.versions[self.version_string] = self.cls.version_number( self.version_string) # (self.cls.games is updated when reading the characters) + # fileformat -> module + elif tag == self.tag_module: + self.module_name = str(attrs["name"]) + self.module_priority = str(attrs["priority"]) + if "depends" in attrs: + self.module_depends = str(attrs["depends"]) + # fileformat -> token + elif tag == self.tag_token: + self.token_name = str(attrs["name"]) + self.token_attrs = str(attrs["attrs"]) + # fileformat -> bitfield + elif tag == self.tag_bitfield: + self.bitfield_name = str(attrs["name"]) + self.bitfield_storage = str(attrs["storage"]) + # self.bitfield_versions = str(attrs["versions"]) else: raise XmlError(""" @@ -591,6 +639,18 @@ def startElement(self, name, attrs): self.class_dict["_enumkeys"].append(attrs["name"]) self.class_dict["_enumvalues"].append(value) + elif self.current_tag == self.tag_token: + self.pushTag(tag) + elif self.current_tag == self.tag_bitfield: + self.pushTag(tag) + elif self.current_tag == self.tag_option: + self.pushTag(tag) + # verexpr_token = attrs["token"] + # verexpr_string = attrs["string"] + # self.class_dict["_enumkeys"].append(attrs["name"]) + # self.class_dict["_enumvalues"].append(value) + + elif self.current_tag == self.tag_bit_struct: self.pushTag(tag) if tag == self.tag_bits: @@ -603,6 +663,9 @@ def startElement(self, name, attrs): # first, calculate current bit position bitpos = sum(bitattr.numbits for bitattr in self.class_dict["_attrs"]) + # avoid crash + if "value" not in attrs: + return # check if extra bits must be inserted numextrabits = int(attrs["value"]) - bitpos if numextrabits < 0: @@ -624,7 +687,8 @@ def startElement(self, name, attrs): "only bits tags allowed in struct type declaration") else: - raise XmlError("unhandled tag %s" % name) + # raise XmlError("unhandled tag %s" % name) + print("unhandled tag %s" % name) def endElement(self, name): """Called at the end of each xml tag. From c37301082ba7af1100ec27b37b590b13cbf738f4 Mon Sep 17 00:00:00 2001 From: HENDRIX-ZT2 Date: Wed, 18 Sep 2019 23:33:25 +0200 Subject: [PATCH 02/22] Fix expressions expressions seem to decode now with tokens from the nif xml --- pyffi/object_models/xml/__init__.py | 54 ++++++++++++++++----------- pyffi/object_models/xml/expression.py | 11 +++++- 2 files changed, 43 insertions(+), 22 deletions(-) diff --git a/pyffi/object_models/xml/__init__.py b/pyffi/object_models/xml/__init__.py index 53aefa767..cc0164e1f 100644 --- a/pyffi/object_models/xml/__init__.py +++ b/pyffi/object_models/xml/__init__.py @@ -189,12 +189,13 @@ class StructAttribute(object): is_abstract = False """Whether the attribute is abstract or not (read and written).""" - def __init__(self, cls, attrs): + def __init__(self, cls, attrs, operators): """Initialize attribute from the xml attrs dictionary of an add tag. :param cls: The class where all types reside. - :param attrs: The xml add tag attribute dictionary.""" + :param attrs: The xml add tag attribute dictionary. + :param operators: A dict containing operator info for Expressions class.""" # mandatory parameters self.displayname = attrs["name"] self.name = cls.name_attribute(self.displayname) @@ -236,13 +237,13 @@ def __init__(self, cls, attrs): # conversion failed; not a big problem self.default = None if self.arr1: - self.arr1 = Expression(self.arr1, cls.name_attribute) + self.arr1 = Expression(self.arr1, cls.name_attribute, operators) if self.arr2: - self.arr2 = Expression(self.arr2, cls.name_attribute) + self.arr2 = Expression(self.arr2, cls.name_attribute, operators) if self.cond: - self.cond = Expression(self.cond, cls.name_attribute) + self.cond = Expression(self.cond, cls.name_attribute, operators) if self.vercond: - self.vercond = Expression(self.vercond, cls.name_attribute) + self.vercond = Expression(self.vercond, cls.name_attribute, operators) if self.arg: try: self.arg = int(self.arg) @@ -259,7 +260,7 @@ def __init__(self, cls, attrs): class BitStructAttribute(object): """Helper class to collect attribute data of bitstruct bits tags.""" - def __init__(self, cls, attrs): + def __init__(self, cls, attrs, operators): """Initialize attribute from the xml attrs dictionary of an add tag. @@ -280,7 +281,7 @@ def __init__(self, cls, attrs): if self.default: self.default = int(self.default) if self.cond: - self.cond = Expression(self.cond, cls.name_attribute) + self.cond = Expression(self.cond, cls.name_attribute, operators) if self.userver: self.userver = int(self.userver) if self.ver1: @@ -328,7 +329,9 @@ class XmlSaxHandler(xml.sax.handler.ContentHandler): "alias": tag_alias, "enum": tag_enum, "option": tag_option, + # no longer in nif xml "bitstruct": tag_bit_struct, + # no longer in nif xml "struct": tag_struct, "bits": tag_bits, "add": tag_attribute, @@ -344,8 +347,8 @@ class XmlSaxHandler(xml.sax.handler.ContentHandler): "operator": tag_operator, "bitfield": tag_bitfield, "member": tag_member, - # seems to behave like option used to do - "field": tag_option + # seems to behave like option used to do OR rather add? + "field": tag_attribute } # for compatibility with niftools @@ -403,6 +406,8 @@ def __init__(self, cls, name, bases, dct): # elements for versions self.version_string = None + self.operators = {} + def pushTag(self, tag): """Push tag C{tag} on the stack and make it the current tag. @@ -447,8 +452,7 @@ def startElement(self, name, attrs): try: tag = self.tags_niftools[name] except KeyError: - # raise XmlError("error unknown element '%s'" % name) - print("error unknown element '%s'" % name) + raise XmlError("error unknown element '%s'" % name) # Check the stack, if the stack does not exist then we must be # at the root of the xml file, and the tag must be "fileformat". @@ -477,11 +481,12 @@ def startElement(self, name, attrs): # string. if self.current_tag == self.tag_struct: self.pushTag(tag) + print(attrs["name"]) # struct -> attribute if tag == self.tag_attribute: # add attribute to class dictionary self.class_dict["_attrs"].append( - StructAttribute(self.cls, attrs)) + StructAttribute(self.cls, attrs, self.operators)) # struct -> version elif tag == self.tag_version: # set the version string @@ -490,8 +495,6 @@ def startElement(self, name, attrs): self.version_string) # (class_dict["_games"] is updated when reading the characters) else: - # raise XmlError( - # "only add and version tags allowed in struct declaration") print( "only add and version tags allowed in struct declaration") elif self.current_tag == self.tag_file: @@ -615,9 +618,8 @@ def startElement(self, name, attrs): # self.bitfield_versions = str(attrs["versions"]) else: - raise XmlError(""" -expected basic, alias, enum, bitstruct, struct, or version, -but got %s instead""" % name) + raise XmlError(""" expected basic, alias, enum, bitstruct, struct, or version, + but got %s instead""" % name) elif self.current_tag == self.tag_version: raise XmlError("version tag must not contain any sub tags") @@ -641,10 +643,20 @@ def startElement(self, name, attrs): elif self.current_tag == self.tag_token: self.pushTag(tag) + if tag == self.tag_operator: + # this is the symbol that represents the op in nif.xml, eg "#EQ#" + op_token = attrs["token"] + op_string = attrs["string"] + self.operators[op_token] = op_string + elif self.current_tag == self.tag_bitfield: self.pushTag(tag) elif self.current_tag == self.tag_option: self.pushTag(tag) + elif self.current_tag == self.tag_attribute: + self.pushTag(tag) + if tag == self.tag_default: + print("Default") # verexpr_token = attrs["token"] # verexpr_string = attrs["string"] # self.class_dict["_enumkeys"].append(attrs["name"]) @@ -656,7 +668,7 @@ def startElement(self, name, attrs): if tag == self.tag_bits: # mandatory parameters self.class_dict["_attrs"].append( - BitStructAttribute(self.cls, attrs)) + BitStructAttribute(self.cls, attrs, self.operators)) elif tag == self.tag_option: # niftools compatibility, we have a bitflags field # so convert value into numbits @@ -676,12 +688,12 @@ def startElement(self, name, attrs): self.cls, dict(name="Reserved Bits %i" % len(self.class_dict["_attrs"]), - numbits=numextrabits))) + numbits=numextrabits), self.operators)) # add the actual attribute self.class_dict["_attrs"].append( BitStructAttribute( self.cls, - dict(name=attrs["name"], numbits=1))) + dict(name=attrs["name"], numbits=1), self.operators)) else: raise XmlError( "only bits tags allowed in struct type declaration") diff --git a/pyffi/object_models/xml/expression.py b/pyffi/object_models/xml/expression.py index 6813ae564..b4efc862e 100644 --- a/pyffi/object_models/xml/expression.py +++ b/pyffi/object_models/xml/expression.py @@ -78,7 +78,9 @@ class Expression(object): operators = set(('==', '!=', '>=', '<=', '&&', '||', '&', '|', '-', '!', '<', '>', '/', '*', '+')) - def __init__(self, expr_str, name_filter=None): + def __init__(self, expr_str, name_filter=None, operators_dict=None): + if operators_dict: + expr_str = self.replace_op_dict(expr_str, operators_dict) try: left, self._op, right = self._partition(expr_str) self._left = self._parse(left, name_filter) @@ -87,6 +89,13 @@ def __init__(self, expr_str, name_filter=None): print("error while parsing expression '%s'" % expr_str) raise + def replace_op_dict(self, expr_str, operators_dict): + """Update string with content of operator dict.""" + for op_token, op_str in operators_dict.items(): + if op_token in expr_str: + expr_str = expr_str.replace(op_token, op_str) + return expr_str + def eval(self, data=None): """Evaluate the expression to an integer.""" From 182793d8db995b0f5b466c1e05a6e366150e698c Mon Sep 17 00:00:00 2001 From: HENDRIX-ZT2 Date: Thu, 19 Sep 2019 00:43:59 +0200 Subject: [PATCH 03/22] parse bitfields bitfields are parsed as now bit structs, but attribs are not set. --- pyffi/object_models/xml/__init__.py | 33 +++++++++++++++++++-------- pyffi/object_models/xml/bit_struct.py | 4 +++- 2 files changed, 27 insertions(+), 10 deletions(-) diff --git a/pyffi/object_models/xml/__init__.py b/pyffi/object_models/xml/__init__.py index cc0164e1f..1c7e30056 100644 --- a/pyffi/object_models/xml/__init__.py +++ b/pyffi/object_models/xml/__init__.py @@ -319,7 +319,7 @@ class XmlSaxHandler(xml.sax.handler.ContentHandler): tag_range = 17 tag_global = 18 tag_operator = 18 - tag_bitfield = 19 + # tag_bitfield = 19 tag_member = 20 tags = { @@ -345,7 +345,7 @@ class XmlSaxHandler(xml.sax.handler.ContentHandler): "range": tag_range, "global": tag_global, "operator": tag_operator, - "bitfield": tag_bitfield, + "bitfield": tag_bit_struct, "member": tag_member, # seems to behave like option used to do OR rather add? "field": tag_attribute @@ -586,6 +586,7 @@ def startElement(self, name, attrs): elif tag == self.tag_bit_struct: self.class_bases += (BitStructBase,) self.class_name = attrs["name"] + print("BITSTRUCT:",self.class_name) try: numbytes = int(attrs["numbytes"]) except KeyError: @@ -594,6 +595,11 @@ def startElement(self, name, attrs): self.class_dict = {"_attrs": [], "__doc__": "", "_numbytes": numbytes, "__module__": self.cls.__module__} + # # fileformat -> bitfield + # elif tag == self.tag_bitfield: + # self.bitfield_name = str(attrs["name"]) + # self.bitfield_storage = str(attrs["storage"]) + # # self.bitfield_versions = str(attrs["versions"]) # fileformat -> version elif tag == self.tag_version: @@ -611,11 +617,6 @@ def startElement(self, name, attrs): elif tag == self.tag_token: self.token_name = str(attrs["name"]) self.token_attrs = str(attrs["attrs"]) - # fileformat -> bitfield - elif tag == self.tag_bitfield: - self.bitfield_name = str(attrs["name"]) - self.bitfield_storage = str(attrs["storage"]) - # self.bitfield_versions = str(attrs["versions"]) else: raise XmlError(""" expected basic, alias, enum, bitstruct, struct, or version, @@ -641,6 +642,15 @@ def startElement(self, name, attrs): self.class_dict["_enumkeys"].append(attrs["name"]) self.class_dict["_enumvalues"].append(value) + # elif self.current_tag == self.tag_bitfield: + # self.pushTag(tag) + # if not tag == self.tag_member: + # raise XmlError("only member tags allowed in bitfield declaration") + # # member width="3" pos="12" mask="0xF000" name="Test Func" type="StencilTestFunc" default="TEST_GREATER + # self.class_dict["width"] = attrs["width"] + # self.class_dict["pos"] = attrs["pos"] + # self.class_dict["name"].append(attrs["name"]) + elif self.current_tag == self.tag_token: self.pushTag(tag) if tag == self.tag_operator: @@ -649,8 +659,6 @@ def startElement(self, name, attrs): op_string = attrs["string"] self.operators[op_token] = op_string - elif self.current_tag == self.tag_bitfield: - self.pushTag(tag) elif self.current_tag == self.tag_option: self.pushTag(tag) elif self.current_tag == self.tag_attribute: @@ -666,6 +674,7 @@ def startElement(self, name, attrs): elif self.current_tag == self.tag_bit_struct: self.pushTag(tag) if tag == self.tag_bits: + # eg. # mandatory parameters self.class_dict["_attrs"].append( BitStructAttribute(self.cls, attrs, self.operators)) @@ -694,6 +703,12 @@ def startElement(self, name, attrs): BitStructAttribute( self.cls, dict(name=attrs["name"], numbits=1), self.operators)) + # new nif xml + elif tag == self.tag_member: + + self.class_dict["_attrs"].append( + BitStructAttribute(self.cls, + dict(name=attrs["name"], numbits=1), self.operators)) else: raise XmlError( "only bits tags allowed in struct type declaration") diff --git a/pyffi/object_models/xml/bit_struct.py b/pyffi/object_models/xml/bit_struct.py index 3f39d3244..440991e8f 100644 --- a/pyffi/object_models/xml/bit_struct.py +++ b/pyffi/object_models/xml/bit_struct.py @@ -68,8 +68,10 @@ def __init__(cls, name, bases, dct): cls._struct = 'H' elif cls._numbytes == 4: cls._struct = 'I' + elif cls._numbytes == 8: + cls._struct = 'II' else: - raise RuntimeError("unsupported bitstruct numbytes") + raise RuntimeError("unsupported bitstruct number of bytes: "+str(cls._numbytes)) # template type? cls._is_template = False From 2ca9bbdf6def37169d3a5a9ac24360bdd5289cf8 Mon Sep 17 00:00:00 2001 From: HENDRIX-ZT2 Date: Thu, 19 Sep 2019 11:10:54 +0200 Subject: [PATCH 04/22] Parse tokens properly Turns them into a list of dicts which are then applied in the expression class --- pyffi/object_models/xml/__init__.py | 27 ++++++++++++++------------- pyffi/object_models/xml/expression.py | 26 +++++++++++++++++--------- 2 files changed, 31 insertions(+), 22 deletions(-) diff --git a/pyffi/object_models/xml/__init__.py b/pyffi/object_models/xml/__init__.py index 1c7e30056..90d22dd82 100644 --- a/pyffi/object_models/xml/__init__.py +++ b/pyffi/object_models/xml/__init__.py @@ -406,7 +406,8 @@ def __init__(self, cls, name, bases, dct): # elements for versions self.version_string = None - self.operators = {} + # a list of dicts that will store each token + self.tokens = [] def pushTag(self, tag): """Push tag C{tag} on the stack and make it the current tag. @@ -486,7 +487,7 @@ def startElement(self, name, attrs): if tag == self.tag_attribute: # add attribute to class dictionary self.class_dict["_attrs"].append( - StructAttribute(self.cls, attrs, self.operators)) + StructAttribute(self.cls, attrs, self.tokens)) # struct -> version elif tag == self.tag_version: # set the version string @@ -615,8 +616,8 @@ def startElement(self, name, attrs): self.module_depends = str(attrs["depends"]) # fileformat -> token elif tag == self.tag_token: - self.token_name = str(attrs["name"]) - self.token_attrs = str(attrs["attrs"]) + # create a new dict for this token to which we add token - str pairs for replacement + self.tokens.append( {} ) else: raise XmlError(""" expected basic, alias, enum, bitstruct, struct, or version, @@ -653,11 +654,11 @@ def startElement(self, name, attrs): elif self.current_tag == self.tag_token: self.pushTag(tag) - if tag == self.tag_operator: - # this is the symbol that represents the op in nif.xml, eg "#EQ#" - op_token = attrs["token"] - op_string = attrs["string"] - self.operators[op_token] = op_string + # this is the symbol that represents the operation in nif.xml, eg "#EQ#" + op_token = attrs["token"] + op_string = attrs["string"] + # store it + self.tokens[-1][op_token] = op_string elif self.current_tag == self.tag_option: self.pushTag(tag) @@ -677,7 +678,7 @@ def startElement(self, name, attrs): # eg. # mandatory parameters self.class_dict["_attrs"].append( - BitStructAttribute(self.cls, attrs, self.operators)) + BitStructAttribute(self.cls, attrs, self.tokens)) elif tag == self.tag_option: # niftools compatibility, we have a bitflags field # so convert value into numbits @@ -697,18 +698,18 @@ def startElement(self, name, attrs): self.cls, dict(name="Reserved Bits %i" % len(self.class_dict["_attrs"]), - numbits=numextrabits), self.operators)) + numbits=numextrabits), self.tokens)) # add the actual attribute self.class_dict["_attrs"].append( BitStructAttribute( self.cls, - dict(name=attrs["name"], numbits=1), self.operators)) + dict(name=attrs["name"], numbits=1), self.tokens)) # new nif xml elif tag == self.tag_member: self.class_dict["_attrs"].append( BitStructAttribute(self.cls, - dict(name=attrs["name"], numbits=1), self.operators)) + dict(name=attrs["name"], numbits=1), self.tokens)) else: raise XmlError( "only bits tags allowed in struct type declaration") diff --git a/pyffi/object_models/xml/expression.py b/pyffi/object_models/xml/expression.py index b4efc862e..5e918fce5 100644 --- a/pyffi/object_models/xml/expression.py +++ b/pyffi/object_models/xml/expression.py @@ -78,9 +78,12 @@ class Expression(object): operators = set(('==', '!=', '>=', '<=', '&&', '||', '&', '|', '-', '!', '<', '>', '/', '*', '+')) - def __init__(self, expr_str, name_filter=None, operators_dict=None): - if operators_dict: - expr_str = self.replace_op_dict(expr_str, operators_dict) + def __init__(self, expr_str, name_filter=None, replacers=None): + self._expr_str = expr_str + self.expr_str = expr_str + if replacers: + expr_str = self.replace_op_dict(expr_str, replacers) + self.expr_str = self.replace_op_dict(expr_str, replacers) try: left, self._op, right = self._partition(expr_str) self._left = self._parse(left, name_filter) @@ -89,16 +92,21 @@ def __init__(self, expr_str, name_filter=None, operators_dict=None): print("error while parsing expression '%s'" % expr_str) raise - def replace_op_dict(self, expr_str, operators_dict): - """Update string with content of operator dict.""" - for op_token, op_str in operators_dict.items(): - if op_token in expr_str: - expr_str = expr_str.replace(op_token, op_str) + def replace_op_dict(self, expr_str, replacers): + """Update string with content of replacers list of dicts.""" + for token in replacers: + for op_token, op_str in token.items(): + if op_token in expr_str: + expr_str = expr_str.replace(op_token, op_str) + # replace 'member of' operator + expr_str = expr_str.replace("\\", ".") return expr_str def eval(self, data=None): """Evaluate the expression to an integer.""" - + print(self._expr_str) + print(self.expr_str) + print(self._left, self._op, self._right) if isinstance(self._left, Expression): left = self._left.eval(data) elif isinstance(self._left, str): From d914cb53f0d344cd5c6cc62089392e05813d8661 Mon Sep 17 00:00:00 2001 From: HENDRIX-ZT2 Date: Thu, 19 Sep 2019 12:39:48 +0200 Subject: [PATCH 05/22] Apply tokens to attrs Tokens are now applied to matching attributes from a structs attrs dict --- pyffi/object_models/xml/__init__.py | 83 ++++++++++++++------------- pyffi/object_models/xml/expression.py | 21 +------ 2 files changed, 45 insertions(+), 59 deletions(-) diff --git a/pyffi/object_models/xml/__init__.py b/pyffi/object_models/xml/__init__.py index 90d22dd82..c9f0eec21 100644 --- a/pyffi/object_models/xml/__init__.py +++ b/pyffi/object_models/xml/__init__.py @@ -189,13 +189,12 @@ class StructAttribute(object): is_abstract = False """Whether the attribute is abstract or not (read and written).""" - def __init__(self, cls, attrs, operators): + def __init__(self, cls, attrs): """Initialize attribute from the xml attrs dictionary of an add tag. :param cls: The class where all types reside. - :param attrs: The xml add tag attribute dictionary. - :param operators: A dict containing operator info for Expressions class.""" + :param attrs: The xml add tag attribute dictionary.""" # mandatory parameters self.displayname = attrs["name"] self.name = cls.name_attribute(self.displayname) @@ -237,13 +236,13 @@ def __init__(self, cls, attrs, operators): # conversion failed; not a big problem self.default = None if self.arr1: - self.arr1 = Expression(self.arr1, cls.name_attribute, operators) + self.arr1 = Expression(self.arr1, cls.name_attribute) if self.arr2: - self.arr2 = Expression(self.arr2, cls.name_attribute, operators) + self.arr2 = Expression(self.arr2, cls.name_attribute) if self.cond: - self.cond = Expression(self.cond, cls.name_attribute, operators) + self.cond = Expression(self.cond, cls.name_attribute) if self.vercond: - self.vercond = Expression(self.vercond, cls.name_attribute, operators) + self.vercond = Expression(self.vercond, cls.name_attribute) if self.arg: try: self.arg = int(self.arg) @@ -256,11 +255,10 @@ def __init__(self, cls, attrs, operators): if self.ver2: self.ver2 = cls.version_number(self.ver2) - class BitStructAttribute(object): """Helper class to collect attribute data of bitstruct bits tags.""" - def __init__(self, cls, attrs, operators): + def __init__(self, cls, attrs): """Initialize attribute from the xml attrs dictionary of an add tag. @@ -281,7 +279,7 @@ def __init__(self, cls, attrs, operators): if self.default: self.default = int(self.default) if self.cond: - self.cond = Expression(self.cond, cls.name_attribute, operators) + self.cond = Expression(self.cond, cls.name_attribute) if self.userver: self.userver = int(self.userver) if self.ver1: @@ -312,13 +310,8 @@ class XmlSaxHandler(xml.sax.handler.ContentHandler): # new stuff tag_module = 11 tag_token = 12 - tag_verexpr = 13 - tag_condexpr = 14 - tag_verset = 15 - tag_default = 16 - tag_range = 17 - tag_global = 18 - tag_operator = 18 + # we don't care about the type of token children + tag_subtoken = 18 # tag_bitfield = 19 tag_member = 20 @@ -338,13 +331,13 @@ class XmlSaxHandler(xml.sax.handler.ContentHandler): #new stuff "module": tag_module, "token": tag_token, - "verexpr": tag_verexpr, - "condexpr": tag_condexpr, - "verset": tag_verset, - "default": tag_default, - "range": tag_range, - "global": tag_global, - "operator": tag_operator, + "verexpr": tag_subtoken, + "condexpr": tag_subtoken, + "verset": tag_subtoken, + "default": tag_subtoken, + "range": tag_subtoken, + "global": tag_subtoken, + "operator": tag_subtoken, "bitfield": tag_bit_struct, "member": tag_member, # seems to behave like option used to do OR rather add? @@ -482,12 +475,13 @@ def startElement(self, name, attrs): # string. if self.current_tag == self.tag_struct: self.pushTag(tag) - print(attrs["name"]) + attrs = self.replace_tokens(attrs, self.tokens) + # print(attrs["name"]) # struct -> attribute if tag == self.tag_attribute: # add attribute to class dictionary self.class_dict["_attrs"].append( - StructAttribute(self.cls, attrs, self.tokens)) + StructAttribute(self.cls, attrs)) # struct -> version elif tag == self.tag_version: # set the version string @@ -617,7 +611,9 @@ def startElement(self, name, attrs): # fileformat -> token elif tag == self.tag_token: # create a new dict for this token to which we add token - str pairs for replacement - self.tokens.append( {} ) + # also the target attributes that these tokens should be applied to + # store both as a tuple + self.tokens.append( ( {}, attrs["attrs"].split(" ") ) ) else: raise XmlError(""" expected basic, alias, enum, bitstruct, struct, or version, @@ -658,27 +654,20 @@ def startElement(self, name, attrs): op_token = attrs["token"] op_string = attrs["string"] # store it - self.tokens[-1][op_token] = op_string + self.tokens[-1][0][op_token] = op_string elif self.current_tag == self.tag_option: self.pushTag(tag) elif self.current_tag == self.tag_attribute: self.pushTag(tag) - if tag == self.tag_default: - print("Default") - # verexpr_token = attrs["token"] - # verexpr_string = attrs["string"] - # self.class_dict["_enumkeys"].append(attrs["name"]) - # self.class_dict["_enumvalues"].append(value) - - elif self.current_tag == self.tag_bit_struct: self.pushTag(tag) + attrs = self.replace_tokens(attrs, self.tokens) if tag == self.tag_bits: # eg. # mandatory parameters self.class_dict["_attrs"].append( - BitStructAttribute(self.cls, attrs, self.tokens)) + BitStructAttribute(self.cls, attrs)) elif tag == self.tag_option: # niftools compatibility, we have a bitflags field # so convert value into numbits @@ -698,18 +687,18 @@ def startElement(self, name, attrs): self.cls, dict(name="Reserved Bits %i" % len(self.class_dict["_attrs"]), - numbits=numextrabits), self.tokens)) + numbits=numextrabits))) # add the actual attribute self.class_dict["_attrs"].append( BitStructAttribute( self.cls, - dict(name=attrs["name"], numbits=1), self.tokens)) + dict(name=attrs["name"], numbits=1))) # new nif xml elif tag == self.tag_member: self.class_dict["_attrs"].append( BitStructAttribute(self.cls, - dict(name=attrs["name"], numbits=1), self.tokens)) + dict(name=attrs["name"], numbits=1))) else: raise XmlError( "only bits tags allowed in struct type declaration") @@ -828,6 +817,20 @@ def endDocument(self): attr.cond.map_( lambda x: klass_filter[x] if x in klass_filter else x) + def replace_tokens(self, attr_dict, replacers): + """Update attr_dict with content of replacers list of dicts.""" + attr_dict = dict(attr_dict) + # replacers is a list of tuples ({tokens}, (target_attribs)) for each + for tokens, target_attribs in replacers: + for target_attrib in target_attribs: + if target_attrib in attr_dict: + expr_str = attr_dict[target_attrib] + for op_token, op_str in tokens.items(): + if op_token in expr_str: + expr_str = expr_str.replace(op_token, op_str) + # replace 'member of' operator & update + attr_dict[target_attrib] = expr_str.replace("\\", ".") + return attr_dict def characters(self, chars): """Add the string C{chars} to the docstring. diff --git a/pyffi/object_models/xml/expression.py b/pyffi/object_models/xml/expression.py index 5e918fce5..cdb012abb 100644 --- a/pyffi/object_models/xml/expression.py +++ b/pyffi/object_models/xml/expression.py @@ -78,12 +78,8 @@ class Expression(object): operators = set(('==', '!=', '>=', '<=', '&&', '||', '&', '|', '-', '!', '<', '>', '/', '*', '+')) - def __init__(self, expr_str, name_filter=None, replacers=None): - self._expr_str = expr_str - self.expr_str = expr_str - if replacers: - expr_str = self.replace_op_dict(expr_str, replacers) - self.expr_str = self.replace_op_dict(expr_str, replacers) + def __init__(self, expr_str, name_filter=None): + try: left, self._op, right = self._partition(expr_str) self._left = self._parse(left, name_filter) @@ -92,21 +88,8 @@ def __init__(self, expr_str, name_filter=None, replacers=None): print("error while parsing expression '%s'" % expr_str) raise - def replace_op_dict(self, expr_str, replacers): - """Update string with content of replacers list of dicts.""" - for token in replacers: - for op_token, op_str in token.items(): - if op_token in expr_str: - expr_str = expr_str.replace(op_token, op_str) - # replace 'member of' operator - expr_str = expr_str.replace("\\", ".") - return expr_str - def eval(self, data=None): """Evaluate the expression to an integer.""" - print(self._expr_str) - print(self.expr_str) - print(self._left, self._op, self._right) if isinstance(self._left, Expression): left = self._left.eval(data) elif isinstance(self._left, str): From da41349fdb91f7f1df8a787b02b53ceaea26c5bc Mon Sep 17 00:00:00 2001 From: HENDRIX-ZT2 Date: Fri, 20 Sep 2019 09:32:21 +0200 Subject: [PATCH 06/22] Add << and >> operators Now functional enough to read a full nif with some modifications to nif.xml --- pyffi/object_models/xml/__init__.py | 10 ++++++++-- pyffi/object_models/xml/expression.py | 11 +++++++++-- 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/pyffi/object_models/xml/__init__.py b/pyffi/object_models/xml/__init__.py index c9f0eec21..142d68ccb 100644 --- a/pyffi/object_models/xml/__init__.py +++ b/pyffi/object_models/xml/__init__.py @@ -581,7 +581,7 @@ def startElement(self, name, attrs): elif tag == self.tag_bit_struct: self.class_bases += (BitStructBase,) self.class_name = attrs["name"] - print("BITSTRUCT:",self.class_name) + # print("BITSTRUCT:",self.class_name) try: numbytes = int(attrs["numbytes"]) except KeyError: @@ -698,7 +698,7 @@ def startElement(self, name, attrs): self.class_dict["_attrs"].append( BitStructAttribute(self.cls, - dict(name=attrs["name"], numbits=1))) + dict(name=attrs["name"], numbits=attrs["width"]))) else: raise XmlError( "only bits tags allowed in struct type declaration") @@ -828,8 +828,14 @@ def replace_tokens(self, attr_dict, replacers): for op_token, op_str in tokens.items(): if op_token in expr_str: expr_str = expr_str.replace(op_token, op_str) + expr_str = expr_str.replace(">", ">") + expr_str = expr_str.replace("<", "<") + expr_str = expr_str.replace("&", "&") + # expr_str = expr_str.replace("&", "&") + #LEN[ # replace 'member of' operator & update attr_dict[target_attrib] = expr_str.replace("\\", ".") + # print(attr_dict) return attr_dict def characters(self, chars): diff --git a/pyffi/object_models/xml/expression.py b/pyffi/object_models/xml/expression.py index cdb012abb..6175dd1b4 100644 --- a/pyffi/object_models/xml/expression.py +++ b/pyffi/object_models/xml/expression.py @@ -75,7 +75,7 @@ class Expression(object): False """ - operators = set(('==', '!=', '>=', '<=', '&&', '||', '&', '|', '-', '!', + operators = set(('==', '!=', '>=', '<=', '&&', '||', '&', '|', '-', '!', '>>', '<<', '<', '>', '/', '*', '+')) def __init__(self, expr_str, name_filter=None): @@ -150,11 +150,18 @@ def eval(self, data=None): elif self._op == '<': return left < right elif self._op == '/': - return left / right + if right > 0: + return left / right + else: + return 0 elif self._op == '*': return left * right elif self._op == '+': return left + right + elif self._op == '<<': + return left << right + elif self._op == '>>': + return left >> right else: raise NotImplementedError("expression syntax error: operator '" + self._op + "' not implemented") From d4961a8eb42d2107a61e224b1257d2d39d016047 Mon Sep 17 00:00:00 2001 From: HENDRIX-ZT2 Date: Fri, 20 Sep 2019 12:13:43 +0200 Subject: [PATCH 07/22] Rename istemplate to generic Also raise error again if check fails. --- pyffi/formats/cgf/cgf.xml | 4 ++-- pyffi/object_models/xml/__init__.py | 11 ++++------- 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/pyffi/formats/cgf/cgf.xml b/pyffi/formats/cgf/cgf.xml index ea47ab9f1..acc5c652a 100644 --- a/pyffi/formats/cgf/cgf.xml +++ b/pyffi/formats/cgf/cgf.xml @@ -46,11 +46,11 @@ A null-terminated string. - + A reference to another block. - + A back-reference to another block (up the hierarchy). diff --git a/pyffi/object_models/xml/__init__.py b/pyffi/object_models/xml/__init__.py index 142d68ccb..3748f9094 100644 --- a/pyffi/object_models/xml/__init__.py +++ b/pyffi/object_models/xml/__init__.py @@ -514,11 +514,11 @@ def startElement(self, name, attrs): % class_basename) else: self.class_bases = (StructBase,) - # istemplate attribute is optional + # 'generic' attribute is optional # if not set, then the struct is not a template # set attributes (see class StructBase) self.class_dict = { - "_is_template": attrs.get("istemplate") == "1", + "_is_template": attrs.get("generic") == "true", "_attrs": [], "_games": {}, "__doc__": "", @@ -532,12 +532,9 @@ def startElement(self, name, attrs): # via the name of the class. self.basic_class = getattr(self.cls, self.class_name) # check the class variables - is_template = (attrs.get("istemplate") == "1") + is_template = (attrs.get("generic") == "true") if self.basic_class._is_template != is_template: - # raise XmlError( - # 'class %s should have _is_template = %s' - # % (self.class_name, is_template)) - print( + raise XmlError( 'class %s should have _is_template = %s' % (self.class_name, is_template)) From 9c75674b7dfe8d9654395b0a5d9bd6adfad60176 Mon Sep 17 00:00:00 2001 From: HENDRIX-ZT2 Date: Fri, 20 Sep 2019 14:49:54 +0200 Subject: [PATCH 08/22] Update 'abstract' checking backward compatible --- pyffi/object_models/xml/__init__.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pyffi/object_models/xml/__init__.py b/pyffi/object_models/xml/__init__.py index 3748f9094..9652810de 100644 --- a/pyffi/object_models/xml/__init__.py +++ b/pyffi/object_models/xml/__init__.py @@ -181,6 +181,10 @@ class StructAttribute(object): there is no upper limit. """ + vercond = None + """The version condition of this member variable, as + :class:`Expression` or ``type(None)``. + """ userver = None """The user version this member exists, as ``int``, and ``None`` if it exists for all user versions. @@ -223,7 +227,7 @@ def __init__(self, cls, attrs): self.ver2 = attrs.get("ver2") self.userver = attrs.get("userver") self.doc = "" # handled in xml parser's characters function - self.is_abstract = (attrs.get("abstract") == "1") + self.is_abstract = (attrs.get("abstract") in ("1", "true")) # post-processing if self.default: From 21dcced6f4808c8c405ab2f7cd958f5c137915af Mon Sep 17 00:00:00 2001 From: HENDRIX-ZT2 Date: Fri, 20 Sep 2019 16:45:47 +0200 Subject: [PATCH 09/22] Add template classes, refactor tokens - represent #T# by class `T`, which is just a dummy (same for #ARG#, but that's probably not needed) - refactor token replacement to make sure fixed tokens are always applied --- pyffi/formats/nif/__init__.py | 10 ++++++++++ pyffi/object_models/xml/__init__.py | 19 +++++++++---------- 2 files changed, 19 insertions(+), 10 deletions(-) diff --git a/pyffi/formats/nif/__init__.py b/pyffi/formats/nif/__init__.py index 4d8999587..241f0afa5 100644 --- a/pyffi/formats/nif/__init__.py +++ b/pyffi/formats/nif/__init__.py @@ -488,6 +488,16 @@ def write(self, stream, data): stream.write(struct.pack(data._byte_order + 'I', int(self._value))) + class T(pyffi.object_models.common.UShort): + """A dummy class""" + def __str__(self): + return "Template" + + class ARG(pyffi.object_models.common.UShort): + """A dummy class""" + def __str__(self): + return "ARG" + class Flags(pyffi.object_models.common.UShort): def __str__(self): return hex(self.get_value()) diff --git a/pyffi/object_models/xml/__init__.py b/pyffi/object_models/xml/__init__.py index 9652810de..c25d6a9b4 100644 --- a/pyffi/object_models/xml/__init__.py +++ b/pyffi/object_models/xml/__init__.py @@ -827,16 +827,15 @@ def replace_tokens(self, attr_dict, replacers): if target_attrib in attr_dict: expr_str = attr_dict[target_attrib] for op_token, op_str in tokens.items(): - if op_token in expr_str: - expr_str = expr_str.replace(op_token, op_str) - expr_str = expr_str.replace(">", ">") - expr_str = expr_str.replace("<", "<") - expr_str = expr_str.replace("&", "&") - # expr_str = expr_str.replace("&", "&") - #LEN[ - # replace 'member of' operator & update - attr_dict[target_attrib] = expr_str.replace("\\", ".") - # print(attr_dict) + expr_str = expr_str.replace(op_token, op_str) + attr_dict[target_attrib] = expr_str + # additional tokens that are not specified by nif.xml + fixed_tokens = ( ("\\", "."), (">", ">"), ("<", "<"), ("&", "&"), ("#T#", "T"), ("#ARG#", "ARG") ) + for attrib, expr_str in attr_dict.items(): + for op_token, op_str in fixed_tokens: + expr_str = expr_str.replace(op_token, op_str) + attr_dict[attrib] = expr_str + return attr_dict def characters(self, chars): From db4167d9acd7debbfc04bb673798e6d34df55bf7 Mon Sep 17 00:00:00 2001 From: HENDRIX-ZT2 Date: Sat, 21 Sep 2019 18:51:07 +0200 Subject: [PATCH 10/22] Refactoring: XML parsing with etree Much more readable and concise. --- pyffi/formats/nif/__init__.py | 12 +- pyffi/object_models/xml/__init__.py | 823 +++++++++----------------- pyffi/object_models/xml/expression.py | 16 +- pyffi/object_models/xml/struct_.py | 16 +- 4 files changed, 315 insertions(+), 552 deletions(-) diff --git a/pyffi/formats/nif/__init__.py b/pyffi/formats/nif/__init__.py index 241f0afa5..3b89db405 100644 --- a/pyffi/formats/nif/__init__.py +++ b/pyffi/formats/nif/__init__.py @@ -488,16 +488,11 @@ def write(self, stream, data): stream.write(struct.pack(data._byte_order + 'I', int(self._value))) - class T(pyffi.object_models.common.UShort): + class TEMPLATE(pyffi.object_models.common.UShort): """A dummy class""" def __str__(self): return "Template" - class ARG(pyffi.object_models.common.UShort): - """A dummy class""" - def __str__(self): - return "ARG" - class Flags(pyffi.object_models.common.UShort): def __str__(self): return hex(self.get_value()) @@ -1314,7 +1309,7 @@ def read(self, stream): self.inspect_version_only(stream) logger.debug("Version 0x%08X" % self.version) self.header.read(stream, data=self) - + # print(self.header) # list of root blocks # for versions < 3.3.0.13 this list is updated through the # "Top Level Object" string while reading the blocks @@ -1400,11 +1395,12 @@ def read(self, stream): # read the block try: block.read(stream, self) + # print(block) except: logger.exception("Reading %s failed" % block.__class__) #logger.error("link stack: %s" % self._link_stack) #logger.error("block that failed:") - #logger.error("%s" % block) + logger.error("%s" % block) raise # complete NiDataStream data if block_type == "NiDataStream": diff --git a/pyffi/object_models/xml/__init__.py b/pyffi/object_models/xml/__init__.py index c25d6a9b4..755d4065c 100644 --- a/pyffi/object_models/xml/__init__.py +++ b/pyffi/object_models/xml/__init__.py @@ -45,6 +45,7 @@ import os.path import sys import xml.sax +import xml.etree.ElementTree as ET import pyffi.object_models from pyffi.object_models.xml.struct_ import StructBase @@ -75,7 +76,6 @@ def __init__(cls, name, bases, dct): :param bases: The base classes, usually (object,). :param dct: A dictionary of class attributes, such as 'xml_file_name'. """ - super(MetaFileFormat, cls).__init__(name, bases, dct) # preparation: make deep copy of lists of enums, structs, etc. @@ -84,30 +84,21 @@ def __init__(cls, name, bases, dct): cls.xml_bit_struct = cls.xml_bit_struct[:] cls.xml_struct = cls.xml_struct[:] - # parse XML - - # we check dct to avoid parsing the same file more than once in - # the hierarchy + # we check dct to avoid parsing the same file more than once in the hierarchy xml_file_name = dct.get('xml_file_name') if xml_file_name: - # set up XML parser - parser = xml.sax.make_parser() - parser.setContentHandler(XmlSaxHandler(cls, name, bases, dct)) - + cls.logger.debug("Parsing %s and generating classes." % xml_file_name) # open XML file - xml_file = cls.openfile(xml_file_name, cls.xml_file_path) - - # parse the XML file: control is now passed on to XmlSaxHandler - # which takes care of the class creation - cls.logger.debug("Parsing %s and generating classes." - % xml_file_name) start = time.time() + xml_file = cls.openfile(xml_file_name, cls.xml_file_path) + xmlp = XmlParser(cls, name, bases) try: - parser.parse(xml_file) + xmlp.load_xml(xml_file) finally: xml_file.close() - cls.logger.debug("Parsing finished in %.3f seconds." - % (time.time() - start)) + + cls.logger.debug("Parsing finished in %.3f seconds." % (time.time() - start)) + class FileFormat(pyffi.object_models.FileFormat, metaclass=MetaFileFormat): @@ -205,19 +196,18 @@ def __init__(self, cls, attrs): try: attrs_type_str = attrs["type"] except KeyError: - raise AttributeError("'%s' is missing a type attribute" - % self.displayname) + raise AttributeError("'%s' is missing a type attribute" % self.displayname) if attrs_type_str != "TEMPLATE": try: self.type_ = getattr(cls, attrs_type_str) except AttributeError: - # forward declaration, resolved at endDocument + # forward declaration, resolved at final_cleanup self.type_ = attrs_type_str else: self.type_ = type(None) # type determined at runtime # optional parameters self.default = attrs.get("default") - self.template = attrs.get("template") # resolved in endDocument + self.template = attrs.get("template") # resolved in final_cleanup self.arg = attrs.get("arg") self.arr1 = attrs.get("arr1") self.arr2 = attrs.get("arr2") @@ -226,7 +216,7 @@ def __init__(self, cls, attrs): self.ver1 = attrs.get("ver1") self.ver2 = attrs.get("ver2") self.userver = attrs.get("userver") - self.doc = "" # handled in xml parser's characters function + self.doc = "" # handled in xml parser self.is_abstract = (attrs.get("abstract") in ("1", "true")) # post-processing @@ -298,337 +288,210 @@ class XmlError(Exception): pass -class XmlSaxHandler(xml.sax.handler.ContentHandler): - """This class contains all functions for parsing the xml and converting - the xml structure into Python classes.""" - tag_file = 1 - tag_version = 2 - tag_basic = 3 - tag_alias = 4 - tag_enum = 5 - tag_option = 6 - tag_bit_struct = 7 - tag_struct = 8 - tag_attribute = 9 - tag_bits = 10 - # new stuff - tag_module = 11 - tag_token = 12 - # we don't care about the type of token children - tag_subtoken = 18 - # tag_bitfield = 19 - tag_member = 20 - - tags = { - "fileformat": tag_file, - "version": tag_version, - "basic": tag_basic, - "alias": tag_alias, - "enum": tag_enum, - "option": tag_option, - # no longer in nif xml - "bitstruct": tag_bit_struct, - # no longer in nif xml - "struct": tag_struct, - "bits": tag_bits, - "add": tag_attribute, - #new stuff - "module": tag_module, - "token": tag_token, - "verexpr": tag_subtoken, - "condexpr": tag_subtoken, - "verset": tag_subtoken, - "default": tag_subtoken, - "range": tag_subtoken, - "global": tag_subtoken, - "operator": tag_subtoken, - "bitfield": tag_bit_struct, - "member": tag_member, - # seems to behave like option used to do OR rather add? - "field": tag_attribute - } - - # for compatibility with niftools - tags_niftools = { - "niftoolsxml": tag_file, - "compound": tag_struct, - "niobject": tag_struct, - "bitflags": tag_bit_struct} - - def __init__(self, cls, name, bases, dct): - """Set up the xml parser. - - Upon instantiation this function does the following: - - Creates a dictionary C{cls.versions} which maps each supported - version strings onto a version integer. - - Creates a dictionary C{cls.games} which maps each supported game - onto a list of versions. - - Makes an alias C{self.cls} for C{cls}. - - Initializes a stack C{self.stack} of xml tags. - - Initializes the current tag. - """ - # initialize base class (no super because base class is old style) - xml.sax.handler.ContentHandler.__init__(self) - - # save dictionary for future use - self.dct = dct +class XmlParser: + struct_types = ("compound", "niobject", "struct") + bitstruct_types = ("bitfield", "bitflags", "bitstruct") + def __init__(self, cls, name, bases): + """Set up the xml parser.""" # initialize dictionaries - # cls.version maps each supported version string to a version number + # map each supported version string to a version number cls.versions = {} - # cls.games maps each supported game to a list of header version - # numbers + # map each supported game to a list of header version numbers cls.games = {} - # note: block versions are stored in the _games attribute of the - # struct class + # note: block versions are stored in the _games attribute of the struct class - # initialize tag stack - self.stack = [] - # keep track last element of self.stack - # storing this reduces overhead as profiling has shown - self.current_tag = None - - # cls needs to be accessed in member functions, so make it an instance - # member variable + # cls needs to be accessed in member functions, so make it an instance member variable self.cls = cls # elements for creating new classes self.class_name = None self.class_dict = None - self.class_bases = () - - # elements for basic classes - self.basic_class = None + self.base_class = () # elements for versions self.version_string = None - # a list of dicts that will store each token - self.tokens = [] - - def pushTag(self, tag): - """Push tag C{tag} on the stack and make it the current tag. - - :param tag: The tag to put on the stack.""" - self.stack.insert(0, tag) - self.current_tag = tag - - def popTag(self): - """Pop the current tag from the stack and return it. Also update - the current tag. - - :return: The tag popped from the stack.""" - lasttag = self.stack.pop(0) - try: - self.current_tag = self.stack[0] - except IndexError: - self.current_tag = None - return lasttag - - def startElement(self, name, attrs): - """Called when parser starts parsing an element called C{name}. - - This function sets up all variables for creating the class - in the C{self.endElement} function. For struct elements, it will set up - C{self.class_name}, C{self.class_bases}, and C{self.class_dict} which - will be used to create the class by invokation of C{type} in - C{self.endElement}. For basic, enum, and bitstruct elements, it will - set up C{self.basic_class} to link to the proper class implemented by - C{self.cls}. The code also performs sanity checks on the attributes. - - For xml add tags, the function will add an entry to the - C{self.class_dict["_attrs"]} list. Note that this list is used by the - struct metaclass: the class attributes are created exactly from this - list. - - :param name: The name of the xml element. - :param attrs: A dictionary of attributes of the element.""" - # get the tag identifier + # list of tuples ({tokens}, (target_attribs)) for each + self.tokens = [ ] + self.versions = [ ([], ("versions", "until", "since")), ] + + def load_xml(self, file): + """Loads an XML (can be filepath or open file) and does all parsing""" + tree = ET.parse(file) + root = tree.getroot() + self.load_root(root) + self.final_cleanup() + + def load_root(self, root): + """Goes over all children of the root node and calls the appropriate function depending on type of the child""" + for child in root: + if child.tag in self.struct_types: + res = self.read_struct(child) + elif child.tag in self.bitstruct_types: + res = self.read_bitstruct(child) + elif child.tag == "basic": + res = self.read_basic(child) + elif child.tag == "alias": + res = self.read_alias(child) + elif child.tag == "enum": + res = self.read_enum(child) + elif child.tag == "module": + res = self.read_module(child) + elif child.tag == "version": + res = self.read_version(child) + elif child.tag == "token": + res = self.read_token(child) + + # the following constructs do not create classes + def read_token(self, token): + """Reads an xml block and stores it in the tokens list""" + self.tokens.append( ([], token.attrib["attrs"].split(" ") ) ) + for sub_token in token: + self.tokens[-1][0].append( (sub_token.attrib["token"], sub_token.attrib["string"]) ) + + def read_version(self, version): + """Reads an xml block and stores it in the versions list""" + # todo [versions] this ignores the user vers! + # versions must be in reverse order so don't append but insert at beginning + if "id" in version.attrib: + self.versions[0][0].insert( 0, (version.attrib["id"], version.attrib["num"]) ) + # add to supported versions + self.version_string = version.attrib["num"] + self.cls.versions[self.version_string] = self.cls.version_number(self.version_string) + self.update_gamesdict(self.cls.games, version.text) + self.version_string = None + + def read_module(self, module): + """Reads a xml block""" + # no children, not interesting for now + pass + + def read_basic(self, basic): + """Maps to a type defined in self.cls""" + self.class_name = basic.attrib["name"] + # Each basic type corresponds to a type defined in C{self.cls}. + # The link between basic types and C{self.cls} types is done via the name of the class. + basic_class = getattr(self.cls, self.class_name) + # check the class variables + is_template = (basic.attrib.get("generic") == "true") + if basic_class._is_template != is_template: + raise XmlError( 'class %s should have _is_template = %s' % (self.class_name, is_template)) + + # link class cls. to basic_class + setattr(self.cls, self.class_name, basic_class) + + # the following constructs create classes + def read_bitstruct(self, bitstruct): + """Create a bitstruct class""" + attrs = self.replace_tokens(bitstruct.attrib) + self.base_class = BitStructBase + self.update_class_dict(attrs, bitstruct.text) try: - tag = self.tags[name] + numbytes = int(attrs["numbytes"]) except KeyError: + # niftools style: storage attribute + numbytes = getattr(self.cls, attrs["storage"]).get_size() + self.class_dict["_attrs"] = [] + self.class_dict["_numbytes"] = numbytes + for member in bitstruct: + attrs = self.replace_tokens(member.attrib) + if member.tag == "bits": + # eg. + # mandatory parameters + bit_attrs = attrs + elif member.tag == "option": + # niftools compatibility, we have a bitflags field + # so convert value into numbits + # first, calculate current bit position + bitpos = sum(bitattr.numbits for bitattr in self.class_dict["_attrs"]) + # avoid crash + if "value" in attrs: + # check if extra bits must be inserted + numextrabits = int(attrs["value"]) - bitpos + if numextrabits < 0: + raise XmlError("values of bitflags must be increasing") + if numextrabits > 0: + reserved = dict(name="Reserved Bits %i"% len(self.class_dict["_attrs"]), numbits=numextrabits) + self.class_dict["_attrs"].append( BitStructAttribute( self.cls, reserved)) + # add the actual attribute + bit_attrs = dict(name=attrs["name"], numbits=1) + # new nif xml + elif member.tag == "member": + bit_attrs = dict(name=attrs["name"], numbits=attrs["width"]) + else: + raise XmlError("only bits tags allowed in struct type declaration") + + self.class_dict["_attrs"].append( BitStructAttribute(self.cls, bit_attrs) ) + self.update_doc(self.class_dict["_attrs"][-1].doc, member.text) + + self.create_class(bitstruct.tag) + + def read_struct(self, struct): + """Create a struct class""" + attrs = self.replace_tokens(struct.attrib) + self.update_class_dict(attrs, struct.text) + # struct types can be organized in a hierarchy + # if inherit attribute is defined, look for corresponding base block + class_basename = attrs.get("inherit") + if class_basename: + # class_basename must have been assigned to a class try: - tag = self.tags_niftools[name] + self.base_class = getattr(self.cls, class_basename) except KeyError: - raise XmlError("error unknown element '%s'" % name) - - # Check the stack, if the stack does not exist then we must be - # at the root of the xml file, and the tag must be "fileformat". - # The fileformat tag has no further attributes of interest, - # so we can exit the function after pushing the tag on the stack. - if not self.stack: - if tag != self.tag_file: - raise XmlError("this is not a fileformat xml file") - self.pushTag(tag) - return - - # Now do a number of things, depending on the tag that was last - # pushed on the stack; this is self.current_tag, and reflects the - # tag in which is embedded. - # - # For each struct, alias, enum, and bitstruct tag, we shall - # create a class. So assign to C{self.class_name} the name of that - # class, C{self.class_bases} to the base of that class, and - # C{self.class_dict} to the class dictionary. - # - # For a basic tag, C{self.class_name} is the name of the - # class and C{self.basic_class} is the corresponding class in - # C{self.cls}. - # - # For a version tag, C{self.version_string} describes the version as a - # string. - if self.current_tag == self.tag_struct: - self.pushTag(tag) - attrs = self.replace_tokens(attrs, self.tokens) - # print(attrs["name"]) - # struct -> attribute - if tag == self.tag_attribute: + raise XmlError( "typo, or forward declaration of struct %s" % class_basename) + else: + self.base_class = StructBase + # 'generic' attribute is optional + # if not set, then the struct is not a template + # set attributes (see class StructBase) + self.class_dict["_is_template" ] = (attrs.get("generic") == "true") + self.class_dict["_attrs" ] = [] + self.class_dict["_games" ] = {} + for field in struct: + attrs = self.replace_tokens(field.attrib) + # the common case + if field.tag in ("add", "field"): # add attribute to class dictionary - self.class_dict["_attrs"].append( - StructAttribute(self.cls, attrs)) - # struct -> version - elif tag == self.tag_version: + self.class_dict["_attrs"].append( StructAttribute(self.cls, attrs) ) + self.update_doc(self.class_dict["_attrs"][-1].doc, field.text) + # not found in current nifxml + elif field.tag == "version": # set the version string - self.version_string = str(attrs["num"]) - self.cls.versions[self.version_string] = self.cls.version_number( - self.version_string) - # (class_dict["_games"] is updated when reading the characters) + self.version_string = attrs["num"] + self.cls.versions[self.version_string] = self.cls.version_number(self.version_string) + self.update_gamesdict(self.class_dict["_games"], field.text) else: - print( - "only add and version tags allowed in struct declaration") - elif self.current_tag == self.tag_file: - self.pushTag(tag) - - # fileformat -> struct - if tag == self.tag_struct: - self.class_name = attrs["name"] - # struct types can be organized in a hierarchy - # if inherit attribute is defined, then look for corresponding - # base block - class_basename = attrs.get("inherit") - if class_basename: - # if that base struct has not yet been assigned to a - # class, then we have a problem - try: - self.class_bases += ( - getattr(self.cls, class_basename), ) - except KeyError: - raise XmlError( - "typo, or forward declaration of struct %s" - % class_basename) - else: - self.class_bases = (StructBase,) - # 'generic' attribute is optional - # if not set, then the struct is not a template - # set attributes (see class StructBase) - self.class_dict = { - "_is_template": attrs.get("generic") == "true", - "_attrs": [], - "_games": {}, - "__doc__": "", - "__module__": self.cls.__module__} - - # fileformat -> basic - elif tag == self.tag_basic: - self.class_name = attrs["name"] - # Each basic type corresponds to a type defined in C{self.cls}. - # The link between basic types and C{self.cls} types is done - # via the name of the class. - self.basic_class = getattr(self.cls, self.class_name) - # check the class variables - is_template = (attrs.get("generic") == "true") - if self.basic_class._is_template != is_template: - raise XmlError( - 'class %s should have _is_template = %s' - % (self.class_name, is_template)) - - # fileformat -> enum - elif tag == self.tag_enum: - self.class_bases += (EnumBase,) - self.class_name = attrs["name"] - try: - numbytes = int(attrs["numbytes"]) - except KeyError: - # niftools format uses a storage - # get number of bytes from that - typename = attrs["storage"] - try: - typ = getattr(self.cls, typename) - except AttributeError: - raise XmlError( - "typo, or forward declaration of type %s" - % typename) - numbytes = typ.get_size() - self.class_dict = {"__doc__": "", - "_numbytes": numbytes, - "_enumkeys": [], "_enumvalues": [], - "__module__": self.cls.__module__} - - # fileformat -> alias - elif tag == self.tag_alias: - self.class_name = attrs["name"] - typename = attrs["type"] - try: - self.class_bases += (getattr(self.cls, typename),) - except AttributeError: - raise XmlError( - "typo, or forward declaration of type %s" % typename) - self.class_dict = {"__doc__": "", - "__module__": self.cls.__module__} - - # fileformat -> bitstruct - # this works like an alias for now, will add special - # BitStruct base class later - elif tag == self.tag_bit_struct: - self.class_bases += (BitStructBase,) - self.class_name = attrs["name"] - # print("BITSTRUCT:",self.class_name) - try: - numbytes = int(attrs["numbytes"]) - except KeyError: - # niftools style: storage attribute - numbytes = getattr(self.cls, attrs["storage"]).get_size() - self.class_dict = {"_attrs": [], "__doc__": "", - "_numbytes": numbytes, - "__module__": self.cls.__module__} - # # fileformat -> bitfield - # elif tag == self.tag_bitfield: - # self.bitfield_name = str(attrs["name"]) - # self.bitfield_storage = str(attrs["storage"]) - # # self.bitfield_versions = str(attrs["versions"]) - - # fileformat -> version - elif tag == self.tag_version: - self.version_string = str(attrs["num"]) - self.cls.versions[self.version_string] = self.cls.version_number( - self.version_string) - # (self.cls.games is updated when reading the characters) - # fileformat -> module - elif tag == self.tag_module: - self.module_name = str(attrs["name"]) - self.module_priority = str(attrs["priority"]) - if "depends" in attrs: - self.module_depends = str(attrs["depends"]) - # fileformat -> token - elif tag == self.tag_token: - # create a new dict for this token to which we add token - str pairs for replacement - # also the target attributes that these tokens should be applied to - # store both as a tuple - self.tokens.append( ( {}, attrs["attrs"].split(" ") ) ) - - else: - raise XmlError(""" expected basic, alias, enum, bitstruct, struct, or version, - but got %s instead""" % name) - - elif self.current_tag == self.tag_version: - raise XmlError("version tag must not contain any sub tags") - - elif self.current_tag == self.tag_alias: - raise XmlError("alias tag must not contain any sub tags") - - elif self.current_tag == self.tag_enum: - self.pushTag(tag) - if not tag == self.tag_option: + print("only add and version tags allowed in struct declaration") + # load defaults for this + for default in field: + if default.tag != "default": + raise AttributeError("struct children's children must be 'default' tag") + self.create_class(struct.tag) + + def read_enum(self, enum): + """Create an enum class""" + attrs = self.replace_tokens(enum.attrib) + self.base_class = EnumBase + self.update_class_dict(attrs, enum.text) + try: + numbytes = int(attrs["numbytes"]) + except KeyError: + # niftools format uses a storage + # get number of bytes from that + typename = attrs["storage"] + try: + typ = getattr(self.cls, typename) + except AttributeError: + raise XmlError("typo, or forward declaration of type %s" % typename) + numbytes = typ.get_size() + # add stuff to classdict + self.class_dict["_numbytes"] = numbytes + self.class_dict["_enumkeys"] = [] + self.class_dict["_enumvalues"] = [] + for option in enum: + attrs = self.replace_tokens(option.attrib) + if option.tag not in ("option",): raise XmlError("only option tags allowed in enum declaration") value = attrs["value"] try: @@ -639,159 +502,108 @@ def startElement(self, name, attrs): value = int(value, 16) self.class_dict["_enumkeys"].append(attrs["name"]) self.class_dict["_enumvalues"].append(value) + self.create_class(enum.tag) - # elif self.current_tag == self.tag_bitfield: - # self.pushTag(tag) - # if not tag == self.tag_member: - # raise XmlError("only member tags allowed in bitfield declaration") - # # member width="3" pos="12" mask="0xF000" name="Test Func" type="StencilTestFunc" default="TEST_GREATER - # self.class_dict["width"] = attrs["width"] - # self.class_dict["pos"] = attrs["pos"] - # self.class_dict["name"].append(attrs["name"]) - - elif self.current_tag == self.tag_token: - self.pushTag(tag) - # this is the symbol that represents the operation in nif.xml, eg "#EQ#" - op_token = attrs["token"] - op_string = attrs["string"] - # store it - self.tokens[-1][0][op_token] = op_string - - elif self.current_tag == self.tag_option: - self.pushTag(tag) - elif self.current_tag == self.tag_attribute: - self.pushTag(tag) - elif self.current_tag == self.tag_bit_struct: - self.pushTag(tag) - attrs = self.replace_tokens(attrs, self.tokens) - if tag == self.tag_bits: - # eg. - # mandatory parameters - self.class_dict["_attrs"].append( - BitStructAttribute(self.cls, attrs)) - elif tag == self.tag_option: - # niftools compatibility, we have a bitflags field - # so convert value into numbits - # first, calculate current bit position - bitpos = sum(bitattr.numbits - for bitattr in self.class_dict["_attrs"]) - # avoid crash - if "value" not in attrs: - return - # check if extra bits must be inserted - numextrabits = int(attrs["value"]) - bitpos - if numextrabits < 0: - raise XmlError("values of bitflags must be increasing") - if numextrabits > 0: - self.class_dict["_attrs"].append( - BitStructAttribute( - self.cls, - dict(name="Reserved Bits %i" - % len(self.class_dict["_attrs"]), - numbits=numextrabits))) - # add the actual attribute - self.class_dict["_attrs"].append( - BitStructAttribute( - self.cls, - dict(name=attrs["name"], numbits=1))) - # new nif xml - elif tag == self.tag_member: - - self.class_dict["_attrs"].append( - BitStructAttribute(self.cls, - dict(name=attrs["name"], numbits=attrs["width"]))) - else: - raise XmlError( - "only bits tags allowed in struct type declaration") + def read_alias(self, alias): + """Create an alias class, ie. one that gives access to another class""" + self.update_class_dict(alias.attrib, alias.text) + typename = alias.attrib["type"] + try: + self.base_class = getattr(self.cls, typename) + except AttributeError: + raise XmlError("typo, or forward declaration of type %s" % typename) + self.create_class(alias.tag) - else: - # raise XmlError("unhandled tag %s" % name) - print("unhandled tag %s" % name) - def endElement(self, name): - """Called at the end of each xml tag. + # the following are helper functions - Creates classes.""" - if not self.stack: - raise XmlError("mismatching end element tag for element %s" % name) - try: - tag = self.tags[name] - except KeyError: - try: - tag = self.tags_niftools[name] - except KeyError: - raise XmlError("error unknown element %s" % name) - if self.popTag() != tag: - raise XmlError("mismatching end element tag for element %s" % name) - elif tag == self.tag_attribute: - return # improves performance - elif tag in (self.tag_struct, - self.tag_enum, - self.tag_alias, - self.tag_bit_struct): - # create class - # assign it to cls. if it has not been implemented - # internally - cls_klass = getattr(self.cls, self.class_name, None) - if cls_klass and issubclass(cls_klass, BasicBase): - # overrides a basic type - not much to do - pass - else: - # check if we have a customizer class - if cls_klass: - # exists: create and add to base class of customizer - gen_klass = type( - "_" + str(self.class_name), - self.class_bases, self.class_dict) - setattr(self.cls, "_" + self.class_name, gen_klass) - # recreate the class, to ensure that the - # metaclass is called!! - # (otherwise, cls_klass does not have correct - # _attribute_list, etc.) - cls_klass = type( - cls_klass.__name__, - (gen_klass,) + cls_klass.__bases__, - dict(cls_klass.__dict__)) - setattr(self.cls, self.class_name, cls_klass) - # if the class derives from Data, then make an alias - if issubclass( - cls_klass, - pyffi.object_models.FileFormat.Data): - self.cls.Data = cls_klass - # for the stuff below - gen_class = cls_klass + def update_gamesdict(self, gamesdict, ver_text): + if ver_text: + # update the gamesdict dictionary + for gamestr in (g.strip() for g in ver_text.split(',')): + if gamestr in gamesdict: + gamesdict[gamestr].append(self.cls.versions[self.version_string]) else: - # does not yet exist: create it and assign to class dict - gen_klass = type( - str(self.class_name), self.class_bases, self.class_dict) - setattr(self.cls, self.class_name, gen_klass) - # append class to the appropriate list - if tag == self.tag_struct: - self.cls.xml_struct.append(gen_klass) - elif tag == self.tag_enum: - self.cls.xml_enum.append(gen_klass) - elif tag == self.tag_alias: - self.cls.xml_alias.append(gen_klass) - elif tag == self.tag_bit_struct: - self.cls.xml_bit_struct.append(gen_klass) - # reset variables - self.class_name = None - self.class_dict = None - self.class_bases = () - elif tag == self.tag_basic: - # link class cls. to self.basic_class - setattr(self.cls, self.class_name, self.basic_class) - # reset variable - self.basic_class = None - elif tag == self.tag_version: - # reset variable - self.version_string = None - - def endDocument(self): + gamesdict[gamestr] = [self.cls.versions[self.version_string]] + + def update_class_dict(self, attrs, doc_text): + """This initializes class_dict, sets the class name and doc text""" + doc_text = doc_text.strip() if doc_text else "" + self.class_name = attrs["name"] + self.class_dict = {"__doc__": doc_text, "__module__": self.cls.__module__} + + def update_doc(self, doc, doc_text): + if doc_text: + doc += doc_text.strip() + + def create_class(self, tag): + """Creates a class for (tag name of the class that was just finished)""" + # assign it to cls. if it has not been implemented internally + + # type(name, bases, dict) returns a new type object, essentially a dynamic form of the class statement + cls_klass = getattr(self.cls, self.class_name, None) + # does the class exist? + if cls_klass: + # do nothing if this is a Basic type + if issubclass(cls_klass, BasicBase): + return + # it has been created in format's __init__.py + # create and add to base class of customizer + gen_klass = type("_"+self.class_name, (self.base_class,), self.class_dict) + setattr(self.cls, "_"+self.class_name, gen_klass) + # recreate the class, to ensure that the metaclass is called!! + # (otherwise, cls_klass does not have correct _attribute_list, etc.) + cls_klass = type(cls_klass.__name__, (gen_klass,) + cls_klass.__bases__, dict(cls_klass.__dict__)) + setattr(self.cls, self.class_name, cls_klass) + # if the class derives from Data, then make an alias + if issubclass(cls_klass, pyffi.object_models.FileFormat.Data): + self.cls.Data = cls_klass + # for the stuff below + gen_class = cls_klass + # I think the above is a typo, should be: + # gen_klass = cls_klass + else: + # does not yet exist: create it and assign to class dict + gen_klass = type(self.class_name, (self.base_class,), self.class_dict) + setattr(self.cls, self.class_name, gen_klass) + # append class to the appropriate list + if tag in self.struct_types: + self.cls.xml_struct.append(gen_klass) + elif tag in self.bitstruct_types: + self.cls.xml_bit_struct.append(gen_klass) + elif tag == "enum": + self.cls.xml_enum.append(gen_klass) + elif tag == "alias": + self.cls.xml_alias.append(gen_klass) + + def replace_tokens(self, attr_dict): + """Update attr_dict with content of tokens+versions list.""" + # replace versions after tokens because tokens include versions + for tokens, target_attribs in self.tokens + self.versions: + for target_attrib in target_attribs: + if target_attrib in attr_dict: + expr_str = attr_dict[target_attrib] + for op_token, op_str in tokens: + expr_str = expr_str.replace(op_token, op_str) + attr_dict[target_attrib] = expr_str + # additional tokens that are not specified by nif.xml + fixed_tokens = ( ("\\", "."), (">", ">"), ("<", "<"), ("&", "&"), ("#ARG#", "ARG"), ("#T#", "TEMPLATE") ) + for attrib, expr_str in attr_dict.items(): + for op_token, op_str in fixed_tokens: + expr_str = expr_str.replace(op_token, op_str) + attr_dict[attrib] = expr_str + # onlyT & excludeT act as aliases for deprecated cond + prefs = ( ("onlyT", ""), ("excludeT", "!") ) + for t, pref in prefs: + if t in attr_dict: + attr_dict["cond"] = pref+attr_dict[t] + break + return attr_dict + + def final_cleanup(self): """Called when the xml is completely parsed. - Searches and adds class customized functions. - For version tags, adds version to version and game lists. + Fixes forward declaration of templates. """ # get 'name_attribute' for all classes # we need this to fix them in cond="..." later @@ -807,59 +619,10 @@ def endDocument(self): for attr in obj._attrs: templ = attr.template if isinstance(templ, str): - attr.template = \ - getattr(self.cls, templ) if templ != "TEMPLATE" \ - else type(None) + attr.template = getattr(self.cls, templ) if templ != "TEMPLATE" else type(None) attrtype = attr.type_ if isinstance(attrtype, str): attr.type_ = getattr(self.cls, attrtype) # fix refs to types in conditions if attr.cond: - attr.cond.map_( - lambda x: klass_filter[x] if x in klass_filter else x) - - def replace_tokens(self, attr_dict, replacers): - """Update attr_dict with content of replacers list of dicts.""" - attr_dict = dict(attr_dict) - # replacers is a list of tuples ({tokens}, (target_attribs)) for each - for tokens, target_attribs in replacers: - for target_attrib in target_attribs: - if target_attrib in attr_dict: - expr_str = attr_dict[target_attrib] - for op_token, op_str in tokens.items(): - expr_str = expr_str.replace(op_token, op_str) - attr_dict[target_attrib] = expr_str - # additional tokens that are not specified by nif.xml - fixed_tokens = ( ("\\", "."), (">", ">"), ("<", "<"), ("&", "&"), ("#T#", "T"), ("#ARG#", "ARG") ) - for attrib, expr_str in attr_dict.items(): - for op_token, op_str in fixed_tokens: - expr_str = expr_str.replace(op_token, op_str) - attr_dict[attrib] = expr_str - - return attr_dict - - def characters(self, chars): - """Add the string C{chars} to the docstring. - For version tags, updates the game version list.""" - if self.current_tag in (self.tag_attribute, self.tag_bits): - self.class_dict["_attrs"][-1].doc += str(chars.strip()) - elif self.current_tag in (self.tag_struct, self.tag_enum, - self.tag_alias): - self.class_dict["__doc__"] += str(chars.strip()) - elif self.current_tag == self.tag_version: - # fileformat -> version - if self.stack[1] == self.tag_file: - gamesdict = self.cls.games - # struct -> version - elif self.stack[1] == self.tag_struct: - gamesdict = self.class_dict["_games"] - else: - raise XmlError("version parsing error at '%s'" % chars) - # update the gamesdict dictionary - for gamestr in (str(g.strip()) for g in chars.split(',')): - if gamestr in gamesdict: - gamesdict[gamestr].append( - self.cls.versions[self.version_string]) - else: - gamesdict[gamestr] = [ - self.cls.versions[self.version_string]] + attr.cond.map_(lambda x: klass_filter[x] if x in klass_filter else x) diff --git a/pyffi/object_models/xml/expression.py b/pyffi/object_models/xml/expression.py index 6175dd1b4..9a30188dd 100644 --- a/pyffi/object_models/xml/expression.py +++ b/pyffi/object_models/xml/expression.py @@ -42,7 +42,7 @@ import re import sys # stderr (for debugging) - +import struct class Expression(object): """This class represents an expression. @@ -82,6 +82,7 @@ def __init__(self, expr_str, name_filter=None): try: left, self._op, right = self._partition(expr_str) + # print(left, self._op, right) self._left = self._parse(left, name_filter) self._right = self._parse(right, name_filter) except: @@ -116,7 +117,11 @@ def eval(self, data=None): if (not self._right) or self._right == '""': right = "" else: - right = getattr(data, self._right) + try: + right = getattr(data, self._right) + except: + print(self._right,"is not an attribute") + right = self._right elif isinstance(self._right, type): right = isinstance(data, self._right) elif self._right is None: @@ -124,7 +129,7 @@ def eval(self, data=None): else: assert (isinstance(self._right, int)) # debug right = self._right - + # print(self._left,self._op,right) if self._op == '==': return left == right elif self._op == '!=': @@ -192,6 +197,11 @@ def _parse(cls, expr_str, name_filter=None): # failed, so return the string, passed through the name filter except ValueError: pass + # it is a hex string, so return it as such + if expr_str.startswith("0x"): + hx = expr_str[2:] + return struct.pack(">i",int(hx,16)) + # return struct.unpack('!f', bytes.fromhex(expr_str[2:]))[0] m = re.match(r'^([0-9]+)\.([0-9]+)\.([0-9]+)\.([0-9]+)$', expr_str) if m: ver = ( diff --git a/pyffi/object_models/xml/struct_.py b/pyffi/object_models/xml/struct_.py index 300b1a868..66699caa9 100644 --- a/pyffi/object_models/xml/struct_.py +++ b/pyffi/object_models/xml/struct_.py @@ -240,9 +240,6 @@ def __init__(self, template = None, argument = None, parent = None): it is described here. :param parent: The parent of this instance, that is, the instance this array is an attribute of.""" - # used to track names of attributes that have already been added - # is faster than self.__dict__.has_key(...) - names = set() # initialize argument self.arg = argument # save parent (note: disabled for performance) @@ -253,12 +250,6 @@ def __init__(self, template = None, argument = None, parent = None): self._items = [] # initialize attributes for attr in self._attribute_list: - # skip attributes with dupiclate names - # (for this to work properly, duplicates must have the same - # type, template, argument, arr1, and arr2) - if attr.name in names: - continue - names.add(attr.name) # things that can only be determined at runtime (rt_xxx) rt_type = attr.type_ if attr.type_ != type(None) \ @@ -536,7 +527,7 @@ def _get_filtered_attribute_list(self, data=None): user_version = None names = set() for attr in self._attribute_list: - #print(attr.name, version, attr.ver1, attr.ver2) # debug + # print(attr.name, version, attr.ver1, attr.ver2) # debug # check version if version is not None: @@ -558,7 +549,9 @@ def _get_filtered_attribute_list(self, data=None): if (version is not None and user_version is not None and attr.vercond is not None): + # print("vercond") if not attr.vercond.eval(data): + # print("passed") continue #print("condition passed") # debug @@ -567,8 +560,9 @@ def _get_filtered_attribute_list(self, data=None): if attr.name in names: continue - #print("duplicate check passed") # debug + # print("duplicate check passed") # debug + # print(attr.name, version, attr.ver1, attr.ver2) # debug names.add(attr.name) # passed all tests # so yield the attribute From ad4e1075e43069caf73580a034981feed9a2d9f9 Mon Sep 17 00:00:00 2001 From: HENDRIX-ZT2 Date: Sat, 21 Sep 2019 19:04:48 +0200 Subject: [PATCH 11/22] Backward compatibility Works with old nif.xml --- pyffi/object_models/xml/__init__.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/pyffi/object_models/xml/__init__.py b/pyffi/object_models/xml/__init__.py index 755d4065c..540b85451 100644 --- a/pyffi/object_models/xml/__init__.py +++ b/pyffi/object_models/xml/__init__.py @@ -374,7 +374,7 @@ def read_basic(self, basic): # The link between basic types and C{self.cls} types is done via the name of the class. basic_class = getattr(self.cls, self.class_name) # check the class variables - is_template = (basic.attrib.get("generic") == "true") + is_template = self.is_generic(basic.attrib) if basic_class._is_template != is_template: raise XmlError( 'class %s should have _is_template = %s' % (self.class_name, is_template)) @@ -442,10 +442,9 @@ def read_struct(self, struct): raise XmlError( "typo, or forward declaration of struct %s" % class_basename) else: self.base_class = StructBase - # 'generic' attribute is optional - # if not set, then the struct is not a template # set attributes (see class StructBase) - self.class_dict["_is_template" ] = (attrs.get("generic") == "true") + # 'generic' attribute is optional- if not set, then the struct is not a template + self.class_dict["_is_template" ] = self.is_generic(attrs) self.class_dict["_attrs" ] = [] self.class_dict["_games" ] = {} for field in struct: @@ -516,6 +515,9 @@ def read_alias(self, alias): # the following are helper functions + def is_generic(self, attr): + # be backward compatible + return (attr.get("generic") == "true") or (attr.get("istemplate") == "1") def update_gamesdict(self, gamesdict, ver_text): if ver_text: From 497e1ee9a964e638dd9286ec54528a0a6d7be9ab Mon Sep 17 00:00:00 2001 From: HENDRIX-ZT2 Date: Sun, 22 Sep 2019 09:37:20 +0200 Subject: [PATCH 12/22] Rollback --- pyffi/formats/cgf/cgf.xml | 4 ++-- pyffi/formats/nif/__init__.py | 19 ++-------------- pyffi/object_models/common.py | 32 --------------------------- pyffi/object_models/xml/__init__.py | 25 +++++++++++---------- pyffi/object_models/xml/bit_struct.py | 4 +--- pyffi/object_models/xml/expression.py | 29 +++++------------------- pyffi/object_models/xml/struct_.py | 16 +++++++++----- 7 files changed, 35 insertions(+), 94 deletions(-) diff --git a/pyffi/formats/cgf/cgf.xml b/pyffi/formats/cgf/cgf.xml index acc5c652a..ea47ab9f1 100644 --- a/pyffi/formats/cgf/cgf.xml +++ b/pyffi/formats/cgf/cgf.xml @@ -46,11 +46,11 @@ A null-terminated string. - + A reference to another block. - + A back-reference to another block (up the hierarchy). diff --git a/pyffi/formats/nif/__init__.py b/pyffi/formats/nif/__init__.py index 3b89db405..849336c1e 100644 --- a/pyffi/formats/nif/__init__.py +++ b/pyffi/formats/nif/__init__.py @@ -400,8 +400,6 @@ class NifFormat(FileFormat): EPSILON = 0.0001 # basic types - uint64 = pyffi.object_models.common.UInt64 - int64 = pyffi.object_models.common.Int64 ulittle32 = pyffi.object_models.common.ULittle32 int = pyffi.object_models.common.Int uint = pyffi.object_models.common.UInt @@ -409,7 +407,6 @@ class NifFormat(FileFormat): char = pyffi.object_models.common.Char short = pyffi.object_models.common.Short ushort = pyffi.object_models.common.UShort - hfloat = pyffi.object_models.common.HFloat float = pyffi.object_models.common.Float BlockTypeIndex = pyffi.object_models.common.UShort StringIndex = pyffi.object_models.common.UInt @@ -423,12 +420,6 @@ def __init__(self, **kwargs): pyffi.object_models.common.Int.__init__(self, **kwargs) self.set_value(-1) - class NiFixedString(pyffi.object_models.common.Int): - """This is just an integer with -1 as default value.""" - def __init__(self, **kwargs): - pyffi.object_models.common.Int.__init__(self, **kwargs) - self.set_value(-1) - class bool(BasicBase, EditableBoolComboBox): """Basic implementation of a 32-bit (8-bit for versions > 4.0.0.2) boolean type. @@ -488,11 +479,6 @@ def write(self, stream, data): stream.write(struct.pack(data._byte_order + 'I', int(self._value))) - class TEMPLATE(pyffi.object_models.common.UShort): - """A dummy class""" - def __str__(self): - return "Template" - class Flags(pyffi.object_models.common.UShort): def __str__(self): return hex(self.get_value()) @@ -1309,7 +1295,7 @@ def read(self, stream): self.inspect_version_only(stream) logger.debug("Version 0x%08X" % self.version) self.header.read(stream, data=self) - # print(self.header) + # list of root blocks # for versions < 3.3.0.13 this list is updated through the # "Top Level Object" string while reading the blocks @@ -1395,12 +1381,11 @@ def read(self, stream): # read the block try: block.read(stream, self) - # print(block) except: logger.exception("Reading %s failed" % block.__class__) #logger.error("link stack: %s" % self._link_stack) #logger.error("block that failed:") - logger.error("%s" % block) + #logger.error("%s" % block) raise # complete NiDataStream data if block_type == "NiDataStream": diff --git a/pyffi/object_models/common.py b/pyffi/object_models/common.py index ac903ff16..9757a4072 100644 --- a/pyffi/object_models/common.py +++ b/pyffi/object_models/common.py @@ -414,38 +414,6 @@ def get_hash(self, data=None): """ return int(self.get_value()*200) -class HFloat(Float): - def read(self, stream, data): - """Read value from stream. - - :param stream: The stream to read from. - :type stream: file - """ - self._value = struct.unpack(data._byte_order + 'fe', - stream.read(2))[0] - - def write(self, stream, data): - """Write value to stream. - - :param stream: The stream to write to. - :type stream: file - """ - try: - stream.write(struct.pack(data._byte_order + 'e', - self._value)) - except OverflowError: - logger = logging.getLogger("pyffi.object_models") - logger.warn("float value overflow, writing zero") - stream.write(struct.pack(data._byte_order + 'e', - 0)) - - def get_size(self, data=None): - """Return number of bytes this type occupies in a file. - - :return: Number of bytes. - """ - return 2 - class ZString(BasicBase, EditableLineEdit): """String of variable length (null terminated). diff --git a/pyffi/object_models/xml/__init__.py b/pyffi/object_models/xml/__init__.py index 540b85451..abfbb2340 100644 --- a/pyffi/object_models/xml/__init__.py +++ b/pyffi/object_models/xml/__init__.py @@ -45,7 +45,6 @@ import os.path import sys import xml.sax -import xml.etree.ElementTree as ET import pyffi.object_models from pyffi.object_models.xml.struct_ import StructBase @@ -76,6 +75,7 @@ def __init__(cls, name, bases, dct): :param bases: The base classes, usually (object,). :param dct: A dictionary of class attributes, such as 'xml_file_name'. """ + super(MetaFileFormat, cls).__init__(name, bases, dct) # preparation: make deep copy of lists of enums, structs, etc. @@ -84,7 +84,10 @@ def __init__(cls, name, bases, dct): cls.xml_bit_struct = cls.xml_bit_struct[:] cls.xml_struct = cls.xml_struct[:] - # we check dct to avoid parsing the same file more than once in the hierarchy + # parse XML + + # we check dct to avoid parsing the same file more than once in + # the hierarchy xml_file_name = dct.get('xml_file_name') if xml_file_name: cls.logger.debug("Parsing %s and generating classes." % xml_file_name) @@ -100,7 +103,6 @@ def __init__(cls, name, bases, dct): cls.logger.debug("Parsing finished in %.3f seconds." % (time.time() - start)) - class FileFormat(pyffi.object_models.FileFormat, metaclass=MetaFileFormat): """This class can be used as a base class for file formats described by an xml file.""" @@ -172,10 +174,6 @@ class StructAttribute(object): there is no upper limit. """ - vercond = None - """The version condition of this member variable, as - :class:`Expression` or ``type(None)``. - """ userver = None """The user version this member exists, as ``int``, and ``None`` if it exists for all user versions. @@ -196,18 +194,19 @@ def __init__(self, cls, attrs): try: attrs_type_str = attrs["type"] except KeyError: - raise AttributeError("'%s' is missing a type attribute" % self.displayname) + raise AttributeError("'%s' is missing a type attribute" + % self.displayname) if attrs_type_str != "TEMPLATE": try: self.type_ = getattr(cls, attrs_type_str) except AttributeError: - # forward declaration, resolved at final_cleanup + # forward declaration, resolved at endDocument self.type_ = attrs_type_str else: self.type_ = type(None) # type determined at runtime # optional parameters self.default = attrs.get("default") - self.template = attrs.get("template") # resolved in final_cleanup + self.template = attrs.get("template") # resolved in endDocument self.arg = attrs.get("arg") self.arr1 = attrs.get("arr1") self.arr2 = attrs.get("arr2") @@ -216,8 +215,8 @@ def __init__(self, cls, attrs): self.ver1 = attrs.get("ver1") self.ver2 = attrs.get("ver2") self.userver = attrs.get("userver") - self.doc = "" # handled in xml parser - self.is_abstract = (attrs.get("abstract") in ("1", "true")) + self.doc = "" # handled in xml parser's characters function + self.is_abstract = (attrs.get("abstract") == "1") # post-processing if self.default: @@ -249,6 +248,7 @@ def __init__(self, cls, attrs): if self.ver2: self.ver2 = cls.version_number(self.ver2) + class BitStructAttribute(object): """Helper class to collect attribute data of bitstruct bits tags.""" @@ -288,6 +288,7 @@ class XmlError(Exception): pass + class XmlParser: struct_types = ("compound", "niobject", "struct") bitstruct_types = ("bitfield", "bitflags", "bitstruct") diff --git a/pyffi/object_models/xml/bit_struct.py b/pyffi/object_models/xml/bit_struct.py index 440991e8f..3f39d3244 100644 --- a/pyffi/object_models/xml/bit_struct.py +++ b/pyffi/object_models/xml/bit_struct.py @@ -68,10 +68,8 @@ def __init__(cls, name, bases, dct): cls._struct = 'H' elif cls._numbytes == 4: cls._struct = 'I' - elif cls._numbytes == 8: - cls._struct = 'II' else: - raise RuntimeError("unsupported bitstruct number of bytes: "+str(cls._numbytes)) + raise RuntimeError("unsupported bitstruct numbytes") # template type? cls._is_template = False diff --git a/pyffi/object_models/xml/expression.py b/pyffi/object_models/xml/expression.py index 9a30188dd..6813ae564 100644 --- a/pyffi/object_models/xml/expression.py +++ b/pyffi/object_models/xml/expression.py @@ -42,7 +42,7 @@ import re import sys # stderr (for debugging) -import struct + class Expression(object): """This class represents an expression. @@ -75,14 +75,12 @@ class Expression(object): False """ - operators = set(('==', '!=', '>=', '<=', '&&', '||', '&', '|', '-', '!', '>>', '<<', + operators = set(('==', '!=', '>=', '<=', '&&', '||', '&', '|', '-', '!', '<', '>', '/', '*', '+')) def __init__(self, expr_str, name_filter=None): - try: left, self._op, right = self._partition(expr_str) - # print(left, self._op, right) self._left = self._parse(left, name_filter) self._right = self._parse(right, name_filter) except: @@ -91,6 +89,7 @@ def __init__(self, expr_str, name_filter=None): def eval(self, data=None): """Evaluate the expression to an integer.""" + if isinstance(self._left, Expression): left = self._left.eval(data) elif isinstance(self._left, str): @@ -117,11 +116,7 @@ def eval(self, data=None): if (not self._right) or self._right == '""': right = "" else: - try: - right = getattr(data, self._right) - except: - print(self._right,"is not an attribute") - right = self._right + right = getattr(data, self._right) elif isinstance(self._right, type): right = isinstance(data, self._right) elif self._right is None: @@ -129,7 +124,7 @@ def eval(self, data=None): else: assert (isinstance(self._right, int)) # debug right = self._right - # print(self._left,self._op,right) + if self._op == '==': return left == right elif self._op == '!=': @@ -155,18 +150,11 @@ def eval(self, data=None): elif self._op == '<': return left < right elif self._op == '/': - if right > 0: - return left / right - else: - return 0 + return left / right elif self._op == '*': return left * right elif self._op == '+': return left + right - elif self._op == '<<': - return left << right - elif self._op == '>>': - return left >> right else: raise NotImplementedError("expression syntax error: operator '" + self._op + "' not implemented") @@ -197,11 +185,6 @@ def _parse(cls, expr_str, name_filter=None): # failed, so return the string, passed through the name filter except ValueError: pass - # it is a hex string, so return it as such - if expr_str.startswith("0x"): - hx = expr_str[2:] - return struct.pack(">i",int(hx,16)) - # return struct.unpack('!f', bytes.fromhex(expr_str[2:]))[0] m = re.match(r'^([0-9]+)\.([0-9]+)\.([0-9]+)\.([0-9]+)$', expr_str) if m: ver = ( diff --git a/pyffi/object_models/xml/struct_.py b/pyffi/object_models/xml/struct_.py index 66699caa9..300b1a868 100644 --- a/pyffi/object_models/xml/struct_.py +++ b/pyffi/object_models/xml/struct_.py @@ -240,6 +240,9 @@ def __init__(self, template = None, argument = None, parent = None): it is described here. :param parent: The parent of this instance, that is, the instance this array is an attribute of.""" + # used to track names of attributes that have already been added + # is faster than self.__dict__.has_key(...) + names = set() # initialize argument self.arg = argument # save parent (note: disabled for performance) @@ -250,6 +253,12 @@ def __init__(self, template = None, argument = None, parent = None): self._items = [] # initialize attributes for attr in self._attribute_list: + # skip attributes with dupiclate names + # (for this to work properly, duplicates must have the same + # type, template, argument, arr1, and arr2) + if attr.name in names: + continue + names.add(attr.name) # things that can only be determined at runtime (rt_xxx) rt_type = attr.type_ if attr.type_ != type(None) \ @@ -527,7 +536,7 @@ def _get_filtered_attribute_list(self, data=None): user_version = None names = set() for attr in self._attribute_list: - # print(attr.name, version, attr.ver1, attr.ver2) # debug + #print(attr.name, version, attr.ver1, attr.ver2) # debug # check version if version is not None: @@ -549,9 +558,7 @@ def _get_filtered_attribute_list(self, data=None): if (version is not None and user_version is not None and attr.vercond is not None): - # print("vercond") if not attr.vercond.eval(data): - # print("passed") continue #print("condition passed") # debug @@ -560,9 +567,8 @@ def _get_filtered_attribute_list(self, data=None): if attr.name in names: continue - # print("duplicate check passed") # debug + #print("duplicate check passed") # debug - # print(attr.name, version, attr.ver1, attr.ver2) # debug names.add(attr.name) # passed all tests # so yield the attribute From 488b9ff5f3b97dac43cd62aa5db00884ba33a863 Mon Sep 17 00:00:00 2001 From: HENDRIX-ZT2 Date: Sun, 22 Sep 2019 09:41:19 +0200 Subject: [PATCH 13/22] Forgot ET import --- pyffi/object_models/xml/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyffi/object_models/xml/__init__.py b/pyffi/object_models/xml/__init__.py index abfbb2340..8671410b4 100644 --- a/pyffi/object_models/xml/__init__.py +++ b/pyffi/object_models/xml/__init__.py @@ -44,7 +44,7 @@ import types import os.path import sys -import xml.sax +import xml.etree.ElementTree as ET import pyffi.object_models from pyffi.object_models.xml.struct_ import StructBase From 83c3ac50eb2fb0aefc10344d36b08b1876edf96c Mon Sep 17 00:00:00 2001 From: HENDRIX-ZT2 Date: Sun, 22 Sep 2019 10:58:48 +0200 Subject: [PATCH 14/22] More refactoring & cleanup --- pyffi/object_models/xml/__init__.py | 96 ++++++++--------------------- 1 file changed, 27 insertions(+), 69 deletions(-) diff --git a/pyffi/object_models/xml/__init__.py b/pyffi/object_models/xml/__init__.py index 8671410b4..404b4192a 100644 --- a/pyffi/object_models/xml/__init__.py +++ b/pyffi/object_models/xml/__init__.py @@ -94,7 +94,7 @@ def __init__(cls, name, bases, dct): # open XML file start = time.time() xml_file = cls.openfile(xml_file_name, cls.xml_file_path) - xmlp = XmlParser(cls, name, bases) + xmlp = XmlParser(cls) try: xmlp.load_xml(xml_file) finally: @@ -102,7 +102,6 @@ def __init__(cls, name, bases, dct): cls.logger.debug("Parsing finished in %.3f seconds." % (time.time() - start)) - class FileFormat(pyffi.object_models.FileFormat, metaclass=MetaFileFormat): """This class can be used as a base class for file formats described by an xml file.""" @@ -127,61 +126,6 @@ class FileFormat(pyffi.object_models.FileFormat, metaclass=MetaFileFormat): class StructAttribute(object): """Helper class to collect attribute data of struct add tags.""" - name = None - """The name of this member variable.""" - - type_ = None - """The type of this member variable (type is ``str`` for forward - declarations, and resolved to :class:`BasicBase` or - :class:`StructBase` later). - """ - - default = None - """The default value of this member variable.""" - - template = None - """The template type of this member variable (initially ``str``, - resolved to :class:`BasicBase` or :class:`StructBase` at the end - of the xml parsing), and if there is no template type, then this - variable will equal ``type(None)``. - """ - - arg = None - """The argument of this member variable.""" - - arr1 = None - """The first array size of this member variable, as - :class:`Expression` or ``type(None)``. - """ - - arr2 = None - """The second array size of this member variable, as - :class:`Expression` or ``type(None)``. - """ - - cond = None - """The condition of this member variable, as - :class:`Expression` or ``type(None)``. - """ - - ver1 = None - """The first version this member exists, as ``int``, and ``None`` if - there is no lower limit. - """ - - ver2 = None - """The last version this member exists, as ``int``, and ``None`` if - there is no upper limit. - """ - - userver = None - """The user version this member exists, as ``int``, and ``None`` if - it exists for all user versions. - """ - - is_abstract = False - """Whether the attribute is abstract or not (read and written).""" - def __init__(self, cls, attrs): """Initialize attribute from the xml attrs dictionary of an add tag. @@ -189,34 +133,52 @@ def __init__(self, cls, attrs): :param cls: The class where all types reside. :param attrs: The xml add tag attribute dictionary.""" # mandatory parameters + + # The name of this member variable. self.displayname = attrs["name"] self.name = cls.name_attribute(self.displayname) + # The type of this member variable (type is ``str`` for forward + # declarations, and resolved to :class:`BasicBase` or :class:`StructBase` later). try: attrs_type_str = attrs["type"] except KeyError: - raise AttributeError("'%s' is missing a type attribute" - % self.displayname) + raise AttributeError("'%s' is missing a type attribute" % self.displayname) if attrs_type_str != "TEMPLATE": try: self.type_ = getattr(cls, attrs_type_str) except AttributeError: - # forward declaration, resolved at endDocument + # forward declaration, resolved in final_cleanup() self.type_ = attrs_type_str else: - self.type_ = type(None) # type determined at runtime + # type determined at runtime + self.type_ = type(None) # optional parameters + + # default value of this member variable. self.default = attrs.get("default") - self.template = attrs.get("template") # resolved in endDocument + # template type of this member variable (initially ``str``, resolved in final_cleanup() to :class:`BasicBase` or :class:`StructBase` at the end + # of the xml parsing), and if there is no template type, then this variable will equal `None`. + self.template = attrs.get("template") + # argument of this member variable. self.arg = attrs.get("arg") + # first array size of this member variable, as :class:`Expression` or `None`. self.arr1 = attrs.get("arr1") + # second array size of this member variable, as :class:`Expression` or `None`. self.arr2 = attrs.get("arr2") + # condition of this member variable, as :class:`Expression` or `None`. self.cond = attrs.get("cond") + # version condition for this member variable self.vercond = attrs.get("vercond") + # first version this member exists, as `int`, and `None` if there is no lower limit. self.ver1 = attrs.get("ver1") + # last version this member exists, as `int`, and `None` if there is no upper limit. self.ver2 = attrs.get("ver2") + # user version this member exists, as `int`, and `None` if it exists for all user versions. self.userver = attrs.get("userver") - self.doc = "" # handled in xml parser's characters function - self.is_abstract = (attrs.get("abstract") == "1") + # docstring is handled in xml parser's characters function + self.doc = "" + # Whether the attribute is abstract or not (read and written). + self.is_abstract = (attrs.get("abstract") in ("1", "true")) # post-processing if self.default: @@ -248,7 +210,6 @@ def __init__(self, cls, attrs): if self.ver2: self.ver2 = cls.version_number(self.ver2) - class BitStructAttribute(object): """Helper class to collect attribute data of bitstruct bits tags.""" @@ -281,18 +242,15 @@ def __init__(self, cls, attrs): if self.ver2: self.ver2 = cls.version_number(self.ver2) - class XmlError(Exception): """The XML handler will throw this exception if something goes wrong while parsing.""" pass - - class XmlParser: struct_types = ("compound", "niobject", "struct") bitstruct_types = ("bitfield", "bitflags", "bitstruct") - def __init__(self, cls, name, bases): + def __init__(self, cls): """Set up the xml parser.""" # initialize dictionaries From 1f2fef7c26fffc2a99115b5121710fd17c070005 Mon Sep 17 00:00:00 2001 From: HENDRIX-ZT2 Date: Sun, 22 Sep 2019 12:18:07 +0200 Subject: [PATCH 15/22] Fix old typo --- pyffi/object_models/xml/__init__.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/pyffi/object_models/xml/__init__.py b/pyffi/object_models/xml/__init__.py index 404b4192a..69a2d6583 100644 --- a/pyffi/object_models/xml/__init__.py +++ b/pyffi/object_models/xml/__init__.py @@ -520,9 +520,7 @@ def create_class(self, tag): if issubclass(cls_klass, pyffi.object_models.FileFormat.Data): self.cls.Data = cls_klass # for the stuff below - gen_class = cls_klass - # I think the above is a typo, should be: - # gen_klass = cls_klass + gen_klass = cls_klass else: # does not yet exist: create it and assign to class dict gen_klass = type(self.class_name, (self.base_class,), self.class_dict) From 0059f8b48230e6ad1ab4164cc5810a40bacdc6e1 Mon Sep 17 00:00:00 2001 From: HENDRIX-ZT2 Date: Wed, 2 Oct 2019 23:49:40 +0200 Subject: [PATCH 16/22] Prototype reading with new nif.xml - arg is now an expression - add HFloat, NiFixedString - inspect header and store bs_header for global access - extend Expressions --- pyffi/formats/nif/__init__.py | 12 +++- pyffi/object_models/common.py | 29 +++++++++ pyffi/object_models/xml/__init__.py | 10 +-- pyffi/object_models/xml/bit_struct.py | 2 + pyffi/object_models/xml/expression.py | 24 +++++++- pyffi/object_models/xml/struct_.py | 89 +++++++++++++++------------ 6 files changed, 118 insertions(+), 48 deletions(-) diff --git a/pyffi/formats/nif/__init__.py b/pyffi/formats/nif/__init__.py index 849336c1e..8e2f9d2e7 100644 --- a/pyffi/formats/nif/__init__.py +++ b/pyffi/formats/nif/__init__.py @@ -400,6 +400,8 @@ class NifFormat(FileFormat): EPSILON = 0.0001 # basic types + uint64 = pyffi.object_models.common.UInt64 + int64 = pyffi.object_models.common.Int64 ulittle32 = pyffi.object_models.common.ULittle32 int = pyffi.object_models.common.Int uint = pyffi.object_models.common.UInt @@ -408,6 +410,7 @@ class NifFormat(FileFormat): short = pyffi.object_models.common.Short ushort = pyffi.object_models.common.UShort float = pyffi.object_models.common.Float + hfloat = pyffi.object_models.common.HFloat BlockTypeIndex = pyffi.object_models.common.UShort StringIndex = pyffi.object_models.common.UInt SizedString = pyffi.object_models.common.SizedString @@ -420,6 +423,12 @@ def __init__(self, **kwargs): pyffi.object_models.common.Int.__init__(self, **kwargs) self.set_value(-1) + class NiFixedString(pyffi.object_models.common.Int): + """This is just an integer with -1 as default value.""" + def __init__(self, **kwargs): + pyffi.object_models.common.Int.__init__(self, **kwargs) + self.set_value(-1) + class bool(BasicBase, EditableBoolComboBox): """Basic implementation of a 32-bit (8-bit for versions > 4.0.0.2) boolean type. @@ -1280,6 +1289,7 @@ def inspect(self, stream): try: self.inspect_version_only(stream) self.header.read(stream, data=self) + self.bs_header = self.header.bs_header finally: stream.seek(pos) @@ -1292,7 +1302,7 @@ def read(self, stream): logger = logging.getLogger("pyffi.nif.data") # read header logger.debug("Reading header at 0x%08X" % stream.tell()) - self.inspect_version_only(stream) + self.inspect(stream) logger.debug("Version 0x%08X" % self.version) self.header.read(stream, data=self) diff --git a/pyffi/object_models/common.py b/pyffi/object_models/common.py index 9757a4072..e53b7b812 100644 --- a/pyffi/object_models/common.py +++ b/pyffi/object_models/common.py @@ -414,6 +414,35 @@ def get_hash(self, data=None): """ return int(self.get_value()*200) +class HFloat(Float): + def read(self, stream, data): + """Read value from stream. + :param stream: The stream to read from. + :type stream: file + """ + self._value = struct.unpack(data._byte_order + 'e', + stream.read(2))[0] + + def write(self, stream, data): + """Write value to stream. + :param stream: The stream to write to. + :type stream: file + """ + try: + stream.write(struct.pack(data._byte_order + 'e', + self._value)) + except OverflowError: + logger = logging.getLogger("pyffi.object_models") + logger.warn("float value overflow, writing zero") + stream.write(struct.pack(data._byte_order + 'e', + 0)) + + def get_size(self, data=None): + """Return number of bytes this type occupies in a file. + :return: Number of bytes. + """ + return 2 + class ZString(BasicBase, EditableLineEdit): """String of variable length (null terminated). diff --git a/pyffi/object_models/xml/__init__.py b/pyffi/object_models/xml/__init__.py index 69a2d6583..0a5eb383a 100644 --- a/pyffi/object_models/xml/__init__.py +++ b/pyffi/object_models/xml/__init__.py @@ -199,10 +199,11 @@ def __init__(self, cls, attrs): if self.vercond: self.vercond = Expression(self.vercond, cls.name_attribute) if self.arg: - try: - self.arg = int(self.arg) - except ValueError: - self.arg = cls.name_attribute(self.arg) + # try: + # self.arg = int(self.arg) + # except ValueError: + # self.arg = cls.name_attribute(self.arg) + self.arg = Expression(self.arg, cls.name_attribute) if self.userver: self.userver = int(self.userver) if self.ver1: @@ -576,6 +577,7 @@ def final_cleanup(self): continue # fix templates for attr in obj._attrs: + # print(attr.name) templ = attr.template if isinstance(templ, str): attr.template = getattr(self.cls, templ) if templ != "TEMPLATE" else type(None) diff --git a/pyffi/object_models/xml/bit_struct.py b/pyffi/object_models/xml/bit_struct.py index 3f39d3244..4720dbc41 100644 --- a/pyffi/object_models/xml/bit_struct.py +++ b/pyffi/object_models/xml/bit_struct.py @@ -68,6 +68,8 @@ def __init__(cls, name, bases, dct): cls._struct = 'H' elif cls._numbytes == 4: cls._struct = 'I' + elif cls._numbytes == 8: + cls._struct = 'Q' else: raise RuntimeError("unsupported bitstruct numbytes") diff --git a/pyffi/object_models/xml/expression.py b/pyffi/object_models/xml/expression.py index 6813ae564..5b24cb542 100644 --- a/pyffi/object_models/xml/expression.py +++ b/pyffi/object_models/xml/expression.py @@ -43,6 +43,7 @@ import re import sys # stderr (for debugging) +from pyffi.object_models.xml.bit_struct import BitStructBase class Expression(object): """This class represents an expression. @@ -75,7 +76,7 @@ class Expression(object): False """ - operators = set(('==', '!=', '>=', '<=', '&&', '||', '&', '|', '-', '!', + operators = set(('==', '!=', '>=', '<=', '&&', '||', '&', '|', '-', '!', '>>', '<<', '<', '>', '/', '*', '+')) def __init__(self, expr_str, name_filter=None): @@ -124,7 +125,8 @@ def eval(self, data=None): else: assert (isinstance(self._right, int)) # debug right = self._right - + + if self._op == '==': return left == right elif self._op == '!=': @@ -150,11 +152,29 @@ def eval(self, data=None): elif self._op == '<': return left < right elif self._op == '/': + if right > 0: + return left / right + else: + return 0 return left / right elif self._op == '*': return left * right elif self._op == '+': return left + right + elif self._op == '<<': + # get underlying value for bitstructs + if isinstance(left, (BitStructBase,) ): + left = left.get_attributes_values(None) + if isinstance(right, (BitStructBase,) ): + right = right.get_attributes_values(None) + return left << right + elif self._op == '>>': + # get underlying value for bitstructs + if isinstance(left, (BitStructBase,) ): + left = left.get_attributes_values(None) + if isinstance(right, (BitStructBase,) ): + right = right.get_attributes_values(None) + return left >> right else: raise NotImplementedError("expression syntax error: operator '" + self._op + "' not implemented") diff --git a/pyffi/object_models/xml/struct_.py b/pyffi/object_models/xml/struct_.py index 300b1a868..ad559e272 100644 --- a/pyffi/object_models/xml/struct_.py +++ b/pyffi/object_models/xml/struct_.py @@ -129,10 +129,33 @@ def __init__(cls, name, bases, dct): # precalculate the attribute list # profiling shows that this speeds up most of the StructBase methods # that rely on parsing the attribute list + + # this is a non-unique list of all StructAttributes of this class and its ancestors + # may continue more than one StructAttribute of the same name cls._attribute_list = cls._get_attribute_list() # precalculate the attribute name list - cls._names = cls._get_names() + # list of unique names in the order they must be read + cls._names = [] + + # per field name, list of all StructAttributes that may apply + cls._attribute_unions = {} + + # populate unions and field names list + for attr in cls._attribute_list: + # we encounter a new field name so create a new union list for it, and add it to field names + if attr.name not in cls._attribute_unions: + cls._attribute_unions[attr.name] = [] + cls._names.append(attr.name) + # add this StructAttribute to union + cls._attribute_unions[attr.name].append(attr) + + # # debug + # if name == "NiNode": + # print(cls._attribute_list ) + # print(cls._attribute_unions) + # for name in cls._names: + # print(name, len(cls._attribute_unions[name]) ) def __repr__(cls): return ""%(cls.__name__) @@ -242,7 +265,7 @@ def __init__(self, template = None, argument = None, parent = None): array is an attribute of.""" # used to track names of attributes that have already been added # is faster than self.__dict__.has_key(...) - names = set() + # names = set() # initialize argument self.arg = argument # save parent (note: disabled for performance) @@ -251,22 +274,20 @@ def __init__(self, template = None, argument = None, parent = None): # this list is used for instance by qskope to display the structure # in a tree view self._items = [] + # initialize attributes + # _attribute_list is a list of StructAttributes, including unions of the same name for attr in self._attribute_list: # skip attributes with dupiclate names # (for this to work properly, duplicates must have the same # type, template, argument, arr1, and arr2) - if attr.name in names: - continue - names.add(attr.name) - + # if attr.name in names: + # continue + # names.add(attr.name) # things that can only be determined at runtime (rt_xxx) - rt_type = attr.type_ if attr.type_ != type(None) \ - else template - rt_template = attr.template if attr.template != type(None) \ - else template - rt_arg = attr.arg if isinstance(attr.arg, (int, type(None))) \ - else getattr(self, attr.arg) + rt_type = attr.type_ if attr.type_ != type(None) else template + rt_template = attr.template if attr.template != type(None) else template + rt_arg = attr.arg # instantiate the class, handling arrays at the same time if attr.arr1 == None: @@ -295,7 +316,7 @@ def __init__(self, template = None, argument = None, parent = None): # add instance to item list self._items.append(attr_instance) - + def deepcopy(self, block): """Copy attributes from a given block (one block class must be a subclass of the other). Returns self.""" @@ -361,12 +382,10 @@ def read(self, stream, data): # skip abstract attributes if attr.is_abstract: continue - # get attribute argument (can only be done at runtime) - rt_arg = attr.arg if isinstance(attr.arg, (int, type(None))) \ - else getattr(self, attr.arg) # read the attribute attr_value = getattr(self, "_%s_value_" % attr.name) - attr_value.arg = rt_arg + # get attribute argument (can only be done at runtime) + attr_value.arg = attr.arg # if hasattr(attr, "type_"): # attr_value._elementType = attr.type_ self._log_struct(stream, attr) @@ -380,13 +399,11 @@ def write(self, stream, data): # skip abstract attributes if attr.is_abstract: continue - # get attribute argument (can only be done at runtime) - rt_arg = attr.arg if isinstance(attr.arg, (int, type(None))) \ - else getattr(self, attr.arg) # write the attribute attr_value = getattr(self, "_%s_value_" % attr.name) - attr_value.arg = rt_arg - getattr(self, "_%s_value_" % attr.name).write(stream, data) + # get attribute argument (can only be done at runtime) + attr_value.arg = attr.arg + attr_value.write(stream, data) self._log_struct(stream, attr) def fix_links(self, data): @@ -488,7 +505,7 @@ def get_versions(cls, game): @classmethod def _get_attribute_list(cls): - """Calculate the list of all attributes of this structure.""" + """Calculate the list of all StructAttributes of this structure, including duplicates.""" # string of attributes of base classes of cls attrs = [] for base in cls.__bases__: @@ -496,26 +513,12 @@ def _get_attribute_list(cls): attrs.extend(base._get_attribute_list()) except AttributeError: # when base class is "object" pass - attrs.extend(cls._attrs) - return attrs - - @classmethod - def _get_names(cls): - """Calculate the list of all attributes names in this structure. - Skips duplicate names.""" - # string of attributes of base classes of cls - names = [] - for base in cls.__bases__: - try: - names.extend(base._get_names()) - except AttributeError: # when base class is "object" - pass for attr in cls._attrs: - if attr.name in names: + if attr in attrs: continue else: - names.append(attr.name) - return names + attrs.append(attr) + return attrs def _get_filtered_attribute_list(self, data=None): """Generator for listing all 'active' attributes, that is, @@ -535,7 +538,7 @@ def _get_filtered_attribute_list(self, data=None): version = None user_version = None names = set() - for attr in self._attribute_list: + for attr, attr_instance in zip(self._attribute_list, self._items): #print(attr.name, version, attr.ver1, attr.ver2) # debug # check version @@ -570,6 +573,10 @@ def _get_filtered_attribute_list(self, data=None): #print("duplicate check passed") # debug names.add(attr.name) + + # assign attribute value + setattr(self, "_%s_value_" % attr.name, attr_instance) + # passed all tests # so yield the attribute yield attr From c2affa9d3a3b4d06335e9bdfc2c4114033a27f31 Mon Sep 17 00:00:00 2001 From: HENDRIX-ZT2 Date: Thu, 3 Oct 2019 14:13:34 +0200 Subject: [PATCH 17/22] Ignore missing bs_header For compatibility with older nif.xml --- pyffi/formats/nif/__init__.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pyffi/formats/nif/__init__.py b/pyffi/formats/nif/__init__.py index 8e2f9d2e7..75f3a511d 100644 --- a/pyffi/formats/nif/__init__.py +++ b/pyffi/formats/nif/__init__.py @@ -1289,7 +1289,11 @@ def inspect(self, stream): try: self.inspect_version_only(stream) self.header.read(stream, data=self) - self.bs_header = self.header.bs_header + # for compatibility during nif xml version transition + try: + self.bs_header = self.header.bs_header + except: + self.bs_header = None finally: stream.seek(pos) From bf3fcf33ca76a83bc50d8566715b1132d0d89604 Mon Sep 17 00:00:00 2001 From: HENDRIX-ZT2 Date: Thu, 3 Oct 2019 23:07:00 +0200 Subject: [PATCH 18/22] Cleanup - refactor getter & setter methods - comment "setattr(self, "_%s_value_" % attr.name, attr_instance)" in _get_filtered_attribute_list() because although it helps for reading with the new nif xml, it breaks writing with the old one --- pyffi/object_models/xml/struct_.py | 50 ++++++++++++++++++++---------- 1 file changed, 33 insertions(+), 17 deletions(-) diff --git a/pyffi/object_models/xml/struct_.py b/pyffi/object_models/xml/struct_.py index ad559e272..7526a8138 100644 --- a/pyffi/object_models/xml/struct_.py +++ b/pyffi/object_models/xml/struct_.py @@ -61,6 +61,7 @@ def __init__(cls, name, bases, dct): cls._has_refs = getattr(cls, '_has_refs', False) # does the type contain a string? cls._has_strings = getattr(cls, '_has_strings', False) + # attr is a StructAttribute for attr in dct.get('_attrs', []): # basestring is a forward compound type declaration # and issubclass must take a type as first argument @@ -68,28 +69,23 @@ def __init__(cls, name, bases, dct): if not isinstance(attr.type_, str) and \ issubclass(attr.type_, BasicBase) and attr.arr1 is None: # get and set basic attributes - setattr(cls, attr.name, property( - partial(StructBase.get_basic_attribute, name=attr.name), - partial(StructBase.set_basic_attribute, name=attr.name), - doc=attr.doc)) + fget = partial(StructBase.get_basic_attribute, name=attr.name) + fset = partial(StructBase.set_basic_attribute, name=attr.name) elif not isinstance(attr.type_, str) and \ issubclass(attr.type_, StructBase) and attr.arr1 is None: # get and set struct attributes - setattr(cls, attr.name, property( - partial(StructBase.get_attribute, name=attr.name), - partial(StructBase.set_attribute, name=attr.name), - doc=attr.doc)) + fget = partial(StructBase.get_attribute, name=attr.name) + fset = partial(StructBase.set_attribute, name=attr.name) elif attr.type_ == type(None) and attr.arr1 is None: # get and set template attributes - setattr(cls, attr.name, property( - partial(StructBase.get_template_attribute, name=attr.name), - partial(StructBase.set_template_attribute, name=attr.name), - doc=attr.doc)) + fget = partial(StructBase.get_template_attribute, name=attr.name) + fset = partial(StructBase.set_template_attribute, name=attr.name) else: # other types of attributes: get only - setattr(cls, attr.name, property( - partial(StructBase.get_attribute, name=attr.name), - doc=attr.doc)) + fget = partial(StructBase.get_attribute, name=attr.name) + fset = None + # add getter & setter functions for attr to class + setattr(cls, attr.name, property(fget, fset, doc=attr.doc)) # check for links and refs and strings if not cls._has_links: @@ -290,12 +286,14 @@ def __init__(self, template = None, argument = None, parent = None): rt_arg = attr.arg # instantiate the class, handling arrays at the same time + # create a single attribute, no array if attr.arr1 == None: attr_instance = rt_type( template = rt_template, argument = rt_arg, parent = self) if attr.default != None: attr_instance.set_value(attr.default) + # create a 1D array elif attr.arr2 == None: attr_instance = Array( element_type = rt_type, @@ -303,6 +301,7 @@ def __init__(self, template = None, argument = None, parent = None): element_type_argument = rt_arg, count1 = attr.arr1, parent = self) + # create a 2D array else: attr_instance = Array( element_type = rt_type, @@ -316,6 +315,17 @@ def __init__(self, template = None, argument = None, parent = None): # add instance to item list self._items.append(attr_instance) + + # per field name, list of all instances that apply + self._attr_instance_unions = {} + + # populate attribute instance unions list + for attr, attr_instance in zip(self._attribute_list, self._items): + # we encounter a new field name so create a new union list for it + if attr.name not in self._attr_instance_unions: + self._attr_instance_unions[attr.name] = [] + # add this instance to union + self._attr_instance_unions[attr.name].append(attr_instance) def deepcopy(self, block): """Copy attributes from a given block (one block class must be a @@ -575,7 +585,7 @@ def _get_filtered_attribute_list(self, data=None): names.add(attr.name) # assign attribute value - setattr(self, "_%s_value_" % attr.name, attr_instance) + # setattr(self, "_%s_value_" % attr.name, attr_instance) # passed all tests # so yield the attribute @@ -597,6 +607,8 @@ def set_attribute(self, value, name): value.__class__.__name__)) # set it setattr(self, "_" + name + "_value_", value) + # for attr in self._attr_instance_unions[name]: + # attr = value def get_basic_attribute(self, name): """Get a basic attribute.""" @@ -606,7 +618,11 @@ def get_basic_attribute(self, name): # name argument must be last def set_basic_attribute(self, value, name): """Set the value of a basic attribute.""" - getattr(self, "_" + name + "_value_").set_value(value) + att = getattr(self, "_" + name + "_value_") + att.set_value(value) + + # for attr in self._attr_instance_unions[name]: + # attr.set_value(value) def get_template_attribute(self, name): """Get a template attribute.""" From 86d2a9f694267b1879c044e20e0a1e48fe438e10 Mon Sep 17 00:00:00 2001 From: HENDRIX-ZT2 Date: Thu, 21 Nov 2019 13:43:37 +0100 Subject: [PATCH 19/22] Revert "Cleanup" This reverts commit bf3fcf33ca76a83bc50d8566715b1132d0d89604. --- pyffi/object_models/xml/struct_.py | 50 ++++++++++-------------------- 1 file changed, 17 insertions(+), 33 deletions(-) diff --git a/pyffi/object_models/xml/struct_.py b/pyffi/object_models/xml/struct_.py index 7526a8138..ad559e272 100644 --- a/pyffi/object_models/xml/struct_.py +++ b/pyffi/object_models/xml/struct_.py @@ -61,7 +61,6 @@ def __init__(cls, name, bases, dct): cls._has_refs = getattr(cls, '_has_refs', False) # does the type contain a string? cls._has_strings = getattr(cls, '_has_strings', False) - # attr is a StructAttribute for attr in dct.get('_attrs', []): # basestring is a forward compound type declaration # and issubclass must take a type as first argument @@ -69,23 +68,28 @@ def __init__(cls, name, bases, dct): if not isinstance(attr.type_, str) and \ issubclass(attr.type_, BasicBase) and attr.arr1 is None: # get and set basic attributes - fget = partial(StructBase.get_basic_attribute, name=attr.name) - fset = partial(StructBase.set_basic_attribute, name=attr.name) + setattr(cls, attr.name, property( + partial(StructBase.get_basic_attribute, name=attr.name), + partial(StructBase.set_basic_attribute, name=attr.name), + doc=attr.doc)) elif not isinstance(attr.type_, str) and \ issubclass(attr.type_, StructBase) and attr.arr1 is None: # get and set struct attributes - fget = partial(StructBase.get_attribute, name=attr.name) - fset = partial(StructBase.set_attribute, name=attr.name) + setattr(cls, attr.name, property( + partial(StructBase.get_attribute, name=attr.name), + partial(StructBase.set_attribute, name=attr.name), + doc=attr.doc)) elif attr.type_ == type(None) and attr.arr1 is None: # get and set template attributes - fget = partial(StructBase.get_template_attribute, name=attr.name) - fset = partial(StructBase.set_template_attribute, name=attr.name) + setattr(cls, attr.name, property( + partial(StructBase.get_template_attribute, name=attr.name), + partial(StructBase.set_template_attribute, name=attr.name), + doc=attr.doc)) else: # other types of attributes: get only - fget = partial(StructBase.get_attribute, name=attr.name) - fset = None - # add getter & setter functions for attr to class - setattr(cls, attr.name, property(fget, fset, doc=attr.doc)) + setattr(cls, attr.name, property( + partial(StructBase.get_attribute, name=attr.name), + doc=attr.doc)) # check for links and refs and strings if not cls._has_links: @@ -286,14 +290,12 @@ def __init__(self, template = None, argument = None, parent = None): rt_arg = attr.arg # instantiate the class, handling arrays at the same time - # create a single attribute, no array if attr.arr1 == None: attr_instance = rt_type( template = rt_template, argument = rt_arg, parent = self) if attr.default != None: attr_instance.set_value(attr.default) - # create a 1D array elif attr.arr2 == None: attr_instance = Array( element_type = rt_type, @@ -301,7 +303,6 @@ def __init__(self, template = None, argument = None, parent = None): element_type_argument = rt_arg, count1 = attr.arr1, parent = self) - # create a 2D array else: attr_instance = Array( element_type = rt_type, @@ -315,17 +316,6 @@ def __init__(self, template = None, argument = None, parent = None): # add instance to item list self._items.append(attr_instance) - - # per field name, list of all instances that apply - self._attr_instance_unions = {} - - # populate attribute instance unions list - for attr, attr_instance in zip(self._attribute_list, self._items): - # we encounter a new field name so create a new union list for it - if attr.name not in self._attr_instance_unions: - self._attr_instance_unions[attr.name] = [] - # add this instance to union - self._attr_instance_unions[attr.name].append(attr_instance) def deepcopy(self, block): """Copy attributes from a given block (one block class must be a @@ -585,7 +575,7 @@ def _get_filtered_attribute_list(self, data=None): names.add(attr.name) # assign attribute value - # setattr(self, "_%s_value_" % attr.name, attr_instance) + setattr(self, "_%s_value_" % attr.name, attr_instance) # passed all tests # so yield the attribute @@ -607,8 +597,6 @@ def set_attribute(self, value, name): value.__class__.__name__)) # set it setattr(self, "_" + name + "_value_", value) - # for attr in self._attr_instance_unions[name]: - # attr = value def get_basic_attribute(self, name): """Get a basic attribute.""" @@ -618,11 +606,7 @@ def get_basic_attribute(self, name): # name argument must be last def set_basic_attribute(self, value, name): """Set the value of a basic attribute.""" - att = getattr(self, "_" + name + "_value_") - att.set_value(value) - - # for attr in self._attr_instance_unions[name]: - # attr.set_value(value) + getattr(self, "_" + name + "_value_").set_value(value) def get_template_attribute(self, name): """Get a template attribute.""" From ef779931f920ba5c42e95cbc7d1151c34c790b5e Mon Sep 17 00:00:00 2001 From: HENDRIX-ZT2 Date: Thu, 21 Nov 2019 13:43:45 +0100 Subject: [PATCH 20/22] Revert "Ignore missing bs_header" This reverts commit c2affa9d3a3b4d06335e9bdfc2c4114033a27f31. --- pyffi/formats/nif/__init__.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/pyffi/formats/nif/__init__.py b/pyffi/formats/nif/__init__.py index 75f3a511d..8e2f9d2e7 100644 --- a/pyffi/formats/nif/__init__.py +++ b/pyffi/formats/nif/__init__.py @@ -1289,11 +1289,7 @@ def inspect(self, stream): try: self.inspect_version_only(stream) self.header.read(stream, data=self) - # for compatibility during nif xml version transition - try: - self.bs_header = self.header.bs_header - except: - self.bs_header = None + self.bs_header = self.header.bs_header finally: stream.seek(pos) From 7967fd63236d2a85b0d7737587c4a5617c383ee2 Mon Sep 17 00:00:00 2001 From: HENDRIX-ZT2 Date: Thu, 21 Nov 2019 13:43:57 +0100 Subject: [PATCH 21/22] Revert "Prototype reading with new nif.xml" This reverts commit 0059f8b48230e6ad1ab4164cc5810a40bacdc6e1. --- pyffi/formats/nif/__init__.py | 12 +--- pyffi/object_models/common.py | 29 --------- pyffi/object_models/xml/__init__.py | 10 ++- pyffi/object_models/xml/bit_struct.py | 2 - pyffi/object_models/xml/expression.py | 24 +------- pyffi/object_models/xml/struct_.py | 89 ++++++++++++--------------- 6 files changed, 48 insertions(+), 118 deletions(-) diff --git a/pyffi/formats/nif/__init__.py b/pyffi/formats/nif/__init__.py index 8e2f9d2e7..849336c1e 100644 --- a/pyffi/formats/nif/__init__.py +++ b/pyffi/formats/nif/__init__.py @@ -400,8 +400,6 @@ class NifFormat(FileFormat): EPSILON = 0.0001 # basic types - uint64 = pyffi.object_models.common.UInt64 - int64 = pyffi.object_models.common.Int64 ulittle32 = pyffi.object_models.common.ULittle32 int = pyffi.object_models.common.Int uint = pyffi.object_models.common.UInt @@ -410,7 +408,6 @@ class NifFormat(FileFormat): short = pyffi.object_models.common.Short ushort = pyffi.object_models.common.UShort float = pyffi.object_models.common.Float - hfloat = pyffi.object_models.common.HFloat BlockTypeIndex = pyffi.object_models.common.UShort StringIndex = pyffi.object_models.common.UInt SizedString = pyffi.object_models.common.SizedString @@ -423,12 +420,6 @@ def __init__(self, **kwargs): pyffi.object_models.common.Int.__init__(self, **kwargs) self.set_value(-1) - class NiFixedString(pyffi.object_models.common.Int): - """This is just an integer with -1 as default value.""" - def __init__(self, **kwargs): - pyffi.object_models.common.Int.__init__(self, **kwargs) - self.set_value(-1) - class bool(BasicBase, EditableBoolComboBox): """Basic implementation of a 32-bit (8-bit for versions > 4.0.0.2) boolean type. @@ -1289,7 +1280,6 @@ def inspect(self, stream): try: self.inspect_version_only(stream) self.header.read(stream, data=self) - self.bs_header = self.header.bs_header finally: stream.seek(pos) @@ -1302,7 +1292,7 @@ def read(self, stream): logger = logging.getLogger("pyffi.nif.data") # read header logger.debug("Reading header at 0x%08X" % stream.tell()) - self.inspect(stream) + self.inspect_version_only(stream) logger.debug("Version 0x%08X" % self.version) self.header.read(stream, data=self) diff --git a/pyffi/object_models/common.py b/pyffi/object_models/common.py index e53b7b812..9757a4072 100644 --- a/pyffi/object_models/common.py +++ b/pyffi/object_models/common.py @@ -414,35 +414,6 @@ def get_hash(self, data=None): """ return int(self.get_value()*200) -class HFloat(Float): - def read(self, stream, data): - """Read value from stream. - :param stream: The stream to read from. - :type stream: file - """ - self._value = struct.unpack(data._byte_order + 'e', - stream.read(2))[0] - - def write(self, stream, data): - """Write value to stream. - :param stream: The stream to write to. - :type stream: file - """ - try: - stream.write(struct.pack(data._byte_order + 'e', - self._value)) - except OverflowError: - logger = logging.getLogger("pyffi.object_models") - logger.warn("float value overflow, writing zero") - stream.write(struct.pack(data._byte_order + 'e', - 0)) - - def get_size(self, data=None): - """Return number of bytes this type occupies in a file. - :return: Number of bytes. - """ - return 2 - class ZString(BasicBase, EditableLineEdit): """String of variable length (null terminated). diff --git a/pyffi/object_models/xml/__init__.py b/pyffi/object_models/xml/__init__.py index 0a5eb383a..69a2d6583 100644 --- a/pyffi/object_models/xml/__init__.py +++ b/pyffi/object_models/xml/__init__.py @@ -199,11 +199,10 @@ def __init__(self, cls, attrs): if self.vercond: self.vercond = Expression(self.vercond, cls.name_attribute) if self.arg: - # try: - # self.arg = int(self.arg) - # except ValueError: - # self.arg = cls.name_attribute(self.arg) - self.arg = Expression(self.arg, cls.name_attribute) + try: + self.arg = int(self.arg) + except ValueError: + self.arg = cls.name_attribute(self.arg) if self.userver: self.userver = int(self.userver) if self.ver1: @@ -577,7 +576,6 @@ def final_cleanup(self): continue # fix templates for attr in obj._attrs: - # print(attr.name) templ = attr.template if isinstance(templ, str): attr.template = getattr(self.cls, templ) if templ != "TEMPLATE" else type(None) diff --git a/pyffi/object_models/xml/bit_struct.py b/pyffi/object_models/xml/bit_struct.py index 4720dbc41..3f39d3244 100644 --- a/pyffi/object_models/xml/bit_struct.py +++ b/pyffi/object_models/xml/bit_struct.py @@ -68,8 +68,6 @@ def __init__(cls, name, bases, dct): cls._struct = 'H' elif cls._numbytes == 4: cls._struct = 'I' - elif cls._numbytes == 8: - cls._struct = 'Q' else: raise RuntimeError("unsupported bitstruct numbytes") diff --git a/pyffi/object_models/xml/expression.py b/pyffi/object_models/xml/expression.py index 5b24cb542..6813ae564 100644 --- a/pyffi/object_models/xml/expression.py +++ b/pyffi/object_models/xml/expression.py @@ -43,7 +43,6 @@ import re import sys # stderr (for debugging) -from pyffi.object_models.xml.bit_struct import BitStructBase class Expression(object): """This class represents an expression. @@ -76,7 +75,7 @@ class Expression(object): False """ - operators = set(('==', '!=', '>=', '<=', '&&', '||', '&', '|', '-', '!', '>>', '<<', + operators = set(('==', '!=', '>=', '<=', '&&', '||', '&', '|', '-', '!', '<', '>', '/', '*', '+')) def __init__(self, expr_str, name_filter=None): @@ -125,8 +124,7 @@ def eval(self, data=None): else: assert (isinstance(self._right, int)) # debug right = self._right - - + if self._op == '==': return left == right elif self._op == '!=': @@ -152,29 +150,11 @@ def eval(self, data=None): elif self._op == '<': return left < right elif self._op == '/': - if right > 0: - return left / right - else: - return 0 return left / right elif self._op == '*': return left * right elif self._op == '+': return left + right - elif self._op == '<<': - # get underlying value for bitstructs - if isinstance(left, (BitStructBase,) ): - left = left.get_attributes_values(None) - if isinstance(right, (BitStructBase,) ): - right = right.get_attributes_values(None) - return left << right - elif self._op == '>>': - # get underlying value for bitstructs - if isinstance(left, (BitStructBase,) ): - left = left.get_attributes_values(None) - if isinstance(right, (BitStructBase,) ): - right = right.get_attributes_values(None) - return left >> right else: raise NotImplementedError("expression syntax error: operator '" + self._op + "' not implemented") diff --git a/pyffi/object_models/xml/struct_.py b/pyffi/object_models/xml/struct_.py index ad559e272..300b1a868 100644 --- a/pyffi/object_models/xml/struct_.py +++ b/pyffi/object_models/xml/struct_.py @@ -129,33 +129,10 @@ def __init__(cls, name, bases, dct): # precalculate the attribute list # profiling shows that this speeds up most of the StructBase methods # that rely on parsing the attribute list - - # this is a non-unique list of all StructAttributes of this class and its ancestors - # may continue more than one StructAttribute of the same name cls._attribute_list = cls._get_attribute_list() # precalculate the attribute name list - # list of unique names in the order they must be read - cls._names = [] - - # per field name, list of all StructAttributes that may apply - cls._attribute_unions = {} - - # populate unions and field names list - for attr in cls._attribute_list: - # we encounter a new field name so create a new union list for it, and add it to field names - if attr.name not in cls._attribute_unions: - cls._attribute_unions[attr.name] = [] - cls._names.append(attr.name) - # add this StructAttribute to union - cls._attribute_unions[attr.name].append(attr) - - # # debug - # if name == "NiNode": - # print(cls._attribute_list ) - # print(cls._attribute_unions) - # for name in cls._names: - # print(name, len(cls._attribute_unions[name]) ) + cls._names = cls._get_names() def __repr__(cls): return ""%(cls.__name__) @@ -265,7 +242,7 @@ def __init__(self, template = None, argument = None, parent = None): array is an attribute of.""" # used to track names of attributes that have already been added # is faster than self.__dict__.has_key(...) - # names = set() + names = set() # initialize argument self.arg = argument # save parent (note: disabled for performance) @@ -274,20 +251,22 @@ def __init__(self, template = None, argument = None, parent = None): # this list is used for instance by qskope to display the structure # in a tree view self._items = [] - # initialize attributes - # _attribute_list is a list of StructAttributes, including unions of the same name for attr in self._attribute_list: # skip attributes with dupiclate names # (for this to work properly, duplicates must have the same # type, template, argument, arr1, and arr2) - # if attr.name in names: - # continue - # names.add(attr.name) + if attr.name in names: + continue + names.add(attr.name) + # things that can only be determined at runtime (rt_xxx) - rt_type = attr.type_ if attr.type_ != type(None) else template - rt_template = attr.template if attr.template != type(None) else template - rt_arg = attr.arg + rt_type = attr.type_ if attr.type_ != type(None) \ + else template + rt_template = attr.template if attr.template != type(None) \ + else template + rt_arg = attr.arg if isinstance(attr.arg, (int, type(None))) \ + else getattr(self, attr.arg) # instantiate the class, handling arrays at the same time if attr.arr1 == None: @@ -316,7 +295,7 @@ def __init__(self, template = None, argument = None, parent = None): # add instance to item list self._items.append(attr_instance) - + def deepcopy(self, block): """Copy attributes from a given block (one block class must be a subclass of the other). Returns self.""" @@ -382,10 +361,12 @@ def read(self, stream, data): # skip abstract attributes if attr.is_abstract: continue + # get attribute argument (can only be done at runtime) + rt_arg = attr.arg if isinstance(attr.arg, (int, type(None))) \ + else getattr(self, attr.arg) # read the attribute attr_value = getattr(self, "_%s_value_" % attr.name) - # get attribute argument (can only be done at runtime) - attr_value.arg = attr.arg + attr_value.arg = rt_arg # if hasattr(attr, "type_"): # attr_value._elementType = attr.type_ self._log_struct(stream, attr) @@ -399,11 +380,13 @@ def write(self, stream, data): # skip abstract attributes if attr.is_abstract: continue + # get attribute argument (can only be done at runtime) + rt_arg = attr.arg if isinstance(attr.arg, (int, type(None))) \ + else getattr(self, attr.arg) # write the attribute attr_value = getattr(self, "_%s_value_" % attr.name) - # get attribute argument (can only be done at runtime) - attr_value.arg = attr.arg - attr_value.write(stream, data) + attr_value.arg = rt_arg + getattr(self, "_%s_value_" % attr.name).write(stream, data) self._log_struct(stream, attr) def fix_links(self, data): @@ -505,7 +488,7 @@ def get_versions(cls, game): @classmethod def _get_attribute_list(cls): - """Calculate the list of all StructAttributes of this structure, including duplicates.""" + """Calculate the list of all attributes of this structure.""" # string of attributes of base classes of cls attrs = [] for base in cls.__bases__: @@ -513,12 +496,26 @@ def _get_attribute_list(cls): attrs.extend(base._get_attribute_list()) except AttributeError: # when base class is "object" pass + attrs.extend(cls._attrs) + return attrs + + @classmethod + def _get_names(cls): + """Calculate the list of all attributes names in this structure. + Skips duplicate names.""" + # string of attributes of base classes of cls + names = [] + for base in cls.__bases__: + try: + names.extend(base._get_names()) + except AttributeError: # when base class is "object" + pass for attr in cls._attrs: - if attr in attrs: + if attr.name in names: continue else: - attrs.append(attr) - return attrs + names.append(attr.name) + return names def _get_filtered_attribute_list(self, data=None): """Generator for listing all 'active' attributes, that is, @@ -538,7 +535,7 @@ def _get_filtered_attribute_list(self, data=None): version = None user_version = None names = set() - for attr, attr_instance in zip(self._attribute_list, self._items): + for attr in self._attribute_list: #print(attr.name, version, attr.ver1, attr.ver2) # debug # check version @@ -573,10 +570,6 @@ def _get_filtered_attribute_list(self, data=None): #print("duplicate check passed") # debug names.add(attr.name) - - # assign attribute value - setattr(self, "_%s_value_" % attr.name, attr_instance) - # passed all tests # so yield the attribute yield attr From 596471150c01a1cbba2b1e6aaefd189e1f834271 Mon Sep 17 00:00:00 2001 From: HENDRIX-ZT2 Date: Thu, 21 Nov 2019 13:50:01 +0100 Subject: [PATCH 22/22] Expression: fix bracketed expressions, add modulo operator Expressions with brackets such as `10 - (4*2)` would silently discard the part before the bracket, `10 - `. --- pyffi/object_models/xml/expression.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/pyffi/object_models/xml/expression.py b/pyffi/object_models/xml/expression.py index 6813ae564..e9d79318d 100644 --- a/pyffi/object_models/xml/expression.py +++ b/pyffi/object_models/xml/expression.py @@ -76,7 +76,7 @@ class Expression(object): """ operators = set(('==', '!=', '>=', '<=', '&&', '||', '&', '|', '-', '!', - '<', '>', '/', '*', '+')) + '<', '>', '/', '*', '+', '%')) def __init__(self, expr_str, name_filter=None): try: @@ -155,6 +155,8 @@ def eval(self, data=None): return left * right elif self._op == '+': return left + right + elif self._op == '%': + return left % right else: raise NotImplementedError("expression syntax error: operator '" + self._op + "' not implemented") @@ -228,7 +230,7 @@ def _partition(cls, expr_str): # and if so, find the position of the starting bracket and the ending # bracket left_startpos, left_endpos = cls._scan_brackets(expr_str) - if left_startpos >= 0: + if left_startpos == 0: # yes, it is a bracketted expression # so remove brackets and whitespace, # and let that be the left hand side