Merge branch 'inf-479-records' into dev

An internal merge adding decoded records classes for Morrowind. These
don't load yet because Morrowind does not have GRUPs, so our current
Mob* classes won't work. Probably easiest to just write some new ones
for this system instead of trying to bang the existing ones into shape.

Some minor changes to the rest of the codebase, but it shouldn't have
much of an impact on anything. Tested building a BP in all games and
it worked fine.

Under #479

Mopy/bash/bosh/save_headers.py

@@ -32,7 +32,6 @@
 __author__ = u'Utumno'
 
 import copy
-import itertools
 import lz4.block
 import StringIO
 import struct
@@ -611,11 +610,9 @@ def load_header(self, ins, load_image=False):
         save_info = ModInfo(self._save_path, load_cache=True)
         ##: Figure out where some more of these are (e.g. level)
         self.header_size = save_info.header.size
-        self.pcName = decode(cstrip(save_info.header.pc_name))
+        self.pcName = save_info.header.pc_name
         self.pcLevel = 0
-        self.pcLocation = decode(cstrip(save_info.header.curr_cell),
-                                 bolt.pluginEncoding,
-                                 avoidEncodings=(u'utf8', u'utf-8'))
+        self.pcLocation = save_info.header.curr_cell
         self.gameDays = self.gameTicks = 0
         self.masters = save_info.masterNames[:]
         self.pc_curr_health = save_info.header.pc_curr_health

Mopy/bash/brec/advanced_elements.py

@@ -37,7 +37,7 @@
 from .basic_elements import MelBase, MelNull, MelObject, MelStruct
 from .mod_io import ModWriter
 from .. import exception
-from ..bolt import sio, struct_pack
+from ..bolt import GPath, sio, struct_pack
 
 #------------------------------------------------------------------------------
 class _MelDistributor(MelNull):
@@ -570,18 +570,22 @@ def _decide_common(self, record):
         """Performs the actual decisions for both loading and dumping."""
         raise exception.AbstractError()
 
-class AttrExistsDecider(ACommonDecider):
-    """Decider that returns True if an attribute with the specified name is
-    present on the record."""
+class FidNotNullDecider(ACommonDecider):
+    """Decider that returns True if the FormID attribute with the specified
+    name is not NULL."""
     def __init__(self, target_attr):
-        """Creates a new AttrExistsDecider with the specified attribute.
+        """Creates a new FidNotNullDecider with the specified attribute.
 
         :param target_attr: The name of the attribute to check.
         :type target_attr: unicode"""
-        self.target_attr = target_attr
+        self._target_attr = target_attr
 
     def _decide_common(self, record):
-        return hasattr(record, self.target_attr)
+        ##: Wasteful, but bush imports brec which uses this decider, so we
+        # can't import bush in __init__...
+        from .. import bush
+        return getattr(record, self._target_attr) != (
+            GPath(bush.game.master_file), 0)
 
 class AttrValDecider(ACommonDecider):
     """Decider that returns an attribute value (may optionally apply a function
@@ -715,7 +719,7 @@ class MelUnion(MelBase):
             u'f': MelFloat(b'DATA', u'value'),
             u's': MelLString(b'DATA', u'value'),
         }, decider=AttrValDecider(
-            u'eid', transformer=lambda eid: decode(eid[0]) if eid else u'i'),
+            u'eid', transformer=lambda e: e[0] if e else u'i'),
             fallback=MelSInt32(b'DATA', u'value')
         ),
     When a DATA subrecord is encountered, the union is asked to load it. It

Mopy/bash/brec/basic_elements.py

@@ -26,7 +26,7 @@
 from __future__ import division, print_function
 import struct
 
-from .utils_constants import FID, null1, _make_hashable
+from .utils_constants import FID, null1, _make_hashable, FixedString
 from .. import bolt, exception
 from ..bolt import decode, encode
 
@@ -487,9 +487,10 @@ def static_size(self):
 class MelString(MelBase):
     """Represents a mod record string element."""
 
-    def __init__(self, subType, attr, default=None, maxSize=0):
+    def __init__(self, subType, attr, default=None, maxSize=0, minSize=0):
         super(MelString, self).__init__(subType, attr, default)
         self.maxSize = maxSize
+        self.minSize = minSize
 
     def loadData(self, record, ins, sub_type, size_, readId):
         value = ins.readString(size_, readId)
@@ -498,7 +499,8 @@ def loadData(self, record, ins, sub_type, size_, readId):
     def dumpData(self,record,out):
         string_val = record.__getattribute__(self.attr)
         if string_val is not None:
-            out.write_string(self.subType, string_val, max_size=self.maxSize)
+            out.write_string(self.subType, string_val, max_size=self.maxSize,
+                min_size=self.minSize)
 
 #------------------------------------------------------------------------------
 class MelUnicode(MelString):
@@ -550,6 +552,18 @@ class MelStruct(MelBase):
     """Represents a structure record."""
 
     def __init__(self, subType, struct_format, *elements):
+        """:type subType: bytes
+        :type struct_format: unicode"""
+        # Sometimes subrecords have to preserve non-aligned sizes, check that
+        # we don't accidentally pad those to alignment
+        if (not struct_format.startswith(u'=') and
+                struct.calcsize(struct_format) != struct.calcsize(
+                    u'=' + struct_format)):
+            raise SyntaxError(
+                u"Automatic padding inserted for struct format '%s', this is "
+                u"almost certainly not what you want. Prepend '=' to preserve "
+                u"the unaligned size or manually pad with 'x' to avoid this "
+                u"error." % struct_format)
         self.subType, self.struct_format = subType, struct_format
         self.attrs,self.defaults,self.actions,self.formAttrs = MelBase.parseElements(*elements)
         # Check for duplicate attrs - can't rely on MelSet.getSlotsUsed only,
@@ -588,7 +602,10 @@ def dumpData(self,record,out):
         getter = record.__getattribute__
         for attr,action in zip(self.attrs,self.actions):
             value = getter(attr)
-            if action: value = value.dump()
+            # Just in case, apply the action to itself before dumping to handle
+            # e.g. a FixedString getting assigned a unicode value. Worst case,
+            # this is just a noop.
+            if action: value = action(value).dump()
             valuesAppend(value)
         out.packSub(self.subType, self.struct_format, *values)
 
@@ -603,6 +620,14 @@ def mapFids(self,record,function,save=False):
     def static_size(self):
         return struct.calcsize(self.struct_format)
 
+#------------------------------------------------------------------------------
+class MelFixedString(MelStruct):
+    """Subrecord that stores a string of a constant length. Just a wrapper
+    around a struct with a single FixedString element."""
+    def __init__(self, signature, attr, str_length, default=b''):
+        super(MelFixedString, self).__init__(signature, u'%us' % str_length,
+            (FixedString(str_length, default), attr))
+
 #------------------------------------------------------------------------------
 # Simple primitive type wrappers
 class _MelSimpleStruct(MelStruct):

Mopy/bash/brec/common_records.py

@@ -29,10 +29,10 @@
 import struct
 from operator import attrgetter
 
-from .advanced_elements import AttrExistsDecider, AttrValDecider, MelArray, \
+from .advanced_elements import FidNotNullDecider, AttrValDecider, MelArray, \
     MelUnion
 from .basic_elements import MelBase, MelFid, MelFids, MelFloat, MelGroups, \
-    MelLString, MelNull, MelStruct, MelUInt32, MelSInt32
+    MelLString, MelNull, MelStruct, MelUInt32, MelSInt32, MelFixedString
 from .common_subrecords import MelEdid
 from .record_structs import MelRecord, MelSet
 from .utils_constants import FID
@@ -174,10 +174,11 @@ class MreGlob(MelRecord):
        (short,long,float), are stored as floats -- which means that very large
        integers lose precision."""
     rec_sig = b'GLOB'
+
     melSet = MelSet(
         MelEdid(),
-        MelStruct('FNAM','s',('format','s')),
-        MelFloat('FLTV', 'value'),
+        MelFixedString(b'FNAM', u'global_format', 1, u's'),
+        MelFloat(b'FLTV', u'global_value'),
     )
     __slots__ = melSet.getSlotsUsed()
 
@@ -195,7 +196,7 @@ class MreGmstBase(MelRecord):
             u'f': MelFloat(b'DATA', u'value'),
             u's': MelLString(b'DATA', u'value'),
         }, decider=AttrValDecider(
-            u'eid', transformer=lambda eid: decode(eid[0]) if eid else u'i'),
+            u'eid', transformer=lambda e: e[0] if e else u'i'),
             fallback=MelSInt32(b'DATA', u'value')
         ),
     )
@@ -219,11 +220,11 @@ class MreLand(MelRecord):
                 b'BTXT': MelStruct(b'BTXT', u'IBsh', (FID, u'btxt_texture'),
                     u'quadrant', u'unknown', u'layer'),
             }),
-            # VTXT only exists for ATXT layers
+            # VTXT only exists for ATXT layers, i.e. if ATXT's FormID is valid
             MelUnion({
                 True:  MelBase(b'VTXT', u'alpha_layer_data'),
                 False: MelNull(b'VTXT'),
-            }, decider=AttrExistsDecider(u'atxt_texture')),
+            }, decider=FidNotNullDecider(u'atxt_texture')),
         ),
         MelArray('vertex_textures',
             MelFid('VTEX', 'vertex_texture'),

Mopy/bash/brec/common_subrecords.py

@@ -30,7 +30,8 @@
     MelUnion, PartialLoadDecider, FlagDecider
 from .basic_elements import MelBase, MelFid, MelGroup, MelGroups, MelLString, \
     MelNull, MelSequential, MelString, MelStruct, MelUInt32, MelOptStruct, \
-    MelOptFloat, MelOptUInt8, MelOptUInt32, MelOptFid, MelReadOnly, MelUInt8
+    MelOptFloat, MelOptUInt8, MelOptUInt32, MelOptFid, MelReadOnly, MelUInt8, \
+    MelFids
 from .utils_constants import _int_unpacker, FID, null1, null2, null3, null4
 from ..bolt import Flags, encode, struct_pack, struct_unpack
 
@@ -298,6 +299,19 @@ def __init__(self, sub_type, attr):
             MelStruct(sub_type, u'2f', u'time', u'value'),
         )
 
+#------------------------------------------------------------------------------
+class MelColor(MelStruct):
+    """Required Color."""
+    def __init__(self, color_sig=b'CNAM'):
+        super(MelColor, self).__init__(color_sig, u'4B', u'red', u'green',
+            u'blue', u'unused_alpha')
+
+class MelColorO(MelOptStruct):
+    """Optional Color."""
+    def __init__(self, color_sig=b'CNAM'):
+        super(MelColorO, self).__init__(color_sig, u'4B', u'red', u'green',
+            u'blue', u'unused_alpha')
+
 #------------------------------------------------------------------------------
 class MelDescription(MelLString):
     """Handles a description (DESC) subrecord."""
@@ -572,7 +586,7 @@ def __init__(self, entry_type_val, element):
             fallback=MelNull(b'NULL')) # ignore
 
 #------------------------------------------------------------------------------
-class MelRef3D(MelStruct):
+class MelRef3D(MelOptStruct):
     """3D position and rotation for a reference record (REFR, ACHR, etc.)."""
     def __init__(self):
         super(MelRef3D, self).__init__(
@@ -585,6 +599,12 @@ class MelRefScale(MelOptFloat):
     def __init__(self):
         super(MelRefScale, self).__init__(b'XSCL', (u'ref_scale', 1.0))
 
+#------------------------------------------------------------------------------
+class MelSpells(MelFids):
+    """Handles the common SPLO subrecord."""
+    def __init__(self):
+        super(MelSpells, self).__init__(b'SPLO', u'spells')
+
 #------------------------------------------------------------------------------
 class MelWorldBounds(MelSequential):
     """Worlspace (WRLD) bounds."""

Mopy/bash/brec/record_structs.py

@@ -46,6 +46,11 @@ def __init__(self,*elements):
             element.getDefaulters(self.defaulters,'')
             element.getLoaders(self.loaders)
             element.hasFids(self.formElements)
+        for sig_candidate in self.loaders:
+            if len(sig_candidate) != 4 or not isinstance(sig_candidate, bytes):
+                raise SyntaxError(u"Invalid signature '%s': Signatures must "
+                                  u'be bytestrings and 4 bytes in '
+                                  u'length.' % sig_candidate)
 
     def getSlotsUsed(self):
         """This function returns all of the attributes used in record instances

Mopy/bash/brec/utils_constants.py

@@ -26,7 +26,8 @@
 from __future__ import division, print_function
 import struct
 
-from ..bolt import decode, Flags, struct_pack, struct_unpack
+from .. import bolt
+from ..bolt import cstrip, decode, Flags, struct_pack, struct_unpack
 # no local imports, imported everywhere in brec
 
 # Random stuff ----------------------------------------------------------------
@@ -66,6 +67,36 @@ def _make_hashable(target_obj):
         return tuple([_make_hashable(x) for x in target_obj])
     return target_obj
 
+class FixedString(unicode):
+    """An action for MelStructs that will decode and encode a fixed-length
+    string. Note that you do not need to specify defaults when using this."""
+    __slots__ = (u'str_length',)
+    _str_encoding = bolt.pluginEncoding
+
+    def __new__(cls, str_length, target_str=b''):
+        if isinstance(target_str, unicode):
+            decoded_str = target_str
+        else:
+            decoded_str = u'\n'.join(
+                decode(x, cls._str_encoding,
+                    avoidEncodings=(u'utf8', u'utf-8'))
+                for x in cstrip(target_str).split(b'\n'))
+        new_str = super(FixedString, cls).__new__(cls, decoded_str)
+        new_str.str_length = str_length
+        return new_str
+
+    def __call__(self, new_str):
+        # 0 is the default, so replace it with whatever we currently have
+        return FixedString(self.str_length, new_str or unicode(self))
+
+    def dump(self):
+        return bolt.encode_complex_string(self, max_size=self.str_length,
+            min_size=self.str_length)
+
+class AutoFixedString(FixedString):
+    """Variant of FixedString that uses chardet to detect encodings."""
+    _str_encoding = None
+
 # Reference (fid) -------------------------------------------------------------
 def strFid(form_id):
     """Return a string representation of the fid."""

Mopy/bash/game/fallout3/records.py

@@ -43,7 +43,7 @@
     MelCtdaFo3, MelRef3D, MelXlod, MelNull, MelWorldBounds, MelEnableParent, \
     MelRefScale, MelMapMarker, MelActionFlags, MelEnchantment, MelScript, \
     MelDecalData, MelDescription, MelLists, MelPickupSound, MelDropSound, \
-    MelActivateParents, BipedFlags
+    MelActivateParents, BipedFlags, MelSpells
 from ...exception import ModError, ModSizeError
 # Set MelModel in brec but only if unset
 if brec.MelModel is None:
@@ -160,9 +160,8 @@ def __init__(self, attr=u'destructible'):
 #------------------------------------------------------------------------------
 class MelEffects(MelGroups):
     """Represents ingredient/potion/enchantment/spell effects."""
-
-    def __init__(self, attr=u'effects'):
-        super(MelEffects, self).__init__(attr,
+    def __init__(self):
+        super(MelEffects, self).__init__(u'effects',
             MelFid(b'EFID', u'baseEffect'),
             MelStruct(b'EFIT', u'4Ii', u'magnitude', u'area', u'duration',
                 u'recipient', u'actorValue'),
@@ -928,7 +927,7 @@ class MreCrea(MreActor):
         MelBounds(),
         MelFull(),
         MelModel(),
-        MelFids('SPLO','spells'),
+        MelSpells(),
         MelEnchantment(),
         MelUInt16('EAMT', 'eamt'),
         MelStrings('NIFZ','bodyParts'),
@@ -2034,7 +2033,7 @@ class MelNpcDnam(MelLists):
         MelEnchantment(),
         MelUInt16('EAMT', 'unarmedAttackAnimation'),
         MelDestructible(),
-        MelFids('SPLO','spells'),
+        MelSpells(),
         MelScript(),
         MelItems(),
         MelStruct('AIDT','=5B3sIbBbBi', ('aggression', 0), ('confidence',2),

Mopy/bash/game/falloutnv/records.py

@@ -63,6 +63,7 @@ class MreTes4(MreHeaderBase):
     )
     __slots__ = melSet.getSlotsUsed()
 
+#------------------------------------------------------------------------------
 class MreAchr(MelRecord):
     """Placed NPC."""
     rec_sig = b'ACHR'