[WIP] Media versioning

py/openage/CMakeLists.txt

@@ -3,4 +3,5 @@ add_py_package(openage)
 add_test_py(openage.assets.test "tests python asset locator")
 
 add_subdirectory("convert")
+add_subdirectory("pack")
 add_subdirectory("testing")

py/openage/convert/dataformat/exportable.py

@@ -1,5 +1,6 @@
-# Copyright 2014-2014 the openage authors. See copying.md for legal info.
+# Copyright 2014-2015 the openage authors. See copying.md for legal info.
 
+import hashlib
 import struct
 
 from ..util import zstr
@@ -381,7 +382,7 @@ def structs(cls):
             dbg(lazymsg=lambda: "%s: exporting member %s<%s>" % (repr(cls), member_name, member_type), lvl=3)
 
             if isinstance(member_type, MultisubtypeMember):
-                for subtype_name, subtype_class in member_type.class_lookup.items():
+                for subtype_name, subtype_class in sorted(member_type.class_lookup.items()):
                     if not issubclass(subtype_class, Exportable):
                         raise Exception("tried to export structs from non-exportable %s" % subtype_class)
                     ret += subtype_class.structs()
@@ -403,12 +404,62 @@ def structs(cls):
 
         return ret
 
+    @classmethod
+    def format_hash(cls, hasher=None):
+        """
+        provides a deterministic hash of all exported structure members
+
+        used for determining changes in the exported data, which requires
+        data reconversion.
+        """
+
+        if not hasher:
+            hasher = hashlib.sha512()
+
+        # struct properties
+        hasher.update(cls.name_struct.encode())
+        hasher.update(cls.name_struct_file.encode())
+        hasher.update(cls.struct_description.encode())
+
+        # only hash exported struct members!
+        # non-exported values don't influence anything.
+        members = cls.get_data_format(
+            allowed_modes=(True, READ_EXPORT, NOREAD_EXPORT),
+            flatten_includes=False,
+        )
+        for is_parent, export, member_name, member_type in members:
+
+            # includemembers etc have no name.
+            if member_name:
+                hasher.update(member_name.encode())
+
+            if isinstance(member_type, DataMember):
+                hasher = member_type.format_hash(hasher)
+
+            elif isinstance(member_type, str):
+                hasher.update(member_type.encode())
+
+            else:
+                raise Exception("can't hash unsupported member")
+
+            hasher.update(export.name.encode())
+
+        return hasher
+
     @classmethod
     def get_effective_type(cls):
         return cls.name_struct
 
     @classmethod
     def get_data_format(cls, allowed_modes=False, flatten_includes=False, is_parent=False):
+        """
+        return all members of this exportable (a struct.)
+
+        can filter by export modes and can also return included members:
+        inherited members can either be returned as to-be-included,
+        or can be fetched and displayed as if they weren't inherited.
+        """
+
         for member in cls.data_format:
             export, member_name, member_type = member
 

py/openage/convert/dataformat/members.py

@@ -73,6 +73,14 @@ def get_struct_entries(self, member_name):
 
         return ["%s %s;" % (self.get_effective_type(), member_name)]
 
+    def format_hash(self, hasher):
+        """
+        hash these member's settings.
+
+        used to determine data format changes.
+        """
+        raise NotImplementedError("return the hasher updated with member settings")
+
     def __repr__(self):
         raise NotImplementedError("return short description of the member type %s" % (type(self)))
 
@@ -109,6 +117,9 @@ def get_parsers(self, idx, member):
             )
         ]
 
+    def format_hash(self, hasher):
+        return self.cls.format_hash(hasher)
+
     def __repr__(self):
         return "GroupMember<%s>" % repr(self.cls)
 
@@ -203,6 +214,15 @@ def is_dynamic_length(self, target=None):
         else:
             raise Exception("unknown length definition supplied: %s" % target)
 
+    def format_hash(self, hasher):
+        if callable(self.length):
+            # update hash with the lambda code
+            hasher.update(self.length.__code__.co_code)
+        else:
+            hasher.update(str(self.length).encode())
+
+        return hasher
+
 
 class RefMember(DataMember):
     """
@@ -218,6 +238,17 @@ def __init__(self, type_name, file_name):
         # would allow reusing a struct definition that lies in another file
         self.resolved  = False
 
+    def format_hash(self, hasher):
+        # the file_name is irrelevant for the format hash
+        # engine-internal relevance only.
+
+        # type name is none for subdata members, hash is determined
+        # by recursing into the subdata member itself.
+        if self.type_name:
+            hasher.update(self.type_name.encode())
+
+        return hasher
+
 
 class NumberMember(DataMember):
     """
@@ -267,6 +298,11 @@ def get_headers(self, output_target):
     def get_effective_type(self):
         return self.number_type
 
+    def format_hash(self, hasher):
+        hasher.update(self.number_type.encode())
+
+        return hasher
+
     def __repr__(self):
         return self.number_type
 
@@ -415,6 +451,14 @@ def get_snippets(self, file_name, format):
         else:
             return list()
 
+    def format_hash(self, hasher):
+        hasher = super().format_hash(hasher)
+
+        for v in sorted(self.values):
+            hasher.update(v.encode())
+
+        return hasher
+
     def __repr__(self):
         return "enum %s" % self.type_name
 
@@ -662,6 +706,15 @@ def get_snippets(self, file_name, format):
         else:
             return list()
 
+    def format_hash(self, hasher):
+        hasher = RefMember.format_hash(self, hasher)
+        hasher = DynLengthMember.format_hash(self, hasher)
+
+        for subtype_name, subtype_class in sorted(self.class_lookup.items()):
+            hasher = subtype_class.format_hash(hasher)
+
+        return hasher
+
     def __repr__(self):
         return "MultisubtypeMember<%s:len=%s>" % (self.type_name, self.length)
 

py/openage/convert/gamedata/empiresdat.py

@@ -78,7 +78,7 @@ class for fighting and beating the compressed empires2*.dat
     data_format = (
         (READ, "versionstr", "char[8]"),
 
-        # terain header data
+        # terrain header data
         (READ, "terrain_restriction_count", "uint16_t"),
         (READ, "terrain_count", "uint16_t"),
         (READ, "terrain_restriction_offset0", "int32_t[terrain_restriction_count]"),

py/openage/convert/gamedata/sound.py

@@ -1,4 +1,4 @@
-# Copyright 2013-2014 the openage authors. See copying.md for legal info.
+# Copyright 2013-2015 the openage authors. See copying.md for legal info.
 
 from ..dataformat.exportable import Exportable
 from ..dataformat.members import SubdataMember
@@ -11,7 +11,7 @@ class SoundItem(Exportable):
     struct_description = "one possible file for a sound."
 
     data_format = (
-        (READ_EXPORT, "filename",    "char[13]"),
+        (READ_EXPORT, "filename",     "char[13]"),
         (READ_EXPORT, "resource_id",  "int32_t"),
         (READ_EXPORT, "probablilty",  "int16_t"),
         (READ_EXPORT, "civilisation", "int16_t"),

py/openage/convert/versioning.py

@@ -0,0 +1,8 @@
+# Copyright 2015-2015 the openage authors. See copying.md for legal info.
+
+# Provides versioning information for converted files.
+
+
+from .gamedata import empiresdat
+
+print(empiresdat.EmpiresDat.format_hash().hexdigest())

py/openage/pack/CMakeLists.txt

@@ -0,0 +1,4 @@
+add_py_package(openage.pack)
+
+add_test_py(openage.pack.tests.packcfg_read "tests reading mod pack root config files")
+add_test_py(openage.pack.tests.packcfg_write "tests reading mod pack root config files")

py/openage/pack/pack_config.py

@@ -0,0 +1,113 @@
+# Copyright 2015-2015 the openage authors. See copying.md for legal info.
+
+import configparser
+import re
+
+
+class PackConfig:
+    """
+    main config file for a mod pack.
+
+    stores the package metainformation for the mod list.
+
+    file contents:
+
+    [openage-pack]
+    name = <package name>
+    version = <package version>
+    author = <pack author>
+    pack_type = (game|mod|...)
+    config_type = (cfg|nyan|...)
+    config = <filename.cfg>
+    """
+
+    # everything has to be in this section:
+    root_cfg = "openage-pack"
+
+    # required attributes
+    pack_attrs = ("name", "version", "author",
+                  "pack_type", "config_type", "config")
+
+    def __init__(self):
+        """
+        init all required attributes to None.
+        """
+
+        self.cfg = dict()
+
+        for attr in self.pack_attrs:
+            self.cfg[attr] = None
+
+    def check_sanity(self):
+        """
+        checks whether the current config is valid.
+        """
+
+        if None in self.cfg.values():
+            nones = [v for k, v in self.cfg.items() if v is None]
+            raise Exception("unset attribute(s): %s" % nones)
+
+        if self.cfg["pack_type"] not in ("game", "mod"):
+            raise Exception("unsupported data pack type")
+
+        if self.cfg["config_type"] not in ("cfg",):
+            raise Exception("config_type not supported: %s" %
+                            self.cfg["config_type"])
+
+        if not re.match(r"[a-zA-Z][a-zA-Z0-9_]*", self.cfg["name"]):
+            raise Exception("disallowed chars in pack name found")
+
+        if not re.match(r"v[0-9]+(\.[0-9]+)*(-r[0-9]+)", self.cfg["version"]):
+            raise Exception("invalid pack version")
+
+    def read(self, handle):
+        """
+        fill this pack config by the contents provided by the file handle.
+        """
+
+        cfp = configparser.ConfigParser()
+
+        # read the config from the (pseudo-?)handle
+        cfp.read_file(handle, source=handle.name)
+
+        if self.root_cfg not in cfp:
+            raise Exception("pack root config doesn't contain '%s' section."
+                            % self.root_cfg)
+        else:
+            self.cfg = cfp[self.root_cfg]
+
+        self.check_sanity()
+
+    def write(self, handle):
+        """
+        store the settings of this configfile to the given file handle.
+        """
+
+        self.check_sanity()
+
+        cfp = configparser.ConfigParser()
+        cfp[self.root_cfg] = self.cfg
+
+        cfp.write(handle)
+
+    def set_attrs(self, **dct):
+        """
+        set all attributes from the dict to this pack config.
+        """
+
+        # just take the known attributes
+        to_set = {k: v for k, v in dct.items() if k in self.pack_attrs}
+
+        # check if all keys were used
+        unknown_keys = dct.keys() - to_set.keys()
+        if unknown_keys:
+            raise Exception("unknown attr name(s) to set: %s" % unknown_keys)
+
+        self.cfg.update(to_set)
+
+    def get_attrs(self):
+        """
+        return this config's settings.
+        """
+
+        return self.cfg