[repo] Cache building :target option

Caches the keys (= partname) of the :target option and parses the
corresponding device file lazily.
Basic md5 file hashing is used to detect changes in modm-devices and
trigger a recompute of the cache.

.gitignore

@@ -31,3 +31,6 @@ examples/**/modm
 examples/**/SConstruct
 examples/**/Makefile
 examples/**/generated
+
+# repo.lb cache file
+ext/modm-devices.cache

repo.lb

@@ -14,8 +14,11 @@
 import os
 import sys
 import glob
+import hashlib
+from pathlib import Path
+from distutils.version import StrictVersion
 
-rootpath = localpath("ext/modm-devices/tools/device")
+rootpath = repopath("ext/modm-devices/tools/device")
 sys.path.append(rootpath)
 
 try:
@@ -34,22 +37,109 @@ if sys.version_info[1] < 6:
     print("modm will require Python 3.6 in the near future.\n"
           "Please check if you can upgrade your installation!\n")
 
-def init(repo):
-    repo.name = "modm"
+import lbuild
+min_lbuild_version = "0.3.6"
+if StrictVersion(lbuild.__version__) < StrictVersion(min_lbuild_version):
+    print("modm requires at least lbuild v{}, please upgrade lbuild!".format(min_lbuild_version))
+    exit(1)
+
+
+# =============================================================================
+class DevicesCache(dict):
+    """
+    Building the device enumeration from modm-device is quite expensive,
+    so we cache the results in `ext/modm-devices.cache`
+    The text file contains two maps:
+
+      1. partname -> file-name.xml
+         We use this to populate the `:target` option, but we only
+         actually parse the device file and build the device on the first
+         access of the value.
+
+      2. file-name.xml -> MD5 hash
+         This is used to check if any files have changed their contents.
+         No additional checks are done, if files have moved, this may fail.
+    """
+
+    def __init__(self):
+        dict.__init__(self)
+        self.device_to_file = {}
+
+    def parse_all(self):
+        mapping = {}
+        device_file_names  = glob.glob(repopath("ext/modm-devices/devices/**/*.xml"))
+        device_file_names += glob.glob(repopath("tools/devices/**/*.xml"))
 
-    devices = {}
-    try:
-        parser = modm.parser.DeviceParser()
-        device_file_names  = glob.glob(localpath("ext/modm-devices/devices/**/*.xml"))
-        device_file_names += glob.glob(localpath("tools/devices/**/*.xml"))
         # roughly filter to supported devices
         supported = ["stm32f0", "stm32f1", "stm32f2", "stm32f3", "stm32f4", "stm32f7",
                      "stm32l4", "at90", "attiny", "atmega", "hosted"]
         device_file_names = [dfn for dfn in device_file_names if any(s in dfn for s in supported)]
+
+        # Parse the files and build the :target enumeration
+        parser = modm.parser.DeviceParser()
         for device_file_name in device_file_names:
             device_file = parser.parse(device_file_name)
             for device in device_file.get_devices():
-                devices[device.partname] = device
+                self[device.partname] = device
+                mapping[device.partname] = device_file_name
+
+        return mapping
+
+    def build(self):
+        cache = Path(repopath("ext/modm-devices.cache"))
+        recompute_required = False
+
+        if cache.exists():
+            # Read cache file and populate :target
+            for line in cache.read_text().splitlines():
+                line = line.split(" ")
+                if line[0].startswith("/"):
+                    # If any file has changed, recompute
+                    file = Path(repopath(line[0][1:]))
+                    if not file.exists():
+                        recompute_required = True
+                        break
+                    if line[1] != hashlib.md5(file.read_bytes()).hexdigest():
+                        recompute_required = True
+                        break
+                else:
+                    # Store None as device file value
+                    self.device_to_file[line[0]] = line[1]
+                    self[line[0]] = None
+
+        if not cache.exists() or recompute_required:
+            print("Recomputing device cache...")
+            content = self.parse_all()
+            # prefix the files with a / so we can distinguish them from partnames
+            files = ["/{} {}".format(Path(f).relative_to(repopath(".")),
+                                    hashlib.md5(Path(f).read_bytes()).hexdigest())
+                     for f in set(content.values())]
+            content = ["{} {}".format(d, Path(f).relative_to(repopath(".")))
+                       for (d, f) in content.items()]
+            content = sorted(content) + sorted(files)
+            cache.write_text("\n".join(content))
+
+    def __getitem__(self, item):
+        value = dict.__getitem__(self, item)
+        if value is None:
+            # Parse the device file and build its devices
+            parser = modm.parser.DeviceParser()
+            device_file = parser.parse(repopath(self.device_to_file[item]))
+            for device in device_file.get_devices():
+                if device.partname == item:
+                    # Store the real device now
+                    self[item] = device
+                    return device
+        return value
+
+
+# =============================================================================
+def init(repo):
+    repo.name = "modm"
+
+    devices = DevicesCache()
+    try:
+        devices.build()
     except (modm.ParserException) as e:
         print(e)
         exit(1)