Transparently handle xz-compressed properties files. (#111)

.github/workflows/ubuntu.yml

@@ -8,6 +8,7 @@ jobs:
     runs-on: ${{ matrix.os }}
 
     strategy:
+      fail-fast: false
       matrix:
         os: [ubuntu-latest]
         python-version: ["3.7", "3.8", "3.9", "3.10", "3.11"]

docs/news.rst

@@ -1,6 +1,18 @@
 Changelog
 =========
 
+v7.3 (unreleased)
+-----------------
+
+Lab
+^^^
+* Transparently handle xz-compressed properties files (Jendrik Seipp).
+
+Downward Lab
+^^^^^^^^^^^^
+* No changes yet.
+
+
 v7.2 (2022-10-09)
 -----------------
 

lab/fetcher.py

@@ -107,7 +107,10 @@ def __call__(self, src_dir, eval_dir=None, merge=None, filter=None, **kwargs):
             os.path.join(src_dir, "runs-00001-00100")
         )
         if fetch_from_eval_dir:
-            src_props = tools.Properties(filename=os.path.join(src_dir, "properties"))
+            src_path = os.path.join(src_dir, "properties")
+            src_props = tools.Properties(filename=src_path)
+            if not src_props:
+                logging.critical(f"No properties found in {src_dir}")
             run_filter.apply(src_props)
             combined_props.update(src_props)
             logging.info(f"Fetched properties of {len(src_props)} runs.")

lab/reports/__init__.py

@@ -415,14 +415,11 @@ def _scan_data(self):
 
     def _load_data(self):
         props_file = os.path.join(self.eval_dir, "properties")
-        if not os.path.exists(props_file):
-            logging.critical(f"Properties file not found at {props_file}")
-
         logging.info("Reading properties file")
         self.props = tools.Properties(filename=props_file)
-        logging.info("Reading properties file finished")
         if not self.props:
-            logging.critical("properties file in evaluation dir is empty.")
+            logging.critical(f"No properties found in {self.eval_dir}")
+        logging.info("Reading properties file finished")
 
     def _apply_filter(self):
         self.run_filter.apply(self.props)

lab/tools.py

@@ -2,6 +2,7 @@
 import colorsys
 import functools
 import logging
+import lzma
 import math
 import os
 from pathlib import Path
@@ -257,37 +258,50 @@ def default(self, o):
             else:
                 return super().default(o)
 
+    JSON_ARGS = {
+        "cls": _PropertiesEncoder,
+        "indent": 2,
+        "separators": (",", ": "),
+        "sort_keys": True,
+    }
+
+    """Transparently handle properties files compressed with xz."""
+
     def __init__(self, filename=None):
-        self.filename = filename
-        self.load(filename)
+        self.path = filename
+        if self.path:
+            self.path = Path(self.path).resolve()
+            xz_path = self.path.with_suffix(".xz")
+            if self.path.is_file() and xz_path.is_file():
+                logging.critical(f"Only one of {self.path} and {xz_path} may exist")
+            if not self.path.is_file() and xz_path.is_file():
+                self.path = xz_path
+            if self.path.is_file():
+                self.load(self.path)
         dict.__init__(self)
 
     def __str__(self):
-        return json.dumps(
-            self,
-            cls=self._PropertiesEncoder,
-            indent=2,
-            separators=(",", ": "),
-            sort_keys=True,
-        )
+        return json.dumps(self, **self.JSON_ARGS)
 
     def load(self, filename):
-        if not filename or not os.path.exists(filename):
-            return
-        with open(filename) as f:
+        path = Path(filename)
+        open_func = lzma.open if path.suffix == ".xz" else open
+        with open_func(path) as f:
             try:
                 self.update(json.load(f))
             except ValueError as e:
-                logging.critical(f"JSON parse error in file '{filename}': {e}")
+                logging.critical(f"JSON parse error in file '{path}': {e}")
 
     def add_unexplained_error(self, error):
         add_unexplained_error(self, error)
 
     def write(self):
         """Write the properties to disk."""
-        assert self.filename
-        makedirs(os.path.dirname(self.filename))
-        write_file(self.filename, str(self))
+        assert self.path
+        self.path.parent.mkdir(parents=True, exist_ok=True)
+        open_func = lzma.open if self.path.suffix == ".xz" else open
+        with open_func(self.path, "w") as f:
+            json.dump(self, f, **self.JSON_ARGS)
 
 
 class RunFilter: