Add subdir option for CachedRevision. (#114)

docs/news.rst

@@ -1,6 +1,19 @@
 Changelog
 =========
 
+v7.4 (unreleased)
+-----------------
+
+Lab
+^^^
+* Require *revision_cache* parameter in :class:`CachedRevision <lab.cached_revision.CachedRevision>` constructor (Jendrik Seipp).
+* Add *subdir* option for :class:`CachedRevision <lab.cached_revision.CachedRevision>` to support solvers residing in monolithic repos (Jendrik Seipp).
+
+Downward Lab
+^^^^^^^^^^^^
+* Add *subdir* option for :class:`CachedFastDownwardRevision <downward.cached_revision.CachedFastDownwardRevision>` to support nested Fast Downward repos (Jendrik Seipp).
+
+
 v7.3 (2023-03-03)
 -----------------
 

downward/cached_revision.py

@@ -1,5 +1,3 @@
-import glob
-import os.path
 import subprocess
 
 from lab import tools
@@ -12,35 +10,43 @@ class CachedFastDownwardRevision(CachedRevision):
     It provides methods for caching and compiling given revisions.
     """
 
-    def __init__(self, repo, rev, build_options):
+    def __init__(self, revision_cache, repo, rev, build_options, subdir=""):
         """
+        * *revision_cache*: Path to revision cache.
         * *repo*: Path to Fast Downward repository.
         * *rev*: Fast Downward revision.
         * *build_options*: List of build.py options.
+        * *subdir*: relative path from *repo* to Fast Downward subdir.
         """
         CachedRevision.__init__(
-            self, repo, rev, ["./build.py"] + build_options, ["experiments", "misc"]
+            self,
+            revision_cache,
+            repo,
+            rev,
+            ["./build.py"] + build_options,
+            exclude=["experiments", "misc"],
+            subdir=subdir,
         )
+        # Store for easy retrieval by class users.
         self.build_options = build_options
 
     def _cleanup(self):
         # Only keep the bin directories in "builds" dir.
-        for path in glob.glob(os.path.join(self.path, "builds", "*", "*")):
-            if os.path.basename(path) != "bin":
+        for path in self.path.glob("builds/*/*"):
+            if path.name != "bin":
                 tools.remove_path(path)
 
         # Remove unneeded files.
-        tools.remove_path(os.path.join(self.path, "build.py"))
+        tools.remove_path(self.path / "build.py")
 
         # Strip binaries.
         binaries = []
-        for path in glob.glob(os.path.join(self.path, "builds", "*", "bin", "*")):
-            if os.path.basename(path) in ["downward", "preprocess"]:
+        for path in self.path.glob("builds/*/bin/*"):
+            if path.name in ["downward", "preprocess"]:
                 binaries.append(path)
-        subprocess.call(["strip"] + binaries)
+        subprocess.check_call(["strip"] + binaries)
 
         # Compress src directory.
-        subprocess.call(
-            ["tar", "-cf", "src.tar", "--remove-files", "src"], cwd=self.path
+        subprocess.check_call(
+            ["tar", "--remove-files", "--xz", "-cf", "src.tar.xz", "src"], cwd=self.path
         )
-        subprocess.call(["xz", "src.tar"], cwd=self.path)

downward/experiment.py

@@ -302,7 +302,7 @@ def add_algorithm(
         ] + (driver_options or [])
         algorithm = _DownwardAlgorithm(
             name,
-            CachedFastDownwardRevision(repo, rev, build_options),
+            CachedFastDownwardRevision(self.revision_cache, repo, rev, build_options),
             driver_options,
             component_options,
         )
@@ -347,7 +347,7 @@ def _get_unique_cached_revisions(self):
 
     def _cache_revisions(self):
         for cached_rev in self._get_unique_cached_revisions():
-            cached_rev.cache(self.revision_cache)
+            cached_rev.cache()
 
     def _add_code(self):
         """Add the compiled code to the experiment."""

examples/downward/2020-09-11-B-bounded-cost.py

@@ -66,15 +66,14 @@
 
 exp = Experiment(environment=ENV)
 for rev, rev_nick in REVS:
-    cached_rev = CachedFastDownwardRevision(REPO, rev, BUILD_OPTIONS)
-    cached_rev.cache(REVISION_CACHE)
-    cache_path = REVISION_CACHE / cached_rev.name
+    cached_rev = CachedFastDownwardRevision(REVISION_CACHE, REPO, rev, BUILD_OPTIONS)
+    cached_rev.cache()
     dest_path = Path(f"code-{cached_rev.name}")
-    exp.add_resource("", cache_path, dest_path)
+    exp.add_resource("", cached_rev.path, dest_path)
     # Overwrite the script to set an environment variable.
     exp.add_resource(
         _get_solver_resource_name(cached_rev),
-        cache_path / "fast-downward.py",
+        cached_rev.path / "fast-downward.py",
         dest_path / "fast-downward.py",
     )
     for config_nick, config in CONFIGS:

lab/cached_revision.py

@@ -2,6 +2,7 @@
 import hashlib
 import logging
 import os.path
+from pathlib import Path
 import shutil
 import subprocess
 import tarfile
@@ -55,15 +56,18 @@ class CachedRevision:
         Feedback is welcome!
     """
 
-    def __init__(self, repo, rev, build_cmd, exclude=None):
+    def __init__(self, revision_cache, repo, rev, build_cmd, exclude=None, subdir=""):
         """
+        * *revision_cache*: path to revision cache directory.
         * *repo*: path to solver repository.
         * *rev*: solver revision.
         * *build_cmd*: list with build script and any build options
-          (e.g., ``["./build.py", "release"]``, ``["make"]``).
-        * *exclude*: list of paths in repo that are not needed for building
-          and running the solver. Instead of this parameter, you can also
+          (e.g., ``["./build.py", "release"]``, ``["make"]``). Will be executed under
+          *subdir*.
+        * *exclude*: list of relative paths under *subdir* that are not needed for
+          building and running the solver. Instead of this parameter, you can also
           use a ``.gitattributes`` file for Git repositories.
+        * *subdir*: relative path from *repo* to solver subdir.
 
         The following example caches a Fast Downward revision. When you
         use the :class:`FastDownwardExperiment
@@ -76,28 +80,33 @@ def __init__(self, repo, rev, build_cmd, exclude=None):
         >>> revision_cache = os.environ.get("DOWNWARD_REVISION_CACHE")
         >>> if revision_cache:
         ...     rev = "main"
-        ...     cr = CachedRevision(repo, rev, ["./build.py"], exclude=["experiments"])
-        ...     # cr.cache(revision_cache)  # Uncomment to actually cache the code.
+        ...     cr = CachedRevision(
+        ...         revision_cache, repo, rev, ["./build.py"], exclude=["experiments"]
+        ...     )
+        ...     # cr.cache()  # Uncomment to actually cache the code.
         ...
 
         You can now copy the cached repo to your experiment:
 
         ...     from lab.experiment import Experiment
         ...     exp = Experiment()
-        ...     cache_path = os.path.join(revision_cache, cr.name)
-        ...     dest_path = os.path.join(exp.path, "code-" + cr.name)
-        ...     exp.add_resource("solver_" +  cr.name, cache_path, dest_path)
+        ...     dest_path = os.path.join(exp.path, f"code-{cr.name}")
+        ...     exp.add_resource(f"solver_{cr.name}", cr.path, dest_path)
 
         """
         if not os.path.isdir(repo):
             logging.critical(f"{repo} is not a local solver repository.")
+        self.revision_cache = Path(revision_cache)
         self.repo = repo
+        self.subdir = subdir
         self.build_cmd = build_cmd
         self.local_rev = rev
         self.global_rev = get_global_rev(repo, rev)
-        self.path = None
         self.exclude = exclude or []
         self.name = self._compute_hashed_name()
+        self.path = self.revision_cache / self.name
+        # Temporary directory for preparing the checkout.
+        self._tmp_path = self.revision_cache / f"{self.name}-tmp"
 
     def __eq__(self, other):
         return self.name == other.name
@@ -106,10 +115,10 @@ def __hash__(self):
         return hash(self.name)
 
     def _compute_hashed_name(self):
-        return f"{self.global_rev}_{_compute_md5_hash(self.build_cmd + self.exclude)}"
+        options_hash = _compute_md5_hash(self.build_cmd + self.exclude + [self.subdir])
+        return f"{self.global_rev}_{options_hash}"
 
-    def cache(self, revision_cache):
-        self.path = os.path.join(revision_cache, self.name)
+    def cache(self):
         if os.path.exists(self.path):
             logging.info(f'Revision is already cached: "{self.path}"')
             if not os.path.exists(self._get_sentinel_file()):
@@ -118,37 +127,43 @@ def cache(self, revision_cache):
                     f"Please delete it and try again."
                 )
         else:
-            tools.makedirs(self.path)
-            tar_archive = os.path.join(self.path, "solver.tgz")
+            tools.remove_path(self._tmp_path)
+            tools.makedirs(self._tmp_path)
+            tar_archive = os.path.join(self._tmp_path, "solver.tgz")
             cmd = ["git", "archive", "--format", "tar", self.global_rev]
             with open(tar_archive, "w") as f:
                 retcode = tools.run_command(cmd, stdout=f, cwd=self.repo)
 
-            if retcode == 0:
-                with tarfile.open(tar_archive) as tf:
-                    tf.extractall(self.path)
-                tools.remove_path(tar_archive)
-
-                for exclude_dir in self.exclude:
-                    path = os.path.join(self.path, exclude_dir)
-                    if os.path.exists(path):
-                        tools.remove_path(path)
-
             if retcode != 0:
-                shutil.rmtree(self.path)
                 logging.critical("Failed to make checkout.")
-            self._compile()
+
+            # Extract only the subdir.
+            with tarfile.open(tar_archive) as tf:
+
+                def members():
+                    for tarinfo in tf.getmembers():
+                        if tarinfo.name.startswith(self.subdir):
+                            yield tarinfo
+
+                tf.extractall(self._tmp_path, members=members())
+            shutil.move(self._tmp_path / self.subdir, self.path)
+            tools.remove_path(tar_archive)
+
+            for exclude_path in self.exclude:
+                path = self.path / exclude_path
+                if path.exists():
+                    tools.remove_path(path)
+
+            retcode = tools.run_command(self.build_cmd, cwd=self.path)
+            if retcode == 0:
+                tools.write_file(self._get_sentinel_file(), "")
+            else:
+                logging.critical(f"Build failed in {self.path}")
+
             self._cleanup()
 
     def _get_sentinel_file(self):
         return os.path.join(self.path, "build_successful")
 
-    def _compile(self):
-        retcode = tools.run_command(self.build_cmd, cwd=self.path)
-        if retcode == 0:
-            tools.write_file(self._get_sentinel_file(), "")
-        else:
-            logging.critical(f"Build failed in {self.path}")
-
     def _cleanup(self):
         pass

lab/tools.py

@@ -155,12 +155,10 @@ def confirm_overwrite_or_abort(path):
 
 
 def remove_path(path):
-    if os.path.isfile(path):
-        try:
-            os.remove(path)
-        except OSError:
-            pass
-    else:
+    path = Path(path)
+    if path.is_file():
+        path.unlink()
+    elif path.is_dir():
         shutil.rmtree(path)