Allow tracking of component-specific execution data (#460)

docs/examples/slurm-and-mpi-execution/mpi.py

@@ -13,7 +13,7 @@ class Config:
     def __call__(self):
         for executable in self.pending_executables:
             script_file = self.save_file(
-                f"mpi-{executable.id}.sh",
+                [executable, "mpi.sh"],
                 executable.dispatch_code(),
             )
             st = os.stat(script_file)

docs/examples/slurm-and-mpi-execution/slurm.py

@@ -7,24 +7,28 @@
 class Slurm(Execution):
     def __call__(self):
         script = "#!/usr/bin/env bash\n"
-        for component in self.pending_executables:
-            resources = component.resources()
+        for executable in self.pending_executables:
+            resources = executable.resources()
             if "--job-name" not in resources:
-                resources["--job-name"] = f"{component.id}"
+                resources["--job-name"] = f"{executable.id}"
             if "--output" not in resources:
-                resources["--output"] = component.local_directory("output.log")
+                resources["--output"] = self.local_directory(
+                    executable.id, "output.log"
+                )
             if "--open-mode" not in resources:
                 resources["--open-mode"] = "append"
 
             sbatch_arguments = []
             for k, v in resources.items():
+                if not k.startswith("--"):
+                    continue
                 line = "#SBATCH " + k
                 if v not in [None, True]:
                     line += f"={v}"
                 sbatch_arguments.append(line)
             script += "\n".join(sbatch_arguments) + "\n"
 
-            script += component.dispatch_code()
+            script += executable.dispatch_code()
 
             # submit to slurm
             process = subprocess.Popen(
@@ -50,13 +54,13 @@ def __call__(self):
             except ValueError:
                 job_id = False
             print(
-                f"{output} for component {component.id} ({component.local_directory()})"
+                f"{output} for component {executable.id} ({executable.local_directory()})"
             )
 
             # save job information
             self.save_file(
-                filepath="slurm.json",
-                data={
+                [executable, "slurm.json"],
+                {
                     "job_id": job_id,
                     "cmd": sbatch_arguments,
                     "script": script,

docs/examples/slurm-and-mpi-execution/test_mpi.py

@@ -27,5 +27,5 @@ def test_mpi_execution(tmp_path):
             component = MpiExample()
             with Execution.get("mpi"):
                 component.launch()
-            assert component.is_finished()
+            assert component.execution.is_finished()
             assert component.load_file("test.txt") == "hello"

docs/examples/slurm-and-mpi-execution/test_slurm.py

@@ -37,7 +37,7 @@ def test_slurm_execution(tmp_path):
                 component.launch()
 
             for _ in range(60):
-                if component.is_finished():
+                if component.execution.is_finished():
                     assert "Hello world from Slurm" in component.output()
                     assert (
                         component.load_file("test_run.json")["success"] is True

docs/guide/component.md

@@ -37,7 +37,7 @@ Downloading 'mnist' ...
 If the execution is successful, the component is marked as finished.
 
 ```python
->>> mnist.is_finished()
+>>> mnist.execution.is_finished()
 True
 ```
 

docs/guide/introduction.md

@@ -35,7 +35,7 @@ class EstimatePi(Component):
         )
 
     def summary(self):
-        if self.is_finished():
+        if self.execution.is_finished():
             print(
                 f"After {self.config.samples} samples, "
                 f"PI is approximately {self.load_file('result.json')['pi']}."

pyproject.toml

@@ -28,7 +28,7 @@ flatten-dict = "^0.4"
 jsonlines = "^3.1"
 pydantic = "^1.10.8"
 arrow = "^1.2"
-importlib-metadata = {version = "^6.6", python = "<3.8"}
+importlib-metadata = {version = "^6.7", python = "<3.8"}
 omegaconf = "^2.3.0"
 dill = "^0.3.6"
 typing-extensions = {version = "^4.5.0", python = "<3.11"}
@@ -40,7 +40,7 @@ pyupgrade = "^3.3"
 black = "^23.3.0"
 pytest = "^7.3"
 pre-commit = "^2.21.0"
-editorconfig-checker = "^2.7.1"
+editorconfig-checker = "^2.7.2"
 pytest-cov = "^4.1.0"
 
 [tool.poetry.extras]

src/machinable/__init__.py

@@ -39,7 +39,7 @@
 def get(
     module: Union[str, Element, None] = None,
     version: VersionType = None,
-    predicate: Optional[str] = get_settings().default_predicate,
+    predicate: Optional[str] = "$",
     **kwargs,
 ) -> Interface:
     return Interface.get(module, version, predicate, **kwargs)

src/machinable/collection.py

@@ -9,7 +9,6 @@
 from json import dumps
 from pprint import pprint
 
-from machinable.settings import get_settings
 from machinable.types import VersionType
 
 long = int
@@ -1384,7 +1383,7 @@ def filter_by_predicate(
         self,
         module: str,
         version: VersionType = None,
-        predicate: str = get_settings().default_predicate,
+        predicate: str = "$",
         **kwargs,
     ):
         from machinable import Element
@@ -1407,7 +1406,7 @@ def singleton(
         self,
         module: str,
         version: VersionType = None,
-        predicate: str = get_settings().default_predicate,
+        predicate: str = "$",
         **kwargs,
     ) -> Union[Any, "Component"]:
         from machinable import Element
@@ -1445,6 +1444,11 @@ def launch(self) -> "ComponentCollection":
 
         return self
 
+
+class ExecutionCollection(ElementCollection):
+    def __str__(self):
+        return f"Executions <{len(self.items)}>"
+
     def status(self, status="started"):
         """Filters the collection by a status attribute
 
@@ -1455,8 +1459,3 @@ def status(self, status="started"):
             return self.filter(lambda item: getattr(item, "is_" + status)())
         except AttributeError as _ex:
             raise ValueError(f"Invalid status field: {status}") from _ex
-
-
-class ExecutionCollection(ElementCollection):
-    def __str__(self):
-        return f"Executions <{len(self.items)}>"