Tab completed loading (#951)

* Black line length

* Add tests for loading and inspecting

* Introduce and test helper classes for loading and inspecting

* Don't deprecate the conversion argument

It gets used in iter methods in a reasonable way

* Use just inspect or load if conversion is not a variable

* Format black

* Update pyiron_base/project/generic.py

Co-authored-by: Marvin Poul <poul@mpie.de>

* Break test into subtests

* Directly check getattr, as suggested by @pmrv

* Move loader classes to their own module, per @jan-janssen's suggestion

* Use full import path

* Move tests to their own test module

* Break sub tests into separate tests

* PEP8

* Also work without a database, per @jan-janssen's concern

* Make sure loader values are always synced with project values

The project knows how to stay up-to-date with the state; instead of re-inventing this security, let's just directly use the project attributes.

* Store and restore all the settings

Let's see if this recovers the `project_path` configuration setting that the projectpath tests seem to be relying on

* Format black

* Add logging to try and debug remote ci

* Remove sibling tests

* Add path to debug print

* More logging

* More logging

* Log more about the loader project

* Add filters progressively in logging

* Log the filetable path

* Log the actual folder contents

* Narrow the search

* Just look at length and look before turning off the database too

* Log the actual recursion check results

* Revert changes, the problem is upstream in FileTable still

I don't see trouble on my local machine because I don't specify TOP_LEVEL_DIRS in my config but the CI does.

* Resolve some codacy nits

* Update class descriptions

* Test sub-project loading

* Update tests to account for extra job in TestWithFilledProject

Making some effort to avoid magic numbers

---------

Co-authored-by: pyiron-runner <pyiron@mpie.de>
Co-authored-by: Marvin Poul <poul@mpie.de>

pyiron_base/_tests.py

@@ -158,12 +158,19 @@ def setUpClass(cls):
         job.run()
         job.status.aborted = True
 
-        with cls.project.open("sub_project") as pr_sub:
-            job = pr_sub.create_job(job_type=ToyJob, job_name="toy_1")
-            job.run()
-            job = pr_sub.create_job(job_type=ToyJob, job_name="toy_2")
-            job.run()
-            job.status.suspended = True
+        cls.pr_sub = cls.project.open("sub_project")
+        job = cls.pr_sub.create_job(job_type=ToyJob, job_name="toy_1")
+        job.run()
+        job = cls.pr_sub.create_job(job_type=ToyJob, job_name="toy_2")
+        job.run()
+        job.status.suspended = True
+        job = cls.pr_sub.create_job(job_type=ToyJob, job_name="toy_3")
+        job.run()
+
+        cls.n_jobs_filled_with = 5
+        # In a number of tests we compare the found jobs to an expected number of jobs
+        # Let's code that number once here instead of magic-numbering it throughout
+        # the tests
 
 
 _TO_SKIP = [

pyiron_base/jobs/job/core.py

@@ -422,7 +422,7 @@ def remove(self, _protect_childs=True):
                     )
                     raise ValueError("Child jobs are protected and cannot be deleted!")
             for job_id in self.child_ids:
-                job = self.project.load(job_id, convert_to_object=False)
+                job = self.project.inspect(job_id)
                 if len(job.child_ids) > 0:
                     job.remove(_protect_childs=False)
                 else:
@@ -547,7 +547,7 @@ def inspect(self, job_specifier):
         Returns:
             JobCore: Access to the HDF5 object - not a GenericJob object - use load() instead.
         """
-        return self.project.load(job_specifier=job_specifier, convert_to_object=False)
+        return self.project.inspect(job_specifier=job_specifier)
 
     def is_master_id(self, job_id):
         """
@@ -616,10 +616,7 @@ def list_childs(self):
         Returns:
             list: list of child jobs
         """
-        return [
-            self.project.load(child_id, convert_to_object=False).job_name
-            for child_id in self.child_ids
-        ]
+        return [self.project.inspect(child_id).job_name for child_id in self.child_ids]
 
     def _list_groups(self):
         return self.project_hdf5.list_groups() + self._list_ext_childs()

pyiron_base/jobs/master/generic.py

@@ -460,7 +460,7 @@ def _get_item_when_str(self, item, child_id_lst, child_name_lst):
             if len(name_lst) > 1:
                 return self.project.inspect(child_id)["/".join(name_lst[1:])]
             else:
-                return self.project.load(child_id, convert_to_object=True)
+                return self.project.load(child_id)
         elif item_obj in self._job_name_lst:
             child = self._load_job_from_cache(job_name=item_obj)
             if len(name_lst) == 1:

pyiron_base/jobs/master/serial.py

@@ -123,7 +123,7 @@ def create_next(self, job_name=None):
             raise ValueError("No job available in job list, please append a job first.")
         if len(self._job_name_lst) > len(self.child_ids):
             return self.pop(-1)
-        ham_old = self.project.load(self.child_ids[-1], convert_to_object=True)
+        ham_old = self.project.load(self.child_ids[-1])
 
         if ham_old.status.aborted:
             ham_old.status.created = True
@@ -206,7 +206,7 @@ def get_from_childs(self, path):
         """
         var_lst = []
         for child_id in self.child_ids:
-            ham = self.project.load(child_id, convert_to_object=False)
+            ham = self.project.inspect(child_id)
             var = ham.__getitem__(path)
             var_lst.append(var)
         return np.array(var_lst)

pyiron_base/project/generic.py

@@ -5,6 +5,8 @@
 The project object is the central import point of pyiron - all other objects can be created from this one
 """
 
+from __future__ import annotations
+
 import os
 import posixpath
 import shutil
@@ -15,6 +17,7 @@
 import math
 import numpy as np
 
+from pyiron_base.project.jobloader import JobLoader, JobInspector
 from pyiron_base.project.maintenance import Maintenance
 from pyiron_base.project.path import ProjectPath
 from pyiron_base.database.filetable import FileTable
@@ -47,7 +50,10 @@
 from pyiron_base.project.external import Notebook
 from pyiron_base.project.data import ProjectData
 from pyiron_base.project.archiving import export_archive, import_archive
-from typing import Generator, Union, Dict
+from typing import Generator, Union, Dict, TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from pyiron_base.jobs.job.generic import GenericJob
 
 __author__ = "Joerg Neugebauer, Jan Janssen"
 __copyright__ = (
@@ -122,6 +128,8 @@ def __init__(
         self._inspect_mode = False
         self._data = None
         self._creator = Creator(project=self)
+        self._loader = JobLoader(project=self)
+        self._inspector = JobInspector(project=self)
 
         self.job_type = JobTypeChoice()
 
@@ -539,17 +547,9 @@ def groups(self):
         new._filter = ["groups"]
         return new
 
-    def inspect(self, job_specifier):
-        """
-        Inspect an existing pyiron object - most commonly a job - from the database
-
-        Args:
-            job_specifier (str, int): name of the job or job ID
-
-        Returns:
-            JobCore: Access to the HDF5 object - not a GenericJob object - use load() instead.
-        """
-        return self.load(job_specifier=job_specifier, convert_to_object=False)
+    @property
+    def inspect(self):
+        return self._inspector
 
     def iter_jobs(
         self,
@@ -586,7 +586,7 @@ def iter_jobs(
             job_id_lst = tqdm(job_id_lst)
         for job_id in job_id_lst:
             if path is not None:
-                yield self.load(job_id, convert_to_object=False)[path]
+                yield self.inspect(job_id)[path]
             else:  # Backwards compatibility - in future the option convert_to_object should be removed
                 yield self.load(job_id, convert_to_object=convert_to_object)
 
@@ -822,34 +822,9 @@ def _list_nodes(self, recursive=False):
             return []
         return self.get_jobs(recursive=recursive, columns=["job"])["job"]
 
-    def load(self, job_specifier, convert_to_object=True):
-        """
-        Load an existing pyiron object - most commonly a job - from the database
-
-        Args:
-            job_specifier (str, int): name of the job or job ID
-            convert_to_object (bool): convert the object to an pyiron object or only access the HDF5 file - default=True
-                                      accessing only the HDF5 file is about an order of magnitude faster, but only
-                                      provides limited functionality. Compare the GenericJob object to JobCore object.
-
-        Returns:
-            GenericJob, JobCore: Either the full GenericJob object or just a reduced JobCore object
-        """
-        if self.sql_query is not None:
-            state.logger.warning(
-                "SQL filter '%s' is active (may exclude job) ", self.sql_query
-            )
-        if not isinstance(job_specifier, (int, np.integer)):
-            job_specifier = _get_safe_job_name(name=job_specifier)
-        job_id = self.get_job_id(job_specifier=job_specifier)
-        if job_id is None:
-            state.logger.warning(
-                "Job '%s' does not exist and cannot be loaded", job_specifier
-            )
-            return None
-        return self.load_from_jobpath(
-            job_id=job_id, convert_to_object=convert_to_object
-        )
+    @property
+    def load(self):
+        return self._loader
 
     def load_from_jobpath(self, job_id=None, db_entry=None, convert_to_object=True):
         """
@@ -1133,9 +1108,7 @@ def remove_job(self, job_specifier, _unprotect=False):
         else:
             if not self.db.view_mode:
                 try:
-                    job = self.load(
-                        job_specifier=job_specifier, convert_to_object=False
-                    )
+                    job = self.inspect(job_specifier=job_specifier)
                     if job is None:
                         state.logger.warning(
                             "Job '%s' does not exist and could not be removed",

pyiron_base/project/jobloader.py

@@ -0,0 +1,120 @@
+# coding: utf-8
+# Copyright (c) Max-Planck-Institut für Eisenforschung GmbH - Computational Materials Design (CM) Department
+# Distributed under the terms of "New BSD License", see the LICENSE file.
+"""
+A helper class to be assigned to the project, which facilitates tab-completion when
+loading jobs.
+"""
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from typing import TYPE_CHECKING
+
+import numpy as np
+
+from pyiron_base.state import state
+from pyiron_base.database.jobtable import get_job_id
+from pyiron_base.jobs.job.util import _get_safe_job_name
+
+if TYPE_CHECKING:
+    from pyiron_base.jobs.job.generic import GenericJob
+    from pyiron_base.jobs.job.path import JobPath
+    from pyiron_base.project.generic import Project
+
+
+class _JobByAttribute(ABC):
+    """
+    A parent class for accessing project jobs by a call and a job specifier, or by tab
+    completion.
+    """
+
+    def __init__(self, project: Project):
+        self._project = project
+
+    @property
+    def _job_table(self):
+        return self._project.job_table(columns=["job"])
+
+    @property
+    def _job_names(self):
+        return self._job_table["job"].values
+
+    def __dir__(self):
+        return self._job_names
+
+    def _id_from_name(self, name):
+        return self._job_table.loc[self._job_names == name, "id"].values[0]
+
+    def __getattr__(self, item):
+        return self._project.load_from_jobpath(
+            job_id=self._id_from_name(item), convert_to_object=self.convert_to_object
+        )
+
+    def __getitem__(self, item):
+        return self.__getattr__(item)
+
+    def __call__(self, job_specifier, convert_to_object=None):
+        if self._project.sql_query is not None:
+            state.logger.warning(
+                f"SQL filter '{self._project.sql_query}' is active (may exclude job)"
+            )
+        if not isinstance(job_specifier, (int, np.integer)):
+            job_specifier = _get_safe_job_name(name=job_specifier)
+        job_id = get_job_id(
+            database=self._project.db,
+            sql_query=self._project.sql_query,
+            user=self._project.user,
+            project_path=self._project.project_path,
+            job_specifier=job_specifier,
+        )
+        if job_id is None:
+            state.logger.warning(
+                f"Job '{job_specifier}' does not exist and cannot be loaded"
+            )
+            return None
+        return self._project.load_from_jobpath(
+            job_id=job_id,
+            convert_to_object=convert_to_object
+            if convert_to_object is not None
+            else self.convert_to_object,
+        )
+
+    @property
+    @abstractmethod
+    def convert_to_object(self):
+        pass
+
+
+class JobLoader(_JobByAttribute):
+    """
+    Load an existing pyiron object - most commonly a job - from the database
+
+    Args:
+        job_specifier (str, int): name of the job or job ID
+
+    Returns:
+        GenericJob, JobCore: Either the full GenericJob object or just a reduced JobCore object
+    """
+
+    convert_to_object = True
+
+    def __call__(self, job_specifier, convert_to_object=None) -> GenericJob:
+        return super().__call__(job_specifier, convert_to_object=convert_to_object)
+
+
+class JobInspector(_JobByAttribute):
+    """
+    Inspect an existing pyiron object - most commonly a job - from the database
+
+    Args:
+        job_specifier (str, int): name of the job or job ID
+
+    Returns:
+        JobCore: Access to the HDF5 object - not a GenericJob object - use :meth:`~.Project.load()`
+            instead.
+    """
+
+    convert_to_object = False
+
+    def __call__(self, job_specifier) -> JobPath:
+        return super().__call__(job_specifier)