Merge pull request #1400 from pyiron/compress_ex

GenericJob: extend the compress() function

pyiron_base/jobs/job/core.py

@@ -209,6 +209,8 @@ def __init__(self, project, job_name):
         self._import_directory = None
         self._database_property = DatabaseProperties()
         self._hdf5_content = HDF5Content(project_hdf5=self._hdf5)
+        self._files_to_remove = list()
+        self._files_to_compress = list()
 
     @property
     def content(self):
@@ -385,6 +387,14 @@ def project_hdf5(self, project):
         """
         self._hdf5 = project.copy()
 
+    @property
+    def files_to_compress(self):
+        return self._files_to_compress
+
+    @property
+    def files_to_remove(self):
+        return self._files_to_remove
+
     def relocate_hdf5(self, h5_path=None):
         """
         Relocate the hdf file. This function is needed when the child job is
@@ -1061,14 +1071,22 @@ def _list_ext_childs(self):
         childs = self.list_childs()
         return list(set(childs) - set(nodes))
 
-    def compress(self, files_to_compress=None):
+    def compress(self, files_to_compress=None, files_to_remove=None):
         """
         Compress the output files of a job object.
 
         Args:
             files_to_compress (list):
         """
-        _job_compress(job=self, files_to_compress=files_to_compress)
+        if files_to_compress is None and len(self._files_to_compress) != 0:
+            files_to_compress = self._files_to_compress
+        elif files_to_compress is None:
+            files_to_compress = self.files.list()
+        if files_to_remove is None:
+            files_to_remove = self._files_to_remove
+        else:
+            files_to_remove = []
+        _job_compress(job=self, files_to_compress=files_to_compress, files_to_remove=files_to_remove)
 
     def decompress(self):
         """

pyiron_base/jobs/job/generic.py

@@ -1069,6 +1069,10 @@ def to_dict(self):
             data_dict["import_directory"] = self._import_directory
         if self._executor_type is not None:
             data_dict["executor_type"] = self._executor_type
+        if len(self._files_to_compress) > 0:
+            data_dict["files_to_compress"] = self._files_to_compress
+        if len(self._files_to_remove) > 0:
+            data_dict["files_to_compress"] = self._files_to_remove
         return data_dict
 
     def from_dict(self, job_dict):

pyiron_base/jobs/job/util.py

@@ -275,17 +275,25 @@ def _get_compressed_job_name(working_directory):
     )
 
 
-def _job_compress(job, files_to_compress=None):
+def _job_compress(job, files_to_compress=[], files_to_remove=[]):
     """
     Compress the output files of a job object.
 
     Args:
         job (JobCore): job object to compress
         files_to_compress (list): list of files to compress
+        files_to_remove (list): list of files to remove
     """
+
+    def delete_file_or_folder(fullname):
+        if os.path.isfile(fullname):
+            os.remove(fullname)
+        elif os.path.isdir(fullname):
+            shutil.rmtree(fullname)
+
     if not _job_is_compressed(job):
-        if files_to_compress is None:
-            files_to_compress = job.files.list()
+        for name in files_to_remove:
+            delete_file_or_folder(fullname=os.path.join(job.working_directory, name))
         cwd = os.getcwd()
         try:
             os.chdir(job.working_directory)
@@ -294,12 +302,10 @@ def _job_compress(job, files_to_compress=None):
                     if "tar" not in name and not stat.S_ISFIFO(os.stat(name).st_mode):
                         tar.add(name)
             for name in files_to_compress:
-                if "tar" not in name:
-                    fullname = os.path.join(job.working_directory, name)
-                    if os.path.isfile(fullname):
-                        os.remove(fullname)
-                    elif os.path.isdir(fullname):
-                        shutil.rmtree(fullname)
+                if name != _job_compressed_name(job):
+                    delete_file_or_folder(
+                        fullname=os.path.join(job.working_directory, name)
+                    )
         finally:
             os.chdir(cwd)
     else: