Merge remote-tracking branch 'refs/remotes/origin/main' into pack_fixes

.ci_support/environment-docs.yml

@@ -8,17 +8,17 @@ dependencies:
 - numpy
 - cloudpickle =3.0.0
 - gitpython =3.1.43
-- h5io_browser =0.0.10
-- h5py =3.10.0
+- h5io_browser =0.0.12
+- h5py =3.11.0
 - jinja2 =3.1.3
 - monty =2024.3.31
 - numpy =1.26.4
-- pandas =2.2.1
+- pandas =2.2.2
 - pint =0.23
 - psutil =5.9.8
-- pyfileindex =0.0.23
+- pyfileindex =0.0.24
 - pympipool =0.7.17
-- pysqa =0.1.17
+- pysqa =0.1.18
 - pytables =3.9.2
 - sqlalchemy =2.0.29
 - tqdm =4.66.2

.ci_support/environment.yml

@@ -7,17 +7,17 @@ dependencies:
 - conda_subprocess =0.0.1
 - cloudpickle =3.0.0
 - gitpython =3.1.43
-- h5io_browser =0.0.10
-- h5py =3.10.0
+- h5io_browser =0.0.12
+- h5py =3.11.0
 - jinja2 =3.1.3
 - monty =2024.3.31
 - numpy =1.26.4
-- pandas =2.2.1
+- pandas =2.2.2
 - pint =0.23
 - psutil =5.9.8
-- pyfileindex =0.0.23
+- pyfileindex =0.0.24
 - pympipool =0.7.17
-- pysqa =0.1.17
+- pysqa =0.1.18
 - pytables =3.9.2
 - sqlalchemy =2.0.29
 - tqdm =4.66.2

.github/workflows/atomistics-compat.yml

@@ -20,18 +20,18 @@ jobs:
       matrix:
         include:
           - operating-system: macos-latest
-            python-version: '3.11'
-            label: osx-64-py-3-10
+            python-version: '3.12'
+            label: osx-64-py-3-12
             prefix: /Users/runner/miniconda3/envs/my-env
 
           - operating-system: windows-latest
-            python-version: '3.11'
-            label: win-64-py-3-10
+            python-version: '3.12'
+            label: win-64-py-3-12
             prefix: C:\Miniconda3\envs\my-env
 
           - operating-system: ubuntu-latest
-            python-version: '3.11'
-            label: linux-64-py-3-10
+            python-version: '3.12'
+            label: linux-64-py-3-12
             prefix: /usr/share/miniconda3/envs/my-env
 
     steps:

.github/workflows/notebooks.yml

@@ -19,7 +19,7 @@ jobs:
         tail --lines=+4 .ci_support/environment-notebooks.yml >> environment.yml
     - uses: conda-incubator/setup-miniconda@v2.2.0
       with:
-        python-version: "3.11"
+        python-version: "3.12"
         mamba-version: "*"
         channels: conda-forge
         miniforge-variant: Mambaforge

pyiron_base/jobs/job/core.py

@@ -209,6 +209,8 @@ def __init__(self, project, job_name):
         self._import_directory = None
         self._database_property = DatabaseProperties()
         self._hdf5_content = HDF5Content(project_hdf5=self._hdf5)
+        self._files_to_remove = list()
+        self._files_to_compress = list()
 
     @property
     def content(self):
@@ -385,6 +387,14 @@ def project_hdf5(self, project):
         """
         self._hdf5 = project.copy()
 
+    @property
+    def files_to_compress(self):
+        return self._files_to_compress
+
+    @property
+    def files_to_remove(self):
+        return self._files_to_remove
+
     def relocate_hdf5(self, h5_path=None):
         """
         Relocate the hdf file. This function is needed when the child job is
@@ -1061,14 +1071,22 @@ def _list_ext_childs(self):
         childs = self.list_childs()
         return list(set(childs) - set(nodes))
 
-    def compress(self, files_to_compress=None):
+    def compress(self, files_to_compress=None, files_to_remove=None):
         """
         Compress the output files of a job object.
 
         Args:
             files_to_compress (list):
         """
-        _job_compress(job=self, files_to_compress=files_to_compress)
+        if files_to_compress is None and len(self._files_to_compress) != 0:
+            files_to_compress = self._files_to_compress
+        elif files_to_compress is None:
+            files_to_compress = self.files.list()
+        if files_to_remove is None:
+            files_to_remove = self._files_to_remove
+        else:
+            files_to_remove = []
+        _job_compress(job=self, files_to_compress=files_to_compress, files_to_remove=files_to_remove)
 
     def decompress(self):
         """

pyiron_base/jobs/job/generic.py

@@ -1069,6 +1069,10 @@ def to_dict(self):
             data_dict["import_directory"] = self._import_directory
         if self._executor_type is not None:
             data_dict["executor_type"] = self._executor_type
+        if len(self._files_to_compress) > 0:
+            data_dict["files_to_compress"] = self._files_to_compress
+        if len(self._files_to_remove) > 0:
+            data_dict["files_to_compress"] = self._files_to_remove
         return data_dict
 
     def from_dict(self, job_dict):

pyiron_base/jobs/job/util.py

@@ -275,17 +275,25 @@ def _get_compressed_job_name(working_directory):
     )
 
 
-def _job_compress(job, files_to_compress=None):
+def _job_compress(job, files_to_compress=[], files_to_remove=[]):
     """
     Compress the output files of a job object.
 
     Args:
         job (JobCore): job object to compress
         files_to_compress (list): list of files to compress
+        files_to_remove (list): list of files to remove
     """
+
+    def delete_file_or_folder(fullname):
+        if os.path.isfile(fullname):
+            os.remove(fullname)
+        elif os.path.isdir(fullname):
+            shutil.rmtree(fullname)
+
     if not _job_is_compressed(job):
-        if files_to_compress is None:
-            files_to_compress = job.files.list()
+        for name in files_to_remove:
+            delete_file_or_folder(fullname=os.path.join(job.working_directory, name))
         cwd = os.getcwd()
         try:
             os.chdir(job.working_directory)
@@ -294,12 +302,10 @@ def _job_compress(job, files_to_compress=None):
                     if "tar" not in name and not stat.S_ISFIFO(os.stat(name).st_mode):
                         tar.add(name)
             for name in files_to_compress:
-                if "tar" not in name:
-                    fullname = os.path.join(job.working_directory, name)
-                    if os.path.isfile(fullname):
-                        os.remove(fullname)
-                    elif os.path.isdir(fullname):
-                        shutil.rmtree(fullname)
+                if name != _job_compressed_name(job):
+                    delete_file_or_folder(
+                        fullname=os.path.join(job.working_directory, name)
+                    )
         finally:
             os.chdir(cwd)
     else:

pyiron_base/storage/flattenedstorage.py

@@ -29,17 +29,22 @@
 from pyiron_base.interfaces.lockable import Lockable, sentinel
 
 
+_CHARSIZE = np.dtype("U1").itemsize
+
+
 def _ensure_str_array_size(array, strlen):
     """
     Ensures that the given array can store at least string of length `strlen`.
 
     Args:
         array (ndarray): array of dtype <U
-        strlen (int): maximum length that should fit in it
+        strlen (int, ndarray): maximum length that should fit in it
     Returns:
         ndarray: either `array` or resized copy
     """
-    current_length = array.itemsize // np.dtype("1U").itemsize
+    current_length = array.itemsize // _CHARSIZE
+    if isinstance(strlen, np.ndarray):
+        strlen = strlen.itemsize // _CHARSIZE
     if current_length < strlen:
         return array.astype(f"{2 * strlen}U")
     else:
@@ -938,6 +943,10 @@ def extend(self, other: "FlattenedStorage"):
                 self.add_array(
                     name=k, dtype=dtype, shape=a.shape[1:], fill=fill, per="chunk"
                 )
+            elif a.dtype.char == "U":
+                self._per_chunk_arrays[k] = _ensure_str_array_size(
+                    self._per_chunk_arrays[k], a
+                )
             self._per_chunk_arrays[k][self.num_chunks : combined_num_chunks] = a[
                 0 : other.num_chunks
             ]
@@ -948,6 +957,10 @@ def extend(self, other: "FlattenedStorage"):
                 self.add_array(
                     name=k, shape=a.shape[1:], dtype=dtype, fill=fill, per="element"
                 )
+            elif a.dtype.char == "U":
+                self._per_element_arrays[k] = _ensure_str_array_size(
+                    self._per_element_arrays[k], a
+                )
             self._per_element_arrays[k][self.num_elements : combined_num_elements] = a[
                 0 : other.num_elements
             ]

pyproject.toml

@@ -11,7 +11,7 @@ authors = [
 readme = "README.rst"
 license = { file = "LICENSE" }
 keywords = ["pyiron"]
-requires-python = ">=3.8"
+requires-python = ">=3.9, <3.13"
 classifiers = [
     "Development Status :: 5 - Production/Stable",
     "Topic :: Scientific/Engineering :: Physics",
@@ -26,17 +26,17 @@ classifiers = [
 dependencies = [
     "cloudpickle==3.0.0",
     "gitpython==3.1.43",
-    "h5io_browser==0.0.10",
-    "h5py==3.10.0",
+    "h5io_browser==0.0.12",
+    "h5py==3.11.0",
     "jinja2==3.1.3",
     "numpy==1.26.4",
     "monty==2024.3.31",
-    "pandas==2.2.1",
+    "pandas==2.2.2",
     "pint==0.23",
     "psutil==5.9.8",
-    "pyfileindex==0.0.23",
+    "pyfileindex==0.0.24",
     "pympipool==0.7.17",
-    "pysqa==0.1.17",
+    "pysqa==0.1.18",
     "sqlalchemy==2.0.29",
     "tables==3.9.2",
     "tqdm==4.66.2",