Improve lazy loading caveats

- Improve documentation of lazy loading (caveats section)
- Add additional checks and options when working with lazy-loaded IDSs

Author: maarten-ic

docs/source/lazy_loading.rst

@@ -35,12 +35,45 @@ Caveats of lazy loaded IDSs
 
 Lazy loading of data may speed up your programs, but also comes with some limitations.
 
-1.  IMASPy **assumes** that the underlying data entry is not modified.
+1.  Some functionality is not implemented or works differently for lazy-loaded IDSs:
+
+    -   Iterating over non-empty nodes works differently, see API documentation:
+        :py:meth:`imaspy.ids_structure.IDSStructure.iter_nonempty_`.
+    -   :py:meth:`~imaspy.ids_structure.IDSStructure.has_value` is not implemented for
+        lazy-loaded structure elements.
+    -   :py:meth:`~imaspy.ids_toplevel.IDSToplevel.validate` will only validate loaded
+        data. Additional data might be loaded from the backend to validate coordinate
+        sizes.
+    -   :py:meth:`imaspy.util.print_tree` will only print data that is loaded when
+        :py:param:`~imaspy.util.print_tree.hide_empty_nodes` is ``True``.
+    -   :py:meth:`imaspy.util.visit_children`:
+
+        -   When :py:param:`~imaspy.util.visit_children.visit_empty` is ``False``
+            (default), this method uses
+            :py:meth:`~imaspy.ids_structure.IDSStructure.iter_nonempty_`. This raises an
+            error for lazy-loaded IDSs, unless you set
+            :py:param:`~imaspy.util.visit_children.accept_lazy` to ``True``.
+        -   When :py:param:`~imaspy.util.visit_children.visit_empty` is ``True``, this
+            will iteratively load `all` data from the backend. This is effectively a
+            full, but less efficient, ``get()``\ /\ ``get_slice()``. It will be faster
+            if you don't use lazy loading in this case.
+
+    -   IDS conversion through :py:meth:`imaspy.convert_ids
+        <imaspy.ids_convert.convert_ids>` is not implemented for lazy loaded IDSs. Note
+        that :ref:`Automatic conversion between DD versions` also applies when lazy
+        loading.
+    -   Lazy loaded IDSs are read-only, setting or changing values, resizing arrays of
+        structures, etc. is not allowed.
+    -   You cannot :py:meth:`~imaspy.db_entry.DBEntry.put`,
+        :py:meth:`~imaspy.db_entry.DBEntry.put_slice` or
+        :py:meth:`~imaspy.ids_toplevel.IDSToplevel.serialize` lazy-loaded IDSs. 
+
+2.  IMASPy **assumes** that the underlying data entry is not modified.
 
     When you (or another user) overwrite or add data to the same data entry, you may end
     up with a mix of old and new data in the lazy loaded IDS.
 
-2.  After you close the data entry, no new elements can be loaded.
+3.  After you close the data entry, no new elements can be loaded.
 
     >>> core_profiles = data_entry.get("core_profiles", lazy=True)
     >>> data_entry.close()
@@ -50,8 +83,6 @@ Lazy loading of data may speed up your programs, but also comes with some limita
     RuntimeError: Cannot lazy load the requested data: the data entry is no longer
     available for reading. Hint: did you close() the DBEntry?
 
-3.  IDSs that are lazy loaded are read-only, and you cannot :code:`put()` or
-    :code:`put_slice()` them.
 4.  Lazy loading has more overhead for reading data from the lowlevel: it is therefore
     more efficient to do a full :code:`get()` or :code:`get_slice()` when you intend to
     use most of the data stored in an IDS.

imaspy/_util.py

@@ -110,7 +110,7 @@ def _make_tree(structure, hide_empty_nodes=True, *, tree=None):
 
     iterator = structure
     if hide_empty_nodes and isinstance(structure, IDSStructure):
-        iterator = structure.iter_nonempty_()
+        iterator = structure.iter_nonempty_(accept_lazy=True)
     for child in iterator:
         if isinstance(child, IDSPrimitive):
             if not child.has_value:

imaspy/ids_convert.py

@@ -341,6 +341,11 @@ def convert_ids(
         factory: Existing IDSFactory to use for as target version.
         target: Use this IDSToplevel as target toplevel instead of creating one.
     """
+    if toplevel._lazy:
+        raise NotImplementedError(
+            "IDS conversion is not implemented for lazy-loaded IDSs"
+        )
+
     ids_name = toplevel.metadata.name
     if target is None:
         if factory is None:

imaspy/ids_structure.py

@@ -160,16 +160,78 @@ def _dd_parent(self) -> IDSBase:
     @property
     def has_value(self) -> bool:
         """True if any of the children has a non-default value"""
+        if self._lazy:
+            raise NotImplementedError(
+                "`has_value` is not implemented for lazy-loaded structures."
+            )
         for _ in self.iter_nonempty_():
             return True
         return False
 
-    def iter_nonempty_(self) -> Generator[IDSBase, None, None]:
+    def iter_nonempty_(self, *, accept_lazy=False) -> Generator[IDSBase, None, None]:
         """Iterate over all child nodes with non-default value.
 
         Note:
             The name ends with an underscore so it won't clash with IDS child names.
+
+        Caution:
+            This method works differently for lazy-loaded IDSs.
+
+            By default, an error is raised when calling this method for lazy loaded
+            IDSs. You may set the keyword argument ``accept_lazy`` to ``True`` to
+            iterate over all `loaded` child nodes with a non-default value. See below
+            example for lazy-loaded IDSs.
+
+        Examples:
+
+            .. code-block:: python
+                :caption: ``iter_nonempty_`` for fully loaded IDSs
+
+                >>> import imaspy.training
+                >>> entry = imaspy.training.get_training_db_entry()
+                >>> cp = entry.get("core_profiles")
+                >>> list(cp.iter_nonempty_())
+                [
+                    <IDSStructure (IDS:core_profiles, ids_properties)>,
+                    <IDSStructArray (IDS:core_profiles, profiles_1d with 3 items)>,
+                    <IDSNumericArray (IDS:core_profiles, time, FLT_1D)>
+                    numpy.ndarray([  3.98722186, 432.93759781, 792.        ])
+                ]
+
+            .. code-block:: python
+                :caption: ``iter_nonempty_`` for lazy-loaded IDSs
+
+                >>> import imaspy.training
+                >>> entry = imaspy.training.get_training_db_entry()
+                >>> cp = entry.get("core_profiles", lazy=True)
+                >>> list(cp.iter_nonempty_())
+                RuntimeError: Iterating over non-empty nodes of a lazy loaded IDS will
+                skip nodes that are not loaded. Set accept_lazy=True to continue.
+                See the documentation for more information: [...]
+                >>> list(cp.iter_nonempty_(accept_lazy=True))
+                []
+                >>> # An empty list because nothing is loaded. Load `time`:
+                >>> cp.time
+                <IDSNumericArray (IDS:core_profiles, time, FLT_1D)>
+                numpy.ndarray([  3.98722186, 432.93759781, 792.        ])
+                >>> list(cp.iter_nonempty_(accept_lazy=True))
+                [<IDSNumericArray (IDS:core_profiles, time, FLT_1D)>
+                numpy.ndarray([  3.98722186, 432.93759781, 792.        ])]
+
+        Keyword Args:
+            accept_lazy: Accept that this method will only yield loaded nodes of a
+                lazy-loaded IDS. Non-empty nodes that have not been loaded from the
+                backend are not iterated over. See detailed explanation above.
         """
+        if self._lazy and not accept_lazy:
+            raise RuntimeError(
+                "Iterating over non-empty nodes of a lazy loaded IDS will skip nodes "
+                "that are not loaded. Set accept_lazy=True to continue. "
+                "See the documentation for more information: "
+                "https://sharepoint.iter.org/departments/POP/CM/IMDesign/"
+                "Code%20Documentation/IMASPy-doc/generated/imaspy.ids_structure."
+                "IDSStructure.html#imaspy.ids_structure.IDSStructure.iter_nonempty_"
+            )
         for child in self._children:
             if child in self.__dict__:
                 child_node = getattr(self, child)
@@ -219,7 +281,8 @@ def _validate(self) -> None:
         # Common validation logic
         super()._validate()
         # IDSStructure specific: validate child nodes
-        for child in self.iter_nonempty_():
+        # accept_lazy=True: users are warned in IDSToplevel.validate()
+        for child in self.iter_nonempty_(accept_lazy=True):
             child._validate()
 
     def _xxhash(self) -> bytes:

imaspy/ids_toplevel.py

@@ -259,7 +259,8 @@ def validate(self):
 
     def _validate(self):
         # Override to skip the self.metadata.type.is_dynamic check in IDSBase._validate
-        for child in self.iter_nonempty_():
+        # accept_lazy=True: users are warned in IDSToplevel.validate()
+        for child in self.iter_nonempty_(accept_lazy=True):
             child._validate()
 
     @needs_imas

imaspy/util.py

@@ -16,7 +16,7 @@
 logger = logging.getLogger(__name__)
 
 
-def visit_children(func, node, *, leaf_only=True, visit_empty=False):
+def visit_children(func, node, *, leaf_only=True, visit_empty=False, accept_lazy=False):
     """Apply a function to node and its children
 
     IMASPy objects generally live in a tree structure. Similar to Pythons
@@ -36,6 +36,9 @@ def visit_children(func, node, *, leaf_only=True, visit_empty=False):
             * ``False``: All nodes, including internal nodes
 
         visit_empty: When set to True, also apply the function to empty nodes.
+        accept_lazy: See documentation of :py:param:`iter_nonempty_()
+            <imaspy.ids_structure.IDSStructure.iter_nonempty_.accept_lazy>`. Only
+            relevant when :param:`visit_empty` is False.
 
     Example:
         .. code-block:: python
@@ -53,7 +56,7 @@ def visit_children(func, node, *, leaf_only=True, visit_empty=False):
         iterator = node
         if not visit_empty and isinstance(node, IDSStructure):
             # Only iterate over non-empty nodes
-            iterator = node.iter_nonempty_()
+            iterator = node.iter_nonempty_(accept_lazy=accept_lazy)
 
         for child in iterator:
             visit_children(func, child, leaf_only=leaf_only, visit_empty=visit_empty)
@@ -71,6 +74,10 @@ def resample(node, old_time, new_time, homogeneousTime=None, inplace=False, **kw
 def print_tree(structure, hide_empty_nodes=True):
     """Print the full tree of an IDS or IDS structure.
 
+    Caution:
+        With :py:param:`hide_empty_nodes` set to ``True``, lazy-loaded IDSs will only
+        show loaded nodes.
+
     Args:
         structure: IDS structure to print
         hide_empty_nodes: Show or hide nodes without value.