Merge branch 'develop' into bugfix/IMAS-5560

Author: maarten-ic

ci/run_benchmark.sh

@@ -52,9 +52,9 @@ asv run --skip-existing-successful main^!
 # Compare results
 if [ `git rev-parse --abbrev-ref HEAD` == develop ]
 then
-    asv compare main develop
+    asv compare main develop --machine $(hostname) || echo "asv compare failed"
 else
-    asv compare develop HEAD
+    asv compare develop HEAD --machine $(hostname) || echo "asv compare failed"
 fi
 
 # Publish results

conftest.py

@@ -24,7 +24,7 @@
 
 from imaspy.backends.imas_core.imas_interface import has_imas as _has_imas
 from imaspy.backends.imas_core.imas_interface import ll_interface, lowlevel
-from imaspy.dd_zip import dd_etree, latest_dd_version
+from imaspy.dd_zip import dd_etree, dd_xml_versions, latest_dd_version
 from imaspy.ids_defs import (
     ASCII_BACKEND,
     HDF5_BACKEND,
@@ -126,6 +126,14 @@ def latest_factory():
     return IDSFactory(latest_version)
 
 
+@pytest.fixture()
+def latest_factory3():
+    """Get most recent DDv3 version."""
+    for version in reversed(dd_xml_versions()):
+        if version.startswith("3."):
+            return IDSFactory(version)
+
+
 # Fixtures for various assets
 @pytest.fixture()
 def imaspy_assets():

imaspy/ids_convert.py

@@ -20,7 +20,12 @@
 from imaspy.ids_data_type import IDSDataType
 from imaspy.ids_factory import IDSFactory
 from imaspy.ids_path import IDSPath
-from imaspy.ids_primitive import IDSNumeric0D, IDSPrimitive, IDSString0D
+from imaspy.ids_primitive import (
+    IDSNumeric0D,
+    IDSNumericArray,
+    IDSPrimitive,
+    IDSString0D,
+)
 from imaspy.ids_struct_array import IDSStructArray
 from imaspy.ids_structure import IDSStructure
 from imaspy.ids_toplevel import IDSToplevel
@@ -302,6 +307,12 @@ def add_rename(old_path: str, new_path: str):
                 self.old_to_new.type_change[old_path] = _repeat_first_point_conditional
                 closed_path = Path(old_path).parent / "closed"
                 self.old_to_new.ignore_missing_paths.add(str(closed_path))
+            elif nbc_description == "remove_last_point_if_open_annular_centreline":
+                old_path = process_parent_renames(new_path)
+                self.new_to_old.type_change[new_path] = _repeat_last_point_centreline
+                self.old_to_new.type_change[old_path] = _remove_last_point_centreline
+                closed_path = Path(old_path) / "closed"
+                self.old_to_new.ignore_missing_paths.add(str(closed_path))
             else:  # Ignore unknown NBC changes
                 log_args = (nbc_description, new_path)
                 logger.error("Ignoring unsupported NBC change: %r for %r.", *log_args)
@@ -775,7 +786,11 @@ def _remove_last_point_conditional(
     for source, target in iterator:
         for child in source.iter_nonempty_():
             value = child.value
-            if closed and child.metadata.name != "time":
+            if (
+                closed
+                and child.metadata.name != "time"
+                and isinstance(child, IDSNumericArray)
+            ):
                 # repeat first point:
                 value = value[:-1]
             target[child.metadata.name] = value
@@ -801,14 +816,57 @@ def _repeat_first_point_conditional(
         iterator = [(source_node, target_node)]
     for source, target in iterator:
         for child in source.iter_nonempty_():
-            if child.metadata.name != "closed":
+            if child.metadata.name != "closed" and not child.metadata.name.endswith(
+                "_error_index"
+            ):
                 value = child.value
-                if closed and child.metadata.name != "time":
+                if (
+                    closed
+                    and child.metadata.name != "time"
+                    and isinstance(child, IDSNumericArray)
+                ):
                     # repeat first point:
                     value = numpy.concatenate((value, [value[0]]))
                 target[child.metadata.name] = value
 
 
+def _remove_last_point_centreline(source_node: IDSBase, target_node: IDSBase) -> None:
+    """Type change method for
+      nbc_description=repeat_children_first_point_conditional_centreline.
+
+    This method handles converting from old (DDv3) to new (DDv4).
+
+    If a centreline is a closed contour, we should do nothing.
+    If it is an open contour the thickness variable had too many entries,
+      and we'll drop the last one.
+    """
+    closed = bool(source_node._parent.centreline.closed)
+
+    if closed:
+        target_node.value = source_node.value
+    else:
+        target_node.value = source_node.value[:-1]
+
+
+def _repeat_last_point_centreline(source_node: IDSBase, target_node: IDSBase) -> None:
+    """Type change method for
+     nbc_description=repeat_children_first_point_conditional_centreline.
+
+    This method handles converting from new (DDv4) to old (DDv3).
+
+    If a centreline is a closed contour, we should do nothing.
+    If it is an open contour the thickness variable in the older
+     dd has one extra entry, so repeat the last one.
+    """
+    closed = bool(target_node._parent.centreline.closed)
+    if closed:
+        target_node.value = source_node.value
+    else:
+        target_node.value = numpy.concatenate(
+            (source_node.value, [source_node.value[-1]])
+        )
+
+
 def _cocos_change(node: IDSBase) -> None:
     """Handle COCOS definition change: multiply values by -1."""
     if not isinstance(node, IDSPrimitive):

imaspy/test/test_ids_convert.py

@@ -3,7 +3,7 @@
 
 import logging
 import re
-from datetime import UTC, datetime, timedelta
+from datetime import datetime, timedelta, timezone
 from unittest.mock import MagicMock
 
 import numpy
@@ -29,6 +29,8 @@
 from imaspy.ids_structure import IDSStructure
 from imaspy.test.test_helpers import compare_children, open_dbentry
 
+UTC = timezone.utc
+
 
 def test_iter_parents():
     assert list(iter_parents("a/b/c/d/e")) == ["a", "a/b", "a/b/c", "a/b/c/d"]
@@ -232,10 +234,10 @@ def test_3to4_ggd_space_identifier(dd4factory):
 
 
 def test_3to4_repeat_children_first_point_conditional(dd4factory):
-    # The wall IDS contains all (three!) cases with conditional repeats
+    # The wall IDS contains all (four!) cases with conditional repeats
     wall = IDSFactory("3.39.0").wall()
     wall.ids_properties.homogeneous_time = IDS_TIME_MODE_HETEROGENEOUS
-    wall.description_2d.resize(1)
+    wall.description_2d.resize(2)
 
     # Case 1: repeat_children_first_point_conditional
     wall.description_2d[0].vessel.unit.resize(2)
@@ -264,8 +266,20 @@ def test_3to4_repeat_children_first_point_conditional(dd4factory):
             unit.outline[j].z = [-1.0, -2.0, -3.0]
             unit.outline[j].time = j / 5
 
+    # Case 4: repeat_children_first_point_conditional_centreline
+    # (see https://jira.iter.org/browse/IMAS-5541)
+    wall.description_2d[1].vessel.unit.resize(2)
+    for i in range(2):
+        centreline = wall.description_2d[1].vessel.unit[i].annular.centreline
+        centreline.closed = i  # first is open, second is closed
+        centreline.r = [1.0, 2.0, 3.0]
+        centreline.z = [-1.0, -2.0, -3.0]
+        # if it was open there were too many thickness values!
+        # The last one will be dropped and repeated
+        wall.description_2d[1].vessel.unit[i].annular.thickness = [1, 0.9, 0.9]
+
     wall4 = convert_ids(wall, None, factory=dd4factory)
-    assert len(wall4.description_2d) == 1
+    assert len(wall4.description_2d) == 2
 
     # Test conversion for case 1:
     assert len(wall4.description_2d[0].vessel.unit) == 2
@@ -302,6 +316,15 @@ def test_3to4_repeat_children_first_point_conditional(dd4factory):
                 assert numpy.array_equal(unit.outline[j].z, [-1.0, -2.0, -3.0, -1.0])
             assert unit.outline[j].time == pytest.approx(j / 5)
 
+    # Test conversion for case 4:
+    assert len(wall4.description_2d[1].vessel.unit) == 2
+    for i in range(2):
+        thickness = wall4.description_2d[1].vessel.unit[i].annular.thickness
+        if i == 0:  # open outline, there was one value too many, drop the last one
+            assert numpy.array_equal(thickness, [1, 0.9])
+        else:  # closed outline, thickness values kept
+            assert numpy.array_equal(thickness, [1, 0.9, 0.9])
+
     # Test conversion back
     wall3 = convert_ids(wall4, "3.39.0")
     compare_children(wall, wall3)

imaspy/test/test_nbc_change.py

@@ -362,25 +362,47 @@ def test_autofill_save_newer(ids_name, backend, worker_id, tmp_path):
         dbentry2.close()
 
 
-def test_convert_min_to_max(ids_name, latest_factory):
+def test_convert_min_to_max_v3(ids_name, latest_factory3):
+    """Convert from DD 3.22.0 to the last DDv3 release."""
     factory = IDSFactory("3.22.0")
     if not factory.exists(ids_name):
         pytest.skip("IDS %s not defined for version 3.22.0" % (ids_name,))
-    if not latest_factory.exists(ids_name):
-        pytest.skip(f"IDS {ids_name} not defined for version {latest_factory.version}")
+    if not latest_factory3.exists(ids_name):
+        pytest.skip(f"IDS {ids_name} not defined for version {latest_factory3.version}")
 
     ids = factory.new(ids_name)
     fill_with_random_data(ids)
-    convert_ids(ids, latest_factory.version)
+    convert_ids(ids, latest_factory3.version)
 
 
-def test_convert_max_to_min(ids_name, latest_factory):
+def test_convert_max_to_min_v3(ids_name, latest_factory3):
+    """Convert from the last DDv3 release to DD 3.22.0."""
     factory = IDSFactory("3.22.0")
     if not factory.exists(ids_name):
         pytest.skip(f"IDS {ids_name} not defined for version 3.22.0")
-    if not latest_factory.exists(ids_name):
-        pytest.skip(f"IDS {ids_name} not defined for version {latest_factory.version}")
+    if not latest_factory3.exists(ids_name):
+        pytest.skip(f"IDS {ids_name} not defined for version {latest_factory3.version}")
 
-    ids = latest_factory.new(ids_name)
+    ids = latest_factory3.new(ids_name)
     fill_with_random_data(ids)
     convert_ids(ids, None, factory=factory)
+
+
+def test_convert_3_to_newest(ids_name, latest_factory3, latest_factory):
+    """Convert from the last DDv3 release to the last released DD."""
+    if not latest_factory3.exists(ids_name) or not latest_factory.exists(ids_name):
+        pytest.skip(f"IDS {ids_name} not defined for both versions.")
+
+    ids = latest_factory3.new(ids_name)
+    fill_with_random_data(ids)
+    convert_ids(ids, None, factory=latest_factory)
+
+
+def test_convert_newest_to_3(ids_name, latest_factory3, latest_factory):
+    """Convert from the last released DD to the last DDv3 release."""
+    if not latest_factory3.exists(ids_name) or not latest_factory.exists(ids_name):
+        pytest.skip(f"IDS {ids_name} not defined for both versions.")
+
+    ids = latest_factory.new(ids_name)
+    fill_with_random_data(ids)
+    convert_ids(ids, None, factory=latest_factory3)