Merge remote-tracking branch 'origin/develop' into feature/IMAS-4918-dd3to4

Author: maarten-ic

asv.conf.json

@@ -21,7 +21,7 @@
     // Customizable commands for building, installing, and
     // uninstalling the project. See asv.conf.json documentation.
     //
-    // "install_command": ["in-dir={env_dir} python -mpip install {wheel_file}"],
+    "install_command": ["in-dir={env_dir} python -mpip install {wheel_file}[netcdf]"],
     // "uninstall_command": ["return-code=any python -mpip uninstall -y {project}"],
     "build_command": [
         // link data-dictionary and build folder so the DD source files remain cached

benchmarks/core_profiles.py

@@ -62,14 +62,6 @@ def fill_slices(core_profiles, times):
             profiles_1d.neutral[i].density = np.zeros(N_GRID)
 
 
-def setup_dbentry(hli, backend):
-    dbentry = create_dbentry(hli, backend, "w")
-    core_profiles = factory[hli].core_profiles()
-    fill_slices(core_profiles, TIME)
-    dbentry.put(core_profiles)
-    dbentry.close()
-
-
 class GetSlice:
     params = [hlis, available_slicing_backends]
     param_names = ["hli", "backend"]
@@ -84,11 +76,16 @@ def time_get_slice(self, hli, backend):
         for t in TIME:
             self.dbentry.get_slice("core_profiles", t, imaspy.ids_defs.CLOSEST_INTERP)
 
+    def teardown(self, hli, backend):
+        if hasattr(self, "dbentry"):  # imas + netCDF has no dbentry
+            self.dbentry.close()
+
 
 class Get:
     params = [hlis, available_backends]
     param_names = ["hli", "backend"]
     setup = GetSlice.setup
+    teardown = GetSlice.teardown
 
     def time_get(self, hli, backend):
         self.dbentry.get("core_profiles")
@@ -108,6 +105,10 @@ def time_lazy_get(self, lazy, backend):
         cp = self.dbentry.get("core_profiles", lazy=lazy)
         np.array([prof_1d.electrons.temperature for prof_1d in cp.profiles_1d])
 
+    def teardown(self, lazy, backend):
+        if hasattr(self, "dbentry"):  # imas + netCDF has no dbentry
+            self.dbentry.close()
+
 
 class Generate:
     params = [hlis]
@@ -132,28 +133,30 @@ class Put:
     param_names = ["disable_validate", "hli", "backend"]
 
     def setup(self, disable_validate, hli, backend):
-        self.dbentry = create_dbentry(hli, backend)
+        create_dbentry(hli, backend).close()  # catch unsupported combinations
         self.core_profiles = factory[hli].core_profiles()
         fill_slices(self.core_profiles, TIME)
         os.environ["IMAS_AL_DISABLE_VALIDATE"] = disable_validate
 
     def time_put(self, disable_validate, hli, backend):
-        self.dbentry.put(self.core_profiles)
+        with create_dbentry(hli, backend) as dbentry:
+            dbentry.put(self.core_profiles)
 
 
 class PutSlice:
     params = [["0", "1"], hlis, available_slicing_backends]
     param_names = ["disable_validate", "hli", "backend"]
 
     def setup(self, disable_validate, hli, backend):
-        self.dbentry = create_dbentry(hli, backend)
+        create_dbentry(hli, backend).close()  # catch unsupported combinations
         self.core_profiles = factory[hli].core_profiles()
         os.environ["IMAS_AL_DISABLE_VALIDATE"] = disable_validate
 
     def time_put_slice(self, disable_validate, hli, backend):
-        for t in TIME:
-            fill_slices(self.core_profiles, [t])
-            self.dbentry.put_slice(self.core_profiles)
+        with create_dbentry(hli, backend) as dbentry:
+            for t in TIME:
+                fill_slices(self.core_profiles, [t])
+                dbentry.put_slice(self.core_profiles)
 
 
 class Serialize:

benchmarks/edge_profiles.py

@@ -124,6 +124,10 @@ def setup(self, hli, backend):
     def time_get(self, hli, backend):
         self.dbentry.get("edge_profiles")
 
+    def teardown(self, hli, backend):
+        if hasattr(self, "dbentry"):  # imas + netCDF has no dbentry
+            self.dbentry.close()
+
 
 class Generate:
     params = [hlis]
@@ -142,10 +146,11 @@ class Put:
     param_names = ["disable_validate", "hli", "backend"]
 
     def setup(self, disable_validate, hli, backend):
-        self.dbentry = create_dbentry(hli, backend)
+        create_dbentry(hli, backend).close()  # catch unsupported combinations
         self.edge_profiles = factory[hli].edge_profiles()
         fill_ggd(self.edge_profiles, TIME)
         os.environ["IMAS_AL_DISABLE_VALIDATE"] = disable_validate
 
     def time_put(self, disable_validate, hli, backend):
-        self.dbentry.put(self.edge_profiles)
+        with create_dbentry(hli, backend) as dbentry:
+            dbentry.put(self.edge_profiles)

benchmarks/utils.py

@@ -3,52 +3,50 @@
 import uuid
 from pathlib import Path
 
-from packaging.version import Version
-
 import imaspy
-from imaspy.backends.imas_core.imas_interface import ll_interface, lowlevel
+import imaspy.exception
 
 # Don't directly import imas: code analyzers break on the huge code base
 imas = importlib.import_module("imas")
 
 
+# Backend constants
+HDF5 = "HDF5"
+MDSPLUS = "MDSplus"
+MEMORY = "memory"
+ASCII = "ASCII"
+NETCDF = "netCDF"
+
+
+def create_uri(backend, path):
+    if backend == NETCDF:
+        return f"{path}.nc"
+    return f"imas:{backend.lower()}?path={path}"
+
+
 def backend_exists(backend):
     """Tries to detect if the lowlevel has support for the given backend."""
-    random_db = str(uuid.uuid4())
-    if ll_interface._al_version < Version("5"):
-        dbentry = imas.DBEntry(backend, random_db, 1, 1)
-        try:
-            # open() raises an exception when there is no support for the backend
-            # when there is support, but the entry cannot be opened, it only gives a
-            # negative return value.
-            # A bit weird (and subject to change per
-            # https://jira.iter.org/browse/IMAS-4671), but we can use it nicely here
-            dbentry.open()
-        except Exception as exc:
-            if "Error calling ual_begin_pulse_action" in str(exc):
-                return False
-            raise
-        return True
-
-    else:
-        dbentry = imaspy.DBEntry(backend, random_db, 1, 1)
-        try:
-            dbentry.open()
-        except Exception as exc:
-            if "backend is not available" in str(exc):
-                return False
-            elif isinstance(exc, lowlevel.ImasCoreBackendException):
-                return True
-            raise
+    uri = create_uri(backend, str(uuid.uuid4()))
+    try:
+        entry = imaspy.DBEntry(uri, "r")
+    except Exception as exc:
+        if "backend is not available" in str(exc):
+            return False
+        elif isinstance(exc, (imaspy.exception.ALException, FileNotFoundError)):
+            return True
         return True
+    # Highly unlikely, but it could succeed without error
+    entry.close()
+    return True
 
 
 # Note: UDA backend is not used for benchmarking
 all_backends = [
-    imaspy.ids_defs.HDF5_BACKEND,
-    imaspy.ids_defs.MDSPLUS_BACKEND,
-    imaspy.ids_defs.MEMORY_BACKEND,
-    imaspy.ids_defs.ASCII_BACKEND,
+    HDF5,
+    MDSPLUS,
+    MEMORY,
+    ASCII,
+    NETCDF,
 ]
 
 # Suppress error logs for testing backend availabitily:
@@ -59,9 +57,7 @@ def backend_exists(backend):
 available_backends = list(filter(backend_exists, all_backends))
 logging.getLogger().setLevel(logging.INFO)
 available_slicing_backends = [
-    backend
-    for backend in available_backends
-    if backend != imaspy.ids_defs.ASCII_BACKEND
+    backend for backend in available_backends if backend not in [ASCII, NETCDF]
 ]
 
 hlis = ["imas", "imaspy"]
@@ -77,12 +73,20 @@ def backend_exists(backend):
 
 
 def create_dbentry(hli, backend):
+    if backend == NETCDF:
+        if hli == "imas":
+            # Raising NotImplementedError will skip the benchmarks for this combination
+            raise NotImplementedError("AL-Python HLI doesn't implement netCDF.")
+        if hli == "imaspy":  # check if netcdf backend is available
+            try:
+                assert (
+                    imaspy.DBEntry._select_implementation("x.nc").__name__
+                    == "NCDBEntryImpl"
+                )
+            except (AttributeError, AssertionError):
+                raise NotImplementedError(
+                    "This version of IMASPy doesn't implement netCDF."
+                ) from None
+
     path = Path.cwd() / f"DB-{hli}-{backend}"
-    path.mkdir(exist_ok=True)
-    if backend == imas.imasdef.MDSPLUS_BACKEND:
-        # Need to ensure that the MDS+ data folder exists:
-        (path / "benchmark" / "3" / "0").mkdir(parents=True, exist_ok=True)
-    dbentry = DBEntry[hli](backend, "benchmark", 1, 1, str(path))
-    options = f"-prefix {path}/" if backend == imas.imasdef.ASCII_BACKEND else None
-    dbentry.create(options=options)
-    return dbentry
+    return DBEntry[hli](create_uri(backend, path), "w")

ci/build_docs_and_dist.sh

@@ -28,7 +28,7 @@ rm -rf dist
 python -m build .
 
 # Install imaspy and documentation dependencies from the just-built wheel
-pip install "`readlink -f dist/*.whl`[docs]"
+pip install "`readlink -f dist/*.whl`[docs,netcdf]"
 
 # Debugging:
 pip freeze

ci/run_pytest.sh

@@ -24,7 +24,7 @@ source venv/bin/activate
 
 # Install imaspy and test dependencies
 pip install --upgrade pip setuptools wheel
-pip install .[test]
+pip install .[h5py,netcdf,test]
 
 # Debugging:
 pip freeze

docs/source/.gitignore

@@ -1 +1,2 @@
 apidoc/*
+netcdf4.objects.inv