Merge pull request #4076 from pnorbert/python-scalar-attributes

Python: add the same treatment to attributes as to variables before: …

bindings/Python/py11Attribute.cpp

@@ -36,6 +36,12 @@ std::string Attribute::Type() const
     return ToString(m_Attribute->m_Type);
 }
 
+bool Attribute::SingleValue() const
+{
+    helper::CheckForNullptr(m_Attribute, "in call to Attribute::SingleValue");
+    return m_Attribute->m_IsSingleValue;
+}
+
 std::vector<std::string> Attribute::DataString()
 {
     helper::CheckForNullptr(m_Attribute, "in call to Attribute::DataStrings");

bindings/Python/py11Attribute.h

@@ -37,6 +37,8 @@ class Attribute
 
     std::string Type() const;
 
+    bool SingleValue() const;
+
     pybind11::array Data();
 
     std::vector<std::string> DataString();

bindings/Python/py11IO.cpp

@@ -180,8 +180,15 @@ Attribute IO::DefineAttribute(const std::string &name, const pybind11::array &ar
     else if (pybind11::isinstance<pybind11::array_t<T, pybind11::array::c_style>>(array))          \
     {                                                                                              \
         const T *data = reinterpret_cast<const T *>(array.data());                                 \
-        const size_t size = static_cast<size_t>(array.size());                                     \
-        attribute = &m_IO->DefineAttribute<T>(name, data, size, variableName, separator);          \
+        if (array.ndim())                                                                          \
+        {                                                                                          \
+            const size_t size = static_cast<size_t>(array.size());                                 \
+            attribute = &m_IO->DefineAttribute<T>(name, data, size, variableName, separator);      \
+        }                                                                                          \
+        else                                                                                       \
+        {                                                                                          \
+            attribute = &m_IO->DefineAttribute<T>(name, data[0], variableName, separator);         \
+        }                                                                                          \
     }
     ADIOS2_FOREACH_NUMPY_ATTRIBUTE_TYPE_1ARG(declare_type)
 #undef declare_type

bindings/Python/py11glue.cpp

@@ -395,7 +395,8 @@ PYBIND11_MODULE(ADIOS2_PYTHON_MODULE_NAME, m)
         .def("Name", &adios2::py11::Attribute::Name)
         .def("Type", &adios2::py11::Attribute::Type)
         .def("DataString", &adios2::py11::Attribute::DataString)
-        .def("Data", &adios2::py11::Attribute::Data);
+        .def("Data", &adios2::py11::Attribute::Data)
+        .def("SingleValue", &adios2::py11::Attribute::SingleValue);
 
     pybind11::class_<adios2::py11::Engine>(m, "Engine")
         // Python 2

docs/user_guide/source/api_python/adios2-doc-read-filereader.py

@@ -9,14 +9,18 @@
         for key, value in info.items():
             print("\t" + key + ": " + value, end=" ")
         print()
+    print()
 
     nproc = s.read("nproc")
-    print(f"nproc is {nproc} of type {type(nproc)}")
+    print(f"nproc is {nproc} of type {type(nproc)} with ndim {nproc.ndim}")
 
     # read variables return a numpy array with corresponding selection
     steps = int(vars["physical_time"]["AvailableStepsCount"])
     physical_time = s.read("physical_time", step_selection=[0, steps])
-    print(f"physical_time is {physical_time} of type {type(physical_time)}")
+    print(
+        f"physical_time is {physical_time} of type {type(physical_time)} with "
+        f"ndim {physical_time.ndim} shape = {physical_time.shape}"
+    )
 
     steps = int(vars["temperature"]["AvailableStepsCount"])
     temperature = s.read("temperature", step_selection=[0, steps])

docs/user_guide/source/api_python/adios2-doc-read.py → docs/user_guide/source/api_python/adios2-doc-read-stream.py

@@ -16,7 +16,7 @@
 
         if s.current_step() == 0:
             nproc = s.read("nproc")
-            print(f"nproc is {nproc} of type {type(nproc)}")
+            print(f"nproc is {nproc} of type {type(nproc)} with ndim {nproc.ndim}")
 
         # read variables return a numpy array with corresponding selection
         physical_time = s.read("physical_time")
@@ -26,6 +26,6 @@
         print(f"temperature array size is {temperature.size} of shape {temperature.shape}")
         print(f"temperature unit is {temp_unit} of type {type(temp_unit)}")
         pressure = s.read("pressure")
-        press_unit = s.read_attribute("pressure/unit")
+        press_unit = s.read_attribute("unit", "pressure")
         print(f"pressure unit is {press_unit} of type {type(press_unit)}")
         print()

docs/user_guide/source/api_python/python_example.rst

@@ -44,6 +44,7 @@ Python Write example
    count = [nx]
    
    temperature = np.zeros(nx, dtype=np.double)
+   pressure = np.ones(nx, dtype=np.double)
    delta_time = 0.01
    physical_time = 0.0
    nsteps = 5
@@ -125,17 +126,17 @@ Python Read "step-by-step" example
     nproc is 4 of type <class 'numpy.ndarray'>
     physical_time is 0.0 of type <class 'numpy.ndarray'>
     temperature array size is 40 of shape (40,)
-    temperature unit is ['K'] of type <class 'list'>
-    pressure unit is ['Pa'] of type <class 'list'>
+    temperature unit is K of type <class 'str'>
+    pressure unit is Pa of type <class 'str'>
 
     Current step is 1
     variable_name: physical_time    AvailableStepsCount: 1  Max: 0.01       Min: 0.01       Shape:          SingleValue: true   Type: double
     variable_name: pressure         AvailableStepsCount: 1  Max: 1  Min: 1  Shape: 40       SingleValue: false      Type: double
     variable_name: temperature      AvailableStepsCount: 1  Max: 0  Min: 0  Shape: 40       SingleValue: false      Type: double
     physical_time is 0.01 of type <class 'numpy.ndarray'>
     temperature array size is 40 of shape (40,)
-    temperature unit is ['K'] of type <class 'list'>
-    pressure unit is ['Pa'] of type <class 'list'>
+    temperature unit is K of type <class 'str'>
+    pressure unit is Pa of type <class 'str'>
 
     ...
 
@@ -156,14 +157,18 @@ Python Read Random Access example
             for key, value in info.items():
                 print("\t" + key + ": " + value, end=" ")
             print()
+        print()
 
         nproc = s.read("nproc")
-        print(f"nproc is {nproc} of type {type(nproc)}")
+        print(f"nproc is {nproc} of type {type(nproc)} with ndim {nproc.ndim}")
         
         # read variables return a numpy array with corresponding selection
         steps = int(vars['physical_time']['AvailableStepsCount'])
         physical_time = s.read("physical_time", step_selection=[0, steps])
-        print(f"physical_time is {physical_time} of type {type(physical_time)}")
+        print(
+            f"physical_time is {physical_time} of type {type(physical_time)} with "
+            f"ndim {physical_time.ndim} shape = {physical_time.shape}"
+        )
 
         steps = int(vars['temperature']['AvailableStepsCount'])
         temperature = s.read("temperature", step_selection=[0, steps])
@@ -183,8 +188,9 @@ Python Read Random Access example
     variable_name: physical_time    AvailableStepsCount: 5  Max: 0.04       Min: 0  Shape:          SingleValue: true       Type: double
     variable_name: pressure         AvailableStepsCount: 5  Max: 1  Min: 1  Shape: 40       SingleValue: false      Type: double
     variable_name: temperature      AvailableStepsCount: 5  Max: 0  Min: 0  Shape: 40       SingleValue: false      Type: double
-    nproc is 4 of type <class 'numpy.ndarray'>
-    physical_time is [0.   0.01 0.02 0.03 0.04] of type <class 'numpy.ndarray'>
+
+    nproc is 4 of type <class 'numpy.ndarray'> with ndim 0
+    physical_time is [0.   0.01 0.02 0.03 0.04] of type <class 'numpy.ndarray'> with ndim 1 shape = (5,)
     temperature array size is 200 of shape (200,)
-    temperature unit is ['K'] of type <class 'list'>
+    temperature unit is K of type <class 'str'>
 

examples/hello/bpReader/bpReaderHeatMap2D.py

@@ -35,27 +35,40 @@
     for j in range(0, Ny):
         value = iGlobal * shape[1] + j
         temperatures[i * Nx + j] = value
-#        print(str(i) + "," + str(j) + " " + str(value))
 
 with Stream("HeatMap2D_py.bp", "w", comm) as obpStream:
     obpStream.write("temperature2D", temperatures, shape, start, count)
     if not rank:
         obpStream.write("N", [size, Nx, Ny])  # will be an array in output
         obpStream.write("Nx", numpy.array(Nx))  # will be a scalar in output
         obpStream.write("Ny", Ny)  # will be a scalar in output
+        obpStream.write_attribute("nproc", size)  # will be a single value attribute in output
         obpStream.write_attribute("dimensions", [size * Nx, Ny], "temperature2D")
 
 if not rank:
     with FileReader("HeatMap2D_py.bp", MPI.COMM_SELF) as ibpFile:
+        # scalar variables are read as a numpy array with 0 dimension
+        in_nx = ibpFile.read("Nx")
+        in_ny = ibpFile.read("Ny")
+        print(f"Incoming nx, ny = {in_nx}, {in_ny}")
+
+        # single value attribute is read as a numpy array with 0 dimension
+        in_nproc = ibpFile.read_attribute("nproc")
+        print(f"Incoming nproc = {in_nproc}")
+        # array attribute is read as a numpy array or string list
+        in_dims = ibpFile.read_attribute("temperature2D/dimensions")
+        print(f"Incoming diumensions = {in_dims}")
+
+        # On option is to inquire a variable to know its type, shape
+        # directly, not as strings, and then we can use the variable
+        # object to set selection and/or set steps to read
         var_inTemperature = ibpFile.inquire_variable("temperature2D")
         if var_inTemperature is not None:
             var_inTemperature.set_selection([[2, 2], [4, 4]])
             inTemperatures = ibpFile.read(var_inTemperature)
-
-        in_nx = ibpFile.read("Nx")  # scalar is read as a numpy array with 1 element
-        in_ny = ibpFile.read("Ny")  # scalar is read as a numpy array with 1 element
-        print(f"Incoming nx, ny = {in_nx}, {in_ny}")
-
-        print("Incoming temperature map")
-        for i in range(0, inTemperatures.shape[1]):
-            print(str(inTemperatures[i]))
+            print(
+                f"Incoming temperature map with selection "
+                f"start = {var_inTemperature.start()}, count = {var_inTemperature.count()}"
+            )
+            for i in range(0, inTemperatures.shape[1]):
+                print(str(inTemperatures[i]))

python/adios2/attribute.py

@@ -42,13 +42,24 @@ def type(self):
         """
         return self.impl.Type()
 
+    def single_value(self):
+        """
+        True if the attribute is a single value, False if it is an array
+
+        Returns:
+            True or False.
+        """
+        return self.impl.SingleValue()
+
     def data(self):
         """
         Content of the Attribute
 
         Returns:
             Content of the Attribute as a non string.
         """
+        if self.single_value():
+            return self.impl.Data()[0]
         return self.impl.Data()
 
     def data_string(self):
@@ -58,4 +69,6 @@ def data_string(self):
         Returns:
             Content of the Attribute as a str.
         """
+        if self.single_value():
+            return self.impl.DataString()[0]
         return self.impl.DataString()

python/adios2/file_reader.py

@@ -25,7 +25,3 @@ def _(self, io: IO, path, comm=None):
         super().__init__(io, path, "rra", comm)
 
     # pylint: enable=E1121
-
-    def variables(self):
-        """Returns the list of variables contained in the opened file"""
-        return [self._io.inquire_variable(var) for var in self.available_variables()]

python/adios2/stream.py

@@ -325,11 +325,9 @@ def _(self, name, content, shape=[], start=[], count=[], operations=None):
 
         self.write(variable, content)
 
-    @singledispatchmethod
-    def read(self, variable: Variable):
+    def _read_var(self, variable: Variable):
         """
-        Random access read allowed to select steps,
-        only valid with Stream Engines
+        Internal common function to read. Settings must be done to Variable before the call.
 
         Parameters
             variable
@@ -342,6 +340,7 @@ def read(self, variable: Variable):
         """
         dtype = type_adios_to_numpy(variable.type())
         count = variable.count()
+
         if count != []:
             # array
             # steps = variable.get_steps_from_step_selection()
@@ -365,7 +364,8 @@ def read(self, variable: Variable):
         else:
             # scalar
             size_all_steps = variable.selection_size()
-            if size_all_steps > 1:
+            # if size_all_steps > 1:
+            if self._mode == bindings.Mode.ReadRandomAccess and variable.steps() > 1:
                 output_shape = [size_all_steps]
             else:
                 output_shape = []
@@ -374,15 +374,17 @@ def read(self, variable: Variable):
         self._engine.get(variable, output)
         return output
 
-    @read.register(str)
-    def _(self, name: str, start=[], count=[], block_id=None, step_selection=None):
+    @singledispatchmethod
+    def read(self, variable: Variable, start=[], count=[], block_id=None, step_selection=None):
         """
-        Random access read allowed to select steps,
-        only valid with Stream Engines
+        Read a variable.
+        Random access read allowed to select steps.
 
         Parameters
-            name
-                variable to be read
+            variable
+                adios2.Variable object to be read
+                Use variable.set_selection(), set_block_selection(), set_step_selection()
+                to prepare a read
 
             start
                 variable offset dimensions
@@ -400,10 +402,6 @@ def _(self, name: str, start=[], count=[], block_id=None, step_selection=None):
             array
                 resulting array from selection
         """
-        variable = self._io.inquire_variable(name)
-        if not variable:
-            raise ValueError()
-
         if step_selection is not None and not self._mode == bindings.Mode.ReadRandomAccess:
             raise RuntimeError("step_selection parameter requires 'rra' mode")
 
@@ -419,7 +417,39 @@ def _(self, name: str, start=[], count=[], block_id=None, step_selection=None):
         if start != [] and count != []:
             variable.set_selection([start, count])
 
-        return self.read(variable)
+        return self._read_var(variable)
+
+    @read.register(str)
+    def _(self, name: str, start=[], count=[], block_id=None, step_selection=None):
+        """
+        Read a variable.
+        Random access read allowed to select steps.
+
+        Parameters
+            name
+                variable to be read
+
+            start
+                variable offset dimensions
+
+            count
+                variable local dimensions from offset
+
+            block_id
+                (int) Required for reading local variables, local array, and local
+                value.
+
+            step_selection
+                (list): On the form of [start, count].
+        Returns
+            array
+                resulting array from selection
+        """
+        variable = self._io.inquire_variable(name)
+        if not variable:
+            raise ValueError()
+
+        return self.read(variable, start, count, block_id, step_selection)
 
     def write_attribute(self, name, content, variable_name="", separator="/"):
         """

testing/adios2/python/TestAttribute.py

@@ -12,14 +12,14 @@ def test_create_write(self):
         with adios.declare_io("BPWriter") as io:
             ts = io.define_attribute("timestamp", "20231122")
             self.assertEqual(ts.name(), "timestamp")
-            self.assertEqual(ts.data_string(), ["20231122"])
+            self.assertEqual(ts.data_string(), "20231122")
 
     def test_create_reader(self):
         adios = Adios()
         with adios.declare_io("BPWriter") as io:
             ts = io.define_attribute("timestamp", "20231122")
             self.assertEqual(ts.name(), "timestamp")
-            self.assertEqual(ts.data_string(), ["20231122"])
+            self.assertEqual(ts.data_string(), "20231122")
 
     def test_create_write_ndarray(self):
         adios = Adios()