Merge pull request #4014 from pnorbert/update-python-examples

Python updates

cmake/DetectOptions.cmake

@@ -400,15 +400,15 @@ if(NOT SHARED_LIBS_SUPPORTED)
 endif()
 
 if(ADIOS2_USE_PIP)
-  find_package(Python 3 REQUIRED COMPONENTS Interpreter Development.Module)
+  find_package(Python 3.8 REQUIRED COMPONENTS Interpreter Development.Module)
   set(ADIOS2_HAVE_PIP TRUE)
 elseif(ADIOS2_USE_Python STREQUAL AUTO)
-  find_package(Python 3 COMPONENTS Interpreter Development)
+  find_package(Python 3.8 COMPONENTS Interpreter Development)
   if(Python_FOUND AND ADIOS2_HAVE_MPI)
     find_package(PythonModule COMPONENTS mpi4py mpi4py/mpi4py.h)
   endif()
 elseif(ADIOS2_USE_Python)
-  find_package(Python 3 REQUIRED COMPONENTS Interpreter Development)
+  find_package(Python 3.8 REQUIRED COMPONENTS Interpreter Development)
   if(ADIOS2_HAVE_MPI)
     find_package(PythonModule REQUIRED COMPONENTS mpi4py mpi4py/mpi4py.h)
   endif()

examples/hello/bpReader/bpReaderHeatMap2D-bindings.py

@@ -0,0 +1,79 @@
+#
+# Distributed under the OSI-approved Apache License, Version 2.0.  See
+# accompanying file Copyright.txt for details.
+#
+# bpReaderHeatMap2D-bindings.py
+#
+#
+#  Created on: Dec 5th, 2017
+#      Author: William F Godoy godoywf@ornl.gov
+#              Norbert Podhorszki pnorbert@ornl.gov
+#
+
+from mpi4py import MPI
+import numpy
+import adios2.bindings as adios2
+
+# MPI
+comm = MPI.COMM_WORLD
+rank = comm.Get_rank()
+size = comm.Get_size()
+
+# User data
+Nx = 10
+Ny = 10
+
+count = [Nx, Ny]
+start = [rank * Nx, 0]
+shape = [size * Nx, Ny]
+
+temperatures = numpy.zeros(Nx * Ny, dtype=int)
+
+for i in range(0, Nx):
+    iGlobal = start[0] + i
+
+    for j in range(0, Ny):
+        value = iGlobal * shape[1] + j
+        temperatures[i * Nx + j] = value
+#        print(str(i) + "," + str(j) + " " + str(value))
+
+
+# ADIOS portion
+adios = adios2.ADIOS(comm)
+ioWrite = adios.DeclareIO("ioWriter")
+
+varTemperature = ioWrite.DefineVariable(
+    "temperature2D", temperatures, shape, start, count, adios2.ConstantDims
+)
+
+obpStream = ioWrite.Open("HeatMap2D_py_bindings.bp", adios2.Mode.Write)
+obpStream.BeginStep()
+obpStream.Put(varTemperature, temperatures)
+obpStream.EndStep()
+obpStream.Close()
+
+
+if rank == 0:
+    ioRead = adios.DeclareIO("ioReader")
+
+    ibpStream = ioRead.Open("HeatMap2D_py_bindings.bp", adios2.Mode.Read, MPI.COMM_SELF)
+
+    ibpStream.BeginStep()
+
+    var_inTemperature = ioRead.InquireVariable("temperature2D")
+
+    if var_inTemperature is not None:
+        var_inTemperature.SetSelection([[2, 2], [4, 4]])
+
+        inSize = var_inTemperature.SelectionSize()
+        print("Incoming size " + str(inSize))
+        inTemperatures = numpy.zeros(var_inTemperature.Count(), dtype=int)
+
+        ibpStream.Get(var_inTemperature, inTemperatures, adios2.Mode.Sync)
+
+        print("Incoming temperature map")
+        for i in range(0, inTemperatures.shape[1]):
+            print(str(inTemperatures[i]) + " ")
+
+    ibpStream.EndStep()
+    ibpStream.Close()

examples/hello/bpReader/bpReaderHeatMap2D.py

@@ -7,11 +7,12 @@
 #
 #  Created on: Dec 5th, 2017
 #      Author: William F Godoy godoywf@ornl.gov
+#              Norbert Podhorszki pnorbert@ornl.gov
 #
 
 from mpi4py import MPI
 import numpy
-import adios2
+from adios2 import Stream
 
 # MPI
 comm = MPI.COMM_WORLD
@@ -26,57 +27,27 @@
 start = [rank * Nx, 0]
 shape = [size * Nx, Ny]
 
-temperatures = numpy.zeros(Nx * Ny, dtype=numpy.int)
+temperatures = numpy.zeros(Nx * Ny, dtype=int)
 
 for i in range(0, Nx):
     iGlobal = start[0] + i
 
     for j in range(0, Ny):
         value = iGlobal * shape[1] + j
         temperatures[i * Nx + j] = value
-        print(str(i) + "," + str(j) + " " + str(value))
-
-
-# ADIOS portion
-adios = adios2.ADIOS(comm)
-ioWrite = adios.DeclareIO("ioWriter")
-
-varTemperature = ioWrite.DefineVariable(
-    "temperature2D", temperatures, shape, start, count, adios2.ConstantDims
-)
-
-obpStream = ioWrite.Open("HeatMap2D_py.bp", adios2.Mode.Write)
-obpStream.BeginStep()
-obpStream.Put(varTemperature, temperatures)
-obpStream.EndStep()
-obpStream.Close()
+#        print(str(i) + "," + str(j) + " " + str(value))
 
+with Stream("HeatMap2D_py.bp", "w", comm) as obpStream:
+    obpStream.write("temperature2D", temperatures, shape, start, count)
 
 if rank == 0:
-    ioRead = adios.DeclareIO("ioReader")
-
-    ibpStream = ioRead.Open("HeatMap2D_py.bp", adios2.Mode.Read, MPI.COMM_SELF)
-
-    ibpStream.BeginStep()
-
-    var_inTemperature = ioRead.InquireVariable("temperature2D")
-
-    if var_inTemperature is not None:
-        var_inTemperature.SetSelection([[2, 2], [4, 4]])
-
-        inSize = var_inTemperature.SelectionSize()
-        print("Incoming size " + str(inSize))
-        inTemperatures = numpy.zeros(inSize, dtype=numpy.int)
-
-        ibpStream.Get(var_inTemperature, inTemperatures, adios2.Mode.Sync)
-
-        print("Incoming temperature map")
-
-        for i in range(0, inTemperatures.size):
-            print(str(inTemperatures[i]) + " ")
-
-            if (i + 1) % 4 == 0:
-                print()
-
-    ibpStream.EndStep()
-    ibpStream.Close()
+    with Stream("HeatMap2D_py.bp", "r", MPI.COMM_SELF) as ibpStream:
+        for _ in ibpStream.steps():
+            var_inTemperature = ibpStream.inquire_variable("temperature2D")
+            if var_inTemperature is not None:
+                var_inTemperature.set_selection([[2, 2], [4, 4]])
+                inTemperatures = ibpStream.read(var_inTemperature)
+
+            print("Incoming temperature map")
+            for i in range(0, inTemperatures.shape[1]):
+                print(str(inTemperatures[i]) + " ")

examples/hello/bpWriter/bpWriter-bindings.py

@@ -0,0 +1,59 @@
+#
+# Distributed under the OSI-approved Apache License, Version 2.0.  See
+# accompanying file Copyright.txt for details.
+#
+# bpWriter-bindings.py : Low-level Python API example
+#  Created on: Feb 2, 2017
+#      Authors: William F Godoy godoywf@ornl.gov
+#               Norbert Podhorszki pnorbert@ornl.gov
+#
+# adios2.bindings gives access to the low level classes that are compiled from the C++ classes
+# Only necessary to use this when the adios2 python API is insufficient for some reason
+# This example doesn't use anything that is missing from the python API though.
+
+from mpi4py import MPI
+import numpy
+import adios2.bindings as adios2
+
+# MPI
+comm = MPI.COMM_WORLD
+rank = comm.Get_rank()
+size = comm.Get_size()
+
+# User data
+myArray = numpy.array([0, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0])
+myArray = myArray + (rank * 10)
+Nx = myArray.size
+
+# ADIOS MPI Communicator
+adios = adios2.ADIOS(comm)
+
+# ADIOS IO
+bpIO = adios.DeclareIO("BPFile_N2N")
+bpIO.SetEngine("BPFile")
+# bpIO.SetParameters( {"Threads" : "2", "ProfileUnits" : "Microseconds",
+# "InitialBufferSize" : "17Kb"} )
+bpIOParams = {}
+bpIOParams["Threads"] = "2"
+bpIOParams["ProfileUnits"] = "Microseconds"
+bpIOParams["InitialBufferSize"] = "17Kb"
+bpIO.SetParameters(bpIOParams)
+
+fileID = bpIO.AddTransport("File", {"Library": "fstream"})
+
+# ADIOS Variable name, shape, start, offset, constant dims
+ioArray = bpIO.DefineVariable(
+    "bpArray", myArray, [size * Nx], [rank * Nx], [Nx], adios2.ConstantDims
+)
+
+# ADIOS Engine
+bpFileWriter = bpIO.Open("bpWriter-py-bindings.bp", adios2.Mode.Write)
+bpFileWriter.BeginStep()
+bpFileWriter.Put(ioArray, myArray, adios2.Mode.Sync)
+bpFileWriter.EndStep()
+bpFileWriter.Close()
+
+# Read content:
+# bpls -la bpWriter-py-bindings.bp
+# bpls -la bpWriter-py-bindings.bp -d bpArray -n 10
+# bpls -la bpWriter-py-bindings.bp -d bpArray -n 10 -D

examples/hello/bpWriter/bpWriter.py

@@ -4,7 +4,12 @@
 #
 # bpWriter.py : only works with MPI version
 #  Created on: Feb 2, 2017
-#      Author: William F Godoy godoywf@ornl.gov
+#      Authors: William F Godoy godoywf@ornl.gov
+#               Norbert Podhorszki pnorbert@ornl.gov
+#
+# We use adios2.Adios and adios2.IO classes to set up the IO parameters.
+# For default IO, it is sufficient to use the adios2.Stream class alone.
+
 from mpi4py import MPI
 import numpy
 import adios2
@@ -16,32 +21,30 @@
 
 # User data
 myArray = numpy.array([0, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0])
+myArray = myArray + (rank * 10)
 Nx = myArray.size
 
 # ADIOS MPI Communicator
-adios = adios2.ADIOS(comm)
+adios = adios2.Adios(config_file=None, comm=comm)
 
 # ADIOS IO
-bpIO = adios.DeclareIO("BPFile_N2N")
-bpIO.SetEngine("BPFile")
-# bpIO.SetParameters( {"Threads" : "2", "ProfileUnits" : "Microseconds",
-# "InitialBufferSize" : "17Kb"} )
+bpIO = adios.declare_io("BPFile_N2N")
+
 bpIOParams = {}
 bpIOParams["Threads"] = "2"
 bpIOParams["ProfileUnits"] = "Microseconds"
 bpIOParams["InitialBufferSize"] = "17Kb"
-bpIO.SetParameters(bpIOParams)
+bpIO.set_parameters(bpIOParams)
 
-fileID = bpIO.AddTransport("File", {"Library": "fstream"})
+bpIO.add_transport("File", {"Library": "fstream"})
+bpIO.set_engine("BPFile")
+a = bpIO.adios()
 
-# ADIOS Variable name, shape, start, offset, constant dims
-ioArray = bpIO.DefineVariable(
-    "bpArray", myArray, [size * Nx], [rank * Nx], [Nx], adios2.ConstantDims
-)
+# ADIOS output stream
+with adios2.Stream(bpIO, "bpWriter-py.bp", "w", comm) as fh:
+    fh.write("bpArray", myArray, [size * Nx], [rank * Nx], [Nx])
 
-# ADIOS Engine
-bpFileWriter = bpIO.Open("npArray.bp", adios2.Mode.Write)
-bpFileWriter.BeginStep()
-bpFileWriter.Put(ioArray, myArray, adios2.Mode.Sync)
-bpFileWriter.EndStep()
-bpFileWriter.Close()
+# Read content:
+# bpls -la bpWriter-py.bp
+# bpls -la bpWriter-py.bp -d bpArray -n 10
+# bpls -la bpWriter-py.bp -d bpArray -n 10 -D

examples/hello/datamanReader/dataManReader.py

@@ -10,31 +10,26 @@
 
 from mpi4py import MPI
 import numpy as np
-import adios2
+from adios2 import Adios, Stream
+from adios2.bindings import StepStatus
 
 comm = MPI.COMM_WORLD
 rank = comm.Get_rank()
 size = comm.Get_size()
 
-adios = adios2.ADIOS(comm)
-io = adios.DeclareIO("whatever")
-io.SetEngine("DataMan")
-io.SetParameters({"IPAddress": "127.0.0.1", "Port": "12306", "Timeout": "5"})
+adios = Adios(comm)
+io = adios.declare_io("whatever")
+io.set_engine("DataMan")
+io.set_parameters({"IPAddress": "127.0.0.1", "Port": "12306", "Timeout": "5"})
 
-engine = io.Open("HelloDataMan", adios2.Mode.Read, comm)
-
-while True:
-    stepStatus = engine.BeginStep()
-    if stepStatus == adios2.StepStatus.OK:
-        var = io.InquireVariable("FloatArray")
-        shape = var.Shape()
-        floatArray = np.zeros(np.prod(shape), dtype=np.float32)
-        engine.Get(var, floatArray, adios2.Mode.Sync)
-        currentStep = engine.CurrentStep()
-        engine.EndStep()
-        print("Step", currentStep, floatArray)
-    elif stepStatus == adios2.StepStatus.EndOfStream:
-        print("End of stream")
-        break
-
-engine.Close()
+with Stream(io, "HelloDataMan", "r") as stream:
+    for _ in stream.steps():
+        stepStatus = stream.step_status()
+        print(f"Step status = {stepStatus}")
+        var = io.inquire_variable("FloatArray")
+        shape = var.shape()
+        floatArray = stream.read(var)
+        currentStep = stream.current_step()
+        loopStep = stream.loop_index()
+        print("Loop index =", loopStep, "stream step =", currentStep, "data =")
+        print(floatArray)

examples/hello/datamanWriter/dataManWriter.cpp

@@ -9,6 +9,7 @@
  */
 
 #include <adios2.h>
+#include <chrono>
 #include <iostream>
 #include <mpi.h>
 #include <numeric>
@@ -85,6 +86,7 @@ int main(int argc, char *argv[])
         engine.Put(floatArrayVar, floatVector.data(), adios2::Mode::Sync);
         PrintData(floatVector, engine.CurrentStep());
         engine.EndStep();
+        std::this_thread::sleep_for(std::chrono::milliseconds(2000));
     }
 
     engine.Close();