Skip to content

Commit

Permalink
Merge pull request #4036 from pnorbert/python-write-scalars
Browse files Browse the repository at this point in the history 
Python write scalars
  • Loading branch information
@pnorbert
pnorbert committed Feb 21, 2024
2 parents c498921 + 45571f6 commit cea2adc
Show file tree
Hide file tree
Showing 31 changed files with 814 additions and 384 deletions.
34 changes: 33 additions & 1 deletion bindings/Python/py11Engine.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -70,12 +70,44 @@ void Engine::Put(Variable variable, const pybind11::array &array, const Mode lau
else
{
throw std::invalid_argument("ERROR: for variable " + variable.Name() +
" numpy array type is not supported or "
" numpy array type " + variable.Type() +
" is not supported (found type " + ToString(type) +
") or "
"is not memory contiguous "
", in call to Put\n");
}
}

void Engine::Put(Variable variable, const std::vector<int64_t> &ints, const Mode launch)
{
helper::CheckForNullptr(m_Engine, "in call to Engine::Put list of ints");
helper::CheckForNullptr(variable.m_VariableBase,
"for variable, in call to Engine::Put list of ints");

m_Engine->Put(*dynamic_cast<core::Variable<int64_t> *>(variable.m_VariableBase),
reinterpret_cast<const int64_t *>(ints.data()), launch);
}

void Engine::Put(Variable variable, const std::vector<double> &floats, const Mode launch)
{
helper::CheckForNullptr(m_Engine, "in call to Engine::Put list of floats");
helper::CheckForNullptr(variable.m_VariableBase,
"for variable, in call to Engine::Put list of floats");

m_Engine->Put(*dynamic_cast<core::Variable<double> *>(variable.m_VariableBase),
reinterpret_cast<const double *>(floats.data()), launch);
}

void Engine::Put(Variable variable, const std::vector<std::complex<double>> &complexes,
const Mode launch)
{
helper::CheckForNullptr(m_Engine, "in call to Engine::Put list of complexes");
helper::CheckForNullptr(variable.m_VariableBase,
"for variable, in call to Engine::Put list of complexes");
m_Engine->Put(*dynamic_cast<core::Variable<std::complex<double>> *>(variable.m_VariableBase),
reinterpret_cast<const std::complex<double> *>(complexes.data()), launch);
}

void Engine::Put(Variable variable, const std::string &string)
{
helper::CheckForNullptr(m_Engine, "for engine, in call to Engine::Put string");
Expand Down
6 changes: 6 additions & 0 deletions bindings/Python/py11Engine.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,12 @@ class Engine
StepStatus BeginStep();

void Put(Variable variable, const pybind11::array &array, const Mode launch = Mode::Deferred);
void Put(Variable variable, const std::vector<int64_t> &ints,
const Mode launch = Mode::Deferred);
void Put(Variable variable, const std::vector<double> &doubles,
const Mode launch = Mode::Deferred);
void Put(Variable variable, const std::vector<std::complex<double>> &complexes,
const Mode launch = Mode::Deferred);
void Put(Variable variable, const std::string &string);
void PerformPuts();
void PerformDataWrite();
Expand Down
116 changes: 113 additions & 3 deletions bindings/Python/py11IO.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -101,6 +101,51 @@ Variable IO::DefineVariable(const std::string &name, const pybind11::array &arra
return Variable(variable);
}

Variable IO::DefineVariable(const std::string &name, const pybind11::object &value,
const Dims &shape, const Dims &start, const Dims &count,
const bool isConstantDims)
{
helper::CheckForNullptr(m_IO, "for variable " + name + ", in call to IO::DefineVariable");
core::VariableBase *variable = nullptr;
const auto t = value.get_type();
const auto ts = pybind11::str(t);
const auto tss = pybind11::cast<std::string>(ts);
if (pybind11::isinstance<pybind11::str>(value))
{
variable = &m_IO->DefineVariable<std::string>(name);
}
else if (pybind11::isinstance<pybind11::int_>(value))
{
variable = &m_IO->DefineVariable<int64_t>(name, shape, start, count, isConstantDims);
}
else if (pybind11::isinstance<pybind11::float_>(value))
{
variable = &m_IO->DefineVariable<double>(name, shape, start, count, isConstantDims);
}
else if (tss == "<class 'complex'>")
{
variable =
&m_IO->DefineVariable<std::complex<double>>(name, shape, start, count, isConstantDims);
}
else if (tss == "<class 'numpy.complex64'>")
{
variable =
&m_IO->DefineVariable<std::complex<float>>(name, shape, start, count, isConstantDims);
}
else if (tss == "<class 'numpy.complex128'>")
{
variable =
&m_IO->DefineVariable<std::complex<double>>(name, shape, start, count, isConstantDims);
}
else
{
throw std::invalid_argument("ERROR: variable " + name +
" can't be defined with an object with type " + tss +
", in call to DefineVariable\n");
}
return Variable(variable);
}

Variable IO::InquireVariable(const std::string &name)
{
helper::CheckForNullptr(m_IO, "for variable " + name + ", in call to IO::InquireVariable");
Expand All @@ -126,7 +171,6 @@ Attribute IO::DefineAttribute(const std::string &name, const pybind11::array &ar
const std::string &variableName, const std::string separator)
{
helper::CheckForNullptr(m_IO, "for attribute " + name + ", in call to IO::DefineAttribute");

core::AttributeBase *attribute = nullptr;

if (false)
Expand Down Expand Up @@ -156,7 +200,6 @@ Attribute IO::DefineAttribute(const std::string &name, const std::string &string
const std::string &variableName, const std::string separator)
{
helper::CheckForNullptr(m_IO, "for attribute " + name + ", in call to IO::DefineAttribute");

return Attribute(
&m_IO->DefineAttribute<std::string>(name, stringValue, variableName, separator));
}
Expand All @@ -165,11 +208,78 @@ Attribute IO::DefineAttribute(const std::string &name, const std::vector<std::st
const std::string &variableName, const std::string separator)
{
helper::CheckForNullptr(m_IO, "for attribute " + name + ", in call to IO::DefineAttribute");

return Attribute(&m_IO->DefineAttribute<std::string>(name, strings.data(), strings.size(),
variableName, separator));
}

Attribute IO::DefineAttribute(const std::string &name, const std::vector<int> &ints,
const std::string &variableName, const std::string separator)
{
helper::CheckForNullptr(m_IO, "for attribute " + name + ", in call to IO::DefineAttribute");
return Attribute(
&m_IO->DefineAttribute<int>(name, ints.data(), ints.size(), variableName, separator));
}

Attribute IO::DefineAttribute(const std::string &name, const std::vector<double> &doubles,
const std::string &variableName, const std::string separator)
{
helper::CheckForNullptr(m_IO, "for attribute " + name + ", in call to IO::DefineAttribute");
return Attribute(&m_IO->DefineAttribute<double>(name, doubles.data(), doubles.size(),
variableName, separator));
}

Attribute IO::DefineAttribute(const std::string &name,
const std::vector<std::complex<double>> &complexdoubles,
const std::string &variableName, const std::string separator)
{
helper::CheckForNullptr(m_IO, "for attribute " + name + ", in call to IO::DefineAttribute");
return Attribute(&m_IO->DefineAttribute<std::complex<double>>(
name, complexdoubles.data(), complexdoubles.size(), variableName, separator));
}

Attribute IO::DefineAttribute(const std::string &name, const pybind11::object &value,
const std::string &variableName, const std::string separator)
{
helper::CheckForNullptr(m_IO, "for attribute " + name + ", in call to IO::DefineAttribute");

core::AttributeBase *attribute = nullptr;
const auto t = value.get_type();
const auto ts = pybind11::str(t);
const auto tss = pybind11::cast<std::string>(ts);
if (pybind11::isinstance<pybind11::int_>(value))
{
auto v = pybind11::cast<const int64_t>(value);
attribute = &m_IO->DefineAttribute(name, v, variableName, separator);
}
else if (pybind11::isinstance<pybind11::float_>(value))
{
auto v = pybind11::cast<const double>(value);
attribute = &m_IO->DefineAttribute(name, v, variableName, separator);
}
else if (tss == "<class 'complex'>")
{
auto v = pybind11::cast<const std::complex<double>>(value);
attribute = &m_IO->DefineAttribute(name, v, variableName, separator);
}
else if (tss == "<class 'numpy.complex64'>")
{
auto v = pybind11::cast<const std::complex<float>>(value);
attribute = &m_IO->DefineAttribute(name, v, variableName, separator);
}
else if (tss == "<class 'numpy.complex128'>")
{
auto v = pybind11::cast<const std::complex<double>>(value);
attribute = &m_IO->DefineAttribute(name, v, variableName, separator);
}
else
{
throw std::invalid_argument("ERROR: attribute " + name +
" can't be defined with an object with type " + tss +
", in call to DefineAttribute\n");
}
return Attribute(attribute);
}

Attribute IO::InquireAttribute(const std::string &name, const std::string &variableName,
const std::string separator)
{
Expand Down
21 changes: 21 additions & 0 deletions bindings/Python/py11IO.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@

#include <pybind11/numpy.h>

#include <complex>
#include <string>

#include "py11Attribute.h"
Expand Down Expand Up @@ -57,6 +58,10 @@ class IO
const Dims &shape, const Dims &start, const Dims &count,
const bool isConstantDims);

Variable DefineVariable(const std::string &name, const pybind11::object &value,
const Dims &shape, const Dims &start, const Dims &count,
const bool isConstantDims);

Variable InquireVariable(const std::string &name);

Attribute DefineAttribute(const std::string &name, const pybind11::array &array,
Expand All @@ -71,6 +76,22 @@ class IO
const std::string &variableName = "",
const std::string separator = "/");

Attribute DefineAttribute(const std::string &name, const std::vector<int> &ints,
const std::string &variableName = "",
const std::string separator = "/");

Attribute DefineAttribute(const std::string &name, const std::vector<double> &doubles,
const std::string &variableName = "",
const std::string separator = "/");

Attribute DefineAttribute(const std::string &name,
const std::vector<std::complex<double>> &complexdoubles,
const std::string &variableName = "",
const std::string separator = "/");

Attribute DefineAttribute(const std::string &name, const pybind11::object &value,
const std::string &variableName, const std::string separator);

Attribute InquireAttribute(const std::string &name, const std::string &variableName = "",
const std::string separator = "/");

Expand Down
64 changes: 64 additions & 0 deletions bindings/Python/py11glue.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@
#include <pybind11/pybind11.h>
#include <pybind11/stl.h>

#include <complex>
#include <sstream>
#include <stdexcept>

Expand Down Expand Up @@ -203,6 +204,15 @@ PYBIND11_MODULE(ADIOS2_PYTHON_MODULE_NAME, m)
pybind11::arg("shape") = adios2::Dims(), pybind11::arg("start") = adios2::Dims(),
pybind11::arg("count") = adios2::Dims(), pybind11::arg("isConstantDims") = false)

.def("DefineVariable",
(adios2::py11::Variable(adios2::py11::IO::*)(
const std::string &, const pybind11::object &, const adios2::Dims &,
const adios2::Dims &, const adios2::Dims &, const bool)) &
adios2::py11::IO::DefineVariable,
pybind11::return_value_policy::move, pybind11::arg("name"), pybind11::arg("value"),
pybind11::arg("shape") = adios2::Dims(), pybind11::arg("start") = adios2::Dims(),
pybind11::arg("count") = adios2::Dims(), pybind11::arg("isConstantDims") = false)

.def("DefineVariable",
(adios2::py11::Variable(adios2::py11::IO::*)(const std::string &)) &
adios2::py11::IO::DefineVariable,
Expand Down Expand Up @@ -243,6 +253,38 @@ PYBIND11_MODULE(ADIOS2_PYTHON_MODULE_NAME, m)
pybind11::arg("name"), pybind11::arg("strings"), pybind11::arg("variable_name") = "",
pybind11::arg("separator") = "/", pybind11::return_value_policy::move)

.def("DefineAttribute",
(adios2::py11::Attribute(adios2::py11::IO::*)(
const std::string &, const std::vector<int> &, const std::string &,
const std::string)) &
adios2::py11::IO::DefineAttribute,
pybind11::arg("name"), pybind11::arg("ints"), pybind11::arg("variable_name") = "",
pybind11::arg("separator") = "/", pybind11::return_value_policy::move)

.def("DefineAttribute",
(adios2::py11::Attribute(adios2::py11::IO::*)(
const std::string &, const std::vector<double> &, const std::string &,
const std::string)) &
adios2::py11::IO::DefineAttribute,
pybind11::arg("name"), pybind11::arg("doubles"), pybind11::arg("variable_name") = "",
pybind11::arg("separator") = "/", pybind11::return_value_policy::move)

.def("DefineAttribute",
(adios2::py11::Attribute(adios2::py11::IO::*)(
const std::string &, const std::vector<std::complex<double>> &,
const std::string &, const std::string)) &
adios2::py11::IO::DefineAttribute,
pybind11::arg("name"), pybind11::arg("complexes"), pybind11::arg("variable_name") = "",
pybind11::arg("separator") = "/", pybind11::return_value_policy::move)

.def("DefineAttribute",
(adios2::py11::Attribute(adios2::py11::IO::*)(
const std::string &, const pybind11::object &, const std::string &,
const std::string)) &
adios2::py11::IO::DefineAttribute,
pybind11::arg("name"), pybind11::arg("value"), pybind11::arg("variable_name") = "",
pybind11::arg("separator") = "/", pybind11::return_value_policy::move)

.def("Open", (adios2::py11::Engine(adios2::py11::IO::*)(const std::string &, const int)) &
adios2::py11::IO::Open)
#if ADIOS2_USE_MPI
Expand Down Expand Up @@ -385,6 +427,28 @@ PYBIND11_MODULE(ADIOS2_PYTHON_MODULE_NAME, m)
.def("Put", (void(adios2::py11::Engine::*)(adios2::py11::Variable, const std::string &)) &
adios2::py11::Engine::Put)

.def("Put",
(void(adios2::py11::Engine::*)(adios2::py11::Variable, const std::vector<int64_t> &,
const adios2::Mode launch)) &
adios2::py11::Engine::Put,
pybind11::arg("variable"), pybind11::arg("ints"),
pybind11::arg("launch") = adios2::Mode::Sync)

.def("Put",
(void(adios2::py11::Engine::*)(adios2::py11::Variable, const std::vector<double> &,
const adios2::Mode launch)) &
adios2::py11::Engine::Put,
pybind11::arg("variable"), pybind11::arg("floats"),
pybind11::arg("launch") = adios2::Mode::Sync)

.def("Put",
(void(adios2::py11::Engine::*)(adios2::py11::Variable,
const std::vector<std::complex<double>> &,
const adios2::Mode launch)) &
adios2::py11::Engine::Put,
pybind11::arg("variable"), pybind11::arg("complexes"),
pybind11::arg("launch") = adios2::Mode::Sync)

.def("PerformPuts", &adios2::py11::Engine::PerformPuts)

.def("PerformDataWrite", &adios2::py11::Engine::PerformDataWrite)
Expand Down
34 changes: 21 additions & 13 deletions examples/hello/bpReader/bpReaderHeatMap2D.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@

from mpi4py import MPI
import numpy
from adios2 import Stream
from adios2 import Stream, FileReader

# MPI
comm = MPI.COMM_WORLD
Expand All @@ -39,15 +39,23 @@

with Stream("HeatMap2D_py.bp", "w", comm) as obpStream:
obpStream.write("temperature2D", temperatures, shape, start, count)

if rank == 0:
with Stream("HeatMap2D_py.bp", "r", MPI.COMM_SELF) as ibpStream:
for _ in ibpStream.steps():
var_inTemperature = ibpStream.inquire_variable("temperature2D")
if var_inTemperature is not None:
var_inTemperature.set_selection([[2, 2], [4, 4]])
inTemperatures = ibpStream.read(var_inTemperature)

print("Incoming temperature map")
for i in range(0, inTemperatures.shape[1]):
print(str(inTemperatures[i]) + " ")
if not rank:
obpStream.write("N", [size, Nx, Ny]) # will be an array in output
obpStream.write("Nx", numpy.array(Nx)) # will be a scalar in output
obpStream.write("Ny", Ny) # will be a scalar in output
obpStream.write_attribute("dimensions", [size * Nx, Ny], "temperature2D")

if not rank:
with FileReader("HeatMap2D_py.bp", MPI.COMM_SELF) as ibpFile:
var_inTemperature = ibpFile.inquire_variable("temperature2D")
if var_inTemperature is not None:
var_inTemperature.set_selection([[2, 2], [4, 4]])
inTemperatures = ibpFile.read(var_inTemperature)

in_nx = ibpFile.read("Nx") # scalar is read as a numpy array with 1 element
in_ny = ibpFile.read("Ny") # scalar is read as a numpy array with 1 element
print(f"Incoming nx, ny = {in_nx}, {in_ny}")

print("Incoming temperature map")
for i in range(0, inTemperatures.shape[1]):
print(str(inTemperatures[i]))

0 comments on commit cea2adc

Please sign in to comment.