Skip to content

Commit

Permalink
wip single array
Browse files Browse the repository at this point in the history
  • Loading branch information
@hobu
hobu committed Sep 3, 2015
1 parent 397582a commit 81f6671
Show file tree
Hide file tree
Showing 4 changed files with 89 additions and 77 deletions.
10 changes: 4 additions & 6 deletions include/pdal/plang/Array.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -56,14 +56,12 @@ class PDAL_DLL Array
// adds the new variable to the arguments dictionary
void update(PointViewPtr view);

void* getArray(const std::string& name) const;
void *extractResult(const std::string& name,
Dimension::Type::Enum dataType);
std::vector<void*> getPythonArrays() const;
inline void* getPythonArray() const { return m_py_array; }

std::vector<std::string> getArrayNames() const;
bool hasOutputVariable(const std::string& name) const;
std::string buildNumpyDescription() const;
void* buildNumpyDescription() const;


// after a call to execute, this function will return you a list of
Expand All @@ -78,8 +76,8 @@ class PDAL_DLL Array
private:
void cleanup();

std::map<std::string, void*> m_py_arrays;
std::map<void*, std::unique_ptr<std::vector<uint8_t> > > m_data_arrays;
void* m_py_array;
std::unique_ptr<std::vector<uint8_t> > m_data_array;
PointLayoutPtr m_layout;

Array& operator=(Array const& rhs);
Expand Down
8 changes: 3 additions & 5 deletions python/libpdalpython/libpdalpython.pyx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ from cython.operator cimport dereference as deref, preincrement as inc

cdef extern from "pdal/plang/Array.hpp" namespace "pdal::plang":
cdef cppclass Array:
vector[void*] getPythonArrays() except+
void* getPythonArray() except+


np.import_array()
Expand All @@ -23,7 +23,6 @@ cdef extern from "Pipeline.hpp" namespace "libpdalpython":
Pipeline(const char* ) except +
void execute() except +
const char* getXML()
# vector[PyArrayObject*] getArrays() except +
vector[shared_ptr[Array]] getArrays() except +


Expand All @@ -46,9 +45,8 @@ cdef class PyPipeline:
while it != v.end():
ptr = deref(it)
a = ptr.get()
o = a.getPythonArrays()
for t in o:
output.append(<object>t)
o = a.getPythonArray()
output.append(<object>o)
inc(it)
return output

Expand Down
2 changes: 1 addition & 1 deletion python/test/test_libpdal.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ def test_arrays(self):
for a in arrays:
print (a[1])
import pdb;pdb.set_trace()
self.assertEqual(len(arrays), 16)
self.assertEqual(len(arrays), 1)

def test_suite():
return unittest.TestSuite(
Expand Down
146 changes: 81 additions & 65 deletions src/plang/Array.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,7 @@ namespace plang
{

Array::Array()
: m_py_array(0)
{
auto initNumpy = []()
{
Expand All @@ -75,43 +76,58 @@ Array::~Array()

void Array::cleanup()
{
for (auto i: m_py_arrays)
{
PyObject* p = (PyObject*)(i.second);
Py_XDECREF(p);
}

for (auto& i: m_data_arrays)
{
i.second.reset(nullptr);
}
m_py_arrays.clear();
m_data_arrays.clear();
// for (auto i: m_py_arrays)
// {
// PyObject* p = (PyObject*)(i.second);
// Py_XDECREF(p);
// }
//
// for (auto& i: m_data_arrays)
// {
// i.second.reset(nullptr);
// }
// m_py_arrays.clear();
// m_data_arrays.clear();
}
std::string Array::buildNumpyDescription() const
void* Array::buildNumpyDescription() const
{
std::string output;
std::stringstream oss;
Dimension::IdList dims = m_layout->dims();

PyObject* list = PyList_New(dims.size());

for (Dimension::IdList::size_type i=0; i < dims.size(); ++i)
{
Dimension::Id::Enum id = (dims[i]);
Dimension::Type::Enum t = m_layout->dimType(id);
const int pyDataType = getPythonDataType(t);

PyArray_Descr *dtype = PyArray_DescrNewFromType(pyDataType);
Py_INCREF(dtype);
npy_intp stride = m_layout->dimSize(id);

std::string name = m_layout->dimName(id);
}

std::string kind("i");
Dimension::BaseType::Enum b = Dimension::base(t);
if (b == Dimension::BaseType::Unsigned)
kind = "u";
else if (b == Dimension::BaseType::Floating)
kind = "f";
PyObject* pyName = PyUnicode_FromString(name.c_str());

oss << kind << stride;
PyObject* pyFormat = PyUnicode_FromString(oss.str().c_str());

PyObject* row = PyTuple_New(2);
PyTuple_SetItem(row, 0, pyName);
PyTuple_SetItem(row, 1, pyFormat);

PyList_SetItem(list, i, row);

return output;
oss.str("");
}
// PyObject* obj = PyUnicode_AsASCIIString(PyObject_Str(list));
// const char* s = PyBytes_AsString(obj);
// std::string output(s);
return (void*) list;
// return output;
}
void Array::update(PointViewPtr view)
{
Expand All @@ -122,73 +138,73 @@ void Array::update(PointViewPtr view)
Dimension::IdList dims = m_layout->dims();
npy_intp mydims = view->size();
npy_intp* ndims = &mydims;
std::vector<npy_intp> strides(mydims);


DataPtr pdata( new std::vector<uint8_t>(m_layout->pointSize()* view->size(), 0));

uint8_t* sp = pdata.get()->data();
for (Dimension::IdList::size_type i=0; i < dims.size(); ++i)
{
Dimension::Id::Enum id = (dims[i]);
Dimension::Type::Enum t = m_layout->dimType(id);

npy_intp stride = m_layout->dimSize(id);
npy_intp* strides = &stride;

strides [i] = stride;

std::string name = m_layout->dimName(id);
std::cout << "name: " << name << " size: " << stride << std::endl;

DataPtr pdata( new std::vector<uint8_t>(stride * view->size(), 0));
uint8_t* sp = pdata.get()->data();

uint8_t* p(sp);
for (PointId idx = 0; idx < view->size(); ++idx)
// uint8_t* p(sp);
// for (PointId idx = 0; idx < view->size(); ++idx)
// {
// view->getRawField(id, idx, p);
// p += stride;
// }

}
int flags = NPY_CARRAY | NPY_BEHAVED; // NPY_BEHAVED
// const int pyDataType = getPythonDataType(t);
// PyObject* pyArray = PyArray_SimpleNewFromData(nd, ndims, pyDataType, sp);
// PyObject* pyName = PyBytes_FromString(name.c_str());
// PyObject* pyAttrName = PyBytes_FromString("name");
// int did_set = PyObject_SetItem(pyArray, pyAttrName, pyName);
// std::cout << "did_set: " << did_set << std::endl;


PyArray_Descr *dtype(0);
PyObject * dtype_list = (PyObject*) buildNumpyDescription();
if (!dtype_list) throw pdal_error("we're nothing!");
int did_convert = PyArray_DescrConverter(dtype_list, &dtype);
if (did_convert == NPY_FAIL) throw pdal_error("did not convert!");
Py_XDECREF(dtype_list);
PyObject * pyArray = PyArray_NewFromDescr(&PyArray_Type, dtype, nd, ndims, strides.data(), sp, flags, NULL);
// PyObject* pyArray = PyArray_New(&PyArray_Type, nd, ndims, pyDataType,
// strides, tp, 0, flags, NULL);

// copy the data
//
uint8_t* p(sp);
for (PointId idx = 0; idx < view->size(); ++idx)
{

for (Dimension::IdList::size_type i=0; i < dims.size(); ++i)
{
Dimension::Id::Enum id = (dims[i]);
npy_intp stride = m_layout->dimSize(id);
view->getRawField(id, idx, p);
p += stride;
}

int flags = NPY_CARRAY; // NPY_BEHAVED
const int pyDataType = getPythonDataType(t);
PyObject* pyArray = PyArray_SimpleNewFromData(nd, ndims, pyDataType, sp);
PyObject* pyName = PyBytes_FromString(name.c_str());
PyObject* pyAttrName = PyBytes_FromString("name");
int did_set = PyObject_SetItem(pyArray, pyAttrName, pyName);
std::cout << "did_set: " << did_set << std::endl;
// PyObject* pyArray = PyArray_New(&PyArray_Type, nd, ndims, pyDataType,
// strides, tp, 0, flags, NULL);

m_py_arrays.insert(std::pair<std::string, PyObject*>(name,pyArray));
m_data_arrays.insert(std::make_pair((void*)pyArray, std::move(pdata)));
}

}
m_py_array = pyArray;
m_data_array = std::move(pdata);

void* Array::getArray(std::string const& name) const
{
auto found = m_py_arrays.find(name);
if (found != m_py_arrays.end())
return found->second;
else
return 0;
}

std::vector<void*> Array::getPythonArrays() const
{
std::vector<void*> output;
for (auto i: m_py_arrays)
{
output.push_back(i.second);
}

return output;

}
std::vector<std::string> Array::getArrayNames() const
{
std::vector<std::string> output;
for (auto i: m_py_arrays)
{
output.push_back(i.first);
}
return output;
}

void *Array::extractResult(std::string const& name,
Dimension::Type::Enum t)
Expand Down

0 comments on commit 81f6671

Please sign in to comment.