refactor filters.programmable and filters.predicate metadata. Support…

… passing spatialreference and schema information via JSON to Python modules
PDAL · Jan 9, 2017 · 252095a · 252095a
1 parent 8e342d8
commit 252095a
Show file tree

Hide file tree

Showing 5 changed files with 123 additions and 56 deletions.
diff --git a/kernels/InfoKernel.cpp b/kernels/InfoKernel.cpp
@@ -330,7 +330,7 @@ MetadataNode InfoKernel::run(const std::string& filename)
 void InfoKernel::dump(MetadataNode& root)
 {
     if (m_showSchema)
-        root.add(m_manager.pointTable().toMetadata().clone("schema"));
+        root.add(m_manager.pointTable().layout()->toMetadata().clone("schema"));
 
     if (m_PointCloudSchemaOutput.size() > 0)
     {

diff --git a/pdal/plang/Environment.cpp b/pdal/plang/Environment.cpp
@@ -238,34 +238,34 @@ PyObject *fromMetadata(MetadataNode m)
     std::string description = m.description();
 
     MetadataNodeList children = m.children();
-    PyObject *submeta = NULL;
+    PyObject *submeta = PyList_New(0);
     if (children.size())
     {
-        submeta = PyList_New(0);
         for (MetadataNode& child : children)
             PyList_Append(submeta, fromMetadata(child));
     }
-    PyObject *data = PyTuple_New(5);
-    PyTuple_SetItem(data, 0, PyUnicode_FromString(name.data()));
-    PyTuple_SetItem(data, 1, PyUnicode_FromString(value.data()));
-    PyTuple_SetItem(data, 2, PyUnicode_FromString(type.data()));
-    PyTuple_SetItem(data, 3, PyUnicode_FromString(description.data()));
-    PyTuple_SetItem(data, 4, submeta);
+    PyObject *data = PyDict_New();
+    PyDict_SetItemString(data, "name", PyUnicode_FromString(name.data()));
+    PyDict_SetItemString(data, "value", PyUnicode_FromString(value.data()));
+    PyDict_SetItemString(data, "type", PyUnicode_FromString(type.data()));
+    PyDict_SetItemString(data, "description", PyUnicode_FromString(description.data()));
+    PyDict_SetItemString(data, "children", submeta);
 
     return data;
 }
 
-std::string readPythonString(PyObject* list, Py_ssize_t index)
+std::string readPythonString(PyObject* dict, const std::string& key)
 {
     std::stringstream ss;
 
-    PyObject* o = PyTuple_GetItem(list, index);
+    PyObject* o = PyDict_GetItemString(dict, key.c_str());
     if (!o)
     {
         std::stringstream oss;
-        oss << "Unable to get list item number " << index << " for list of length " << PyTuple_Size(list);
+        oss << "Unable to get dictionary item '" << key << "'";
         throw pdal_error(oss.str());
     }
+
     PyObject* r = PyObject_Str(o);
     if (!r)
         throw pdal::pdal_error("unable to get repr in readPythonString");
@@ -279,31 +279,38 @@ std::string readPythonString(PyObject* list, Py_ssize_t index)
 
     return ss.str();
 }
-void addMetadata(PyObject *list, MetadataNode m)
+void addMetadata(PyObject *dict, MetadataNode m)
 {
 
-    if (!PyList_Check(list))
+    if (! dict)
+    {
         return;
+    }
 
-    for (Py_ssize_t i = 0; i < PyList_Size(list); ++i)
-    {
-        PyObject *tuple = PyList_GetItem(list, i);
-        if (!PyTuple_Check(tuple) || PyTuple_Size(tuple) != 5)
-            continue;
+    if (!PyDict_Check(dict) )
+        throw pdal::pdal_error("'metadata' member must be a dictionary!");
 
-        std::string name = readPythonString(tuple, 0);
-        std::string value = readPythonString(tuple, 1);
+    std::string name = readPythonString(dict, "name");
+    std::string value = readPythonString(dict, "value");
 
-        std::string type = readPythonString(tuple, 2);
-        if (type.empty())
-            type = Metadata::inferType(value);
+    std::string type = readPythonString(dict, "type");
+    if (type.empty())
+        type = Metadata::inferType(value);
 
-        std::string description = readPythonString(tuple, 3);
+    std::string description = readPythonString(dict, "description");
 
-        PyObject *submeta = PyTuple_GetItem(tuple, 4);
+    PyObject *submeta = PyDict_GetItemString(dict, "children");
+    if (submeta)
+    {
+        if (!PyList_Check(submeta))
+            throw pdal::pdal_error("'children' metadata member must be a list!");
+
+        for (Py_ssize_t i = 0; i < PyList_Size(submeta); ++i)
+        {
+            PyObject* p = PyList_GetItem(submeta, i);
+            addMetadata(p, m);
+        }
         MetadataNode child =  m.addWithType(name, value, type, description);
-        if (submeta)
-            addMetadata(submeta, child);
     }
 }
 

diff --git a/pdal/plang/Invocation.cpp b/pdal/plang/Invocation.cpp
@@ -76,9 +76,7 @@ namespace plang
 {
 
 Invocation::Invocation(const Script& script)
-    : m_metaIn(NULL)
-    , m_metaOut(NULL)
-    , m_script(script)
+    : m_script(script)
     , m_bytecode(NULL)
     , m_module(NULL)
     , m_dictionary(NULL)
@@ -87,6 +85,9 @@ Invocation::Invocation(const Script& script)
     , m_varsOut(NULL)
     , m_scriptArgs(NULL)
     , m_scriptResult(NULL)
+    , m_metadata_PyObject(NULL)
+    , m_schema_PyObject(NULL)
+    , m_srs_PyObject(NULL)
 {
     plang::Environment::get();
     resetArguments();
@@ -137,8 +138,6 @@ void Invocation::cleanup()
         Py_XDECREF(m_pyInputArrays[i]);
     m_pyInputArrays.clear();
     Py_XDECREF(m_bytecode);
-    Py_XDECREF(m_metaIn);
-    Py_XDECREF(m_metaOut);
 }
 
 
@@ -147,8 +146,6 @@ void Invocation::resetArguments()
     cleanup();
     m_varsIn = PyDict_New();
     m_varsOut = PyDict_New();
-    m_metaIn = PyList_New(0);
-    m_metaOut = PyList_New(0);
 }
 
 
@@ -268,24 +265,72 @@ bool Invocation::execute()
     Py_INCREF(m_varsOut);
     Py_ssize_t numArgs = argCount(m_function);
     m_scriptArgs = PyTuple_New(numArgs);
+
     PyTuple_SetItem(m_scriptArgs, 0, m_varsIn);
     if (numArgs > 1)
         PyTuple_SetItem(m_scriptArgs, 1, m_varsOut);
-    if (numArgs > 2)
-        PyTuple_SetItem(m_scriptArgs, 2, m_metaIn);
-    if (numArgs > 3)
-        PyTuple_SetItem(m_scriptArgs, 3, m_metaOut);
 
-    m_scriptResult = PyObject_CallObject(m_function, m_scriptArgs);
+
+    int success = PyModule_AddObject(m_module, "metadata", m_metadata_PyObject);
+    if (success)
+        throw pdal::pdal_error("unable to set metadata global");
+
+    success = PyModule_AddObject(m_module, "schema", m_schema_PyObject);
+    if (success)
+        throw pdal::pdal_error("unable to set schema global");
+
+    success = PyModule_AddObject(m_module, "spatialreference", m_srs_PyObject);
+    if (success)
+        throw pdal::pdal_error("unable to set spatialreference global");
+
+    m_scriptResult = PyObject_Call(m_function, m_scriptArgs, NULL);
     if (!m_scriptResult)
         throw pdal::pdal_error(getTraceback());
 
-    if (!PyBool_Check(m_scriptResult))
-        throw pdal::pdal_error("User function return value not a boolean type.");
+    PyObject* mod_vars = PyModule_GetDict(m_module);
+
+    PyObject* b =  PyUnicode_FromString("metadata");
+    if (PyDict_Contains(mod_vars, PyUnicode_FromString("metadata")) == 1)
+        m_metadata_PyObject = PyDict_GetItem(m_dictionary, b);
 
-    return (m_scriptResult == Py_True);
+    return true;
 }
 
+PyObject* getPyJSON(MetadataNode node)
+{
+
+    std::ostringstream schema_strm;
+    Utils::toJSON(node, schema_strm);
+
+    PyObject* raw_json =  PyUnicode_FromString(schema_strm.str().c_str());
+    PyObject* json_module = PyImport_ImportModule("json");
+    if (!json_module)
+        throw pdal::pdal_error(getTraceback());
+
+    PyObject* json_mod_dict = PyModule_GetDict(json_module);
+    if (!json_mod_dict)
+        throw pdal::pdal_error(getTraceback());
+
+    PyObject* loads_func = PyDict_GetItemString(json_mod_dict, "loads");
+    if (!loads_func)
+        throw pdal::pdal_error(getTraceback());
+
+    PyObject* json_args = PyTuple_New(1);
+    if (!json_args)
+        throw pdal::pdal_error(getTraceback());
+
+    int success = PyTuple_SetItem(json_args, 0, raw_json);
+    if (success != 0)
+        throw pdal::pdal_error(getTraceback());
+
+    PyObject* json = PyObject_CallObject(loads_func, json_args);
+    if (!json)
+        throw pdal::pdal_error(getTraceback());
+
+    return json;
+}
+
+
 void Invocation::begin(PointView& view, MetadataNode m)
 {
     PointLayoutPtr layout(view.m_pointTable.layout());
@@ -306,8 +351,19 @@ void Invocation::begin(PointView& view, MetadataNode m)
         std::string name = layout->dimName(*di);
         insertArgument(name, (uint8_t *)data, dd->type(), view.size());
     }
-    Py_XDECREF(m_metaIn);
-    m_metaIn = plang::fromMetadata(m);
+
+    // Put pipeline 'metadata' variable into module scope
+    Py_XDECREF(m_metadata_PyObject);
+    m_metadata_PyObject= plang::fromMetadata(m);
+
+    // Put 'schema' dict into module scope
+    Py_XDECREF(m_schema_PyObject);
+    MetadataNode s = view.layout()->toMetadata();
+    m_schema_PyObject = getPyJSON(s);
+
+    Py_XDECREF(m_srs_PyObject);
+    MetadataNode srs = view.spatialReference().toMetadata();
+    m_srs_PyObject = getPyJSON(srs);
 }
 
 
@@ -347,7 +403,7 @@ void Invocation::end(PointView& view, MetadataNode m)
     for (auto bi = m_buffers.begin(); bi != m_buffers.end(); ++bi)
         free(*bi);
     m_buffers.clear();
-    addMetadata(m_metaOut, m);
+    addMetadata(m_metadata_PyObject, m);
 }
 
 } // namespace plang

diff --git a/pdal/plang/Invocation.hpp b/pdal/plang/Invocation.hpp
@@ -85,10 +85,6 @@ class PDAL_DLL Invocation
     void begin(PointView& view, MetadataNode m);
     void end(PointView& view, MetadataNode m);
 
-protected:
-    PyObject* m_metaIn;
-    PyObject* m_metaOut;
-
 private:
     void cleanup();
 
@@ -108,6 +104,9 @@ class PDAL_DLL Invocation
     Invocation& operator=(Invocation const& rhs); // nope
 
     std::vector<void *> m_buffers;
+    PyObject* m_metadata_PyObject;
+    PyObject* m_schema_PyObject;
+    PyObject* m_srs_PyObject;
 };
 
 } // namespace plang

diff --git a/plugins/python/test/ProgrammableFilterTest.cpp b/plugins/python/test/ProgrammableFilterTest.cpp
@@ -204,9 +204,13 @@ TEST_F(ProgrammableFilterTest, metadata)
     reader.setOptions(ops);
 
     Option source("source", "import numpy\n"
-        "def myfunc(ins,outs,inmeta,outmeta):\n"
-        "  t = ('name', 'value', '', '', [])\n"
-        "  outmeta.append(t)\n"
+        "import sys\n"
+        "import redirector\n"
+        "def myfunc(ins,outs):\n"
+        "  global metadata\n"
+        "  #print('before', globals(),  file=sys.stderr,)\n"
+        "  metadata = {'name': 'root', 'value': 'a string', 'type': 'string', 'description': 'a description', 'children': [{'name': 'filters.programmable', 'value': 52, 'type': 'integer', 'description': 'a filter description', 'children': []}, {'name': 'readers.faux', 'value': 'another string', 'type': 'string', 'description': 'a reader description', 'children': []}]}\n"
+        " # print ('schema', schema, file=sys.stderr,)\n"
         "  return True\n"
     );
     Option module("module", "MyModule");
@@ -230,7 +234,8 @@ TEST_F(ProgrammableFilterTest, metadata)
     MetadataNode m = table.metadata();
     m = m.findChild("filters.programmable");
     MetadataNodeList l = m.children();
-    EXPECT_EQ(l.size(), 1u);
-    EXPECT_EQ(l[0].name(), "name");
-    EXPECT_EQ(l[0].value(), "value");
+    EXPECT_EQ(l.size(), 3u);
+    EXPECT_EQ(l[0].name(), "filters.programmable");
+    EXPECT_EQ(l[0].value(), "52");
+    EXPECT_EQ(l[0].description(), "a filter description");
 }