Merge remote-tracking branch 'origin/master' into ept

doc/stages/filters.python.rst

@@ -185,19 +185,24 @@ for the in-scope :ref:`filters.python` :cpp:class:`pdal::Stage`.
        outs = ins
        return True
 
-Updating metadata
+Setting stage metadata
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-The filter can update the global ``metadata`` dictionary as needed,
-define it as a
-**global** Python variable for the function's scope, and the updates will be
-reflected back into the pipeline from that stage forward.
+.. note::
+    The name of the output metadata variable has changed from ``metadata`` to ``out_metadata``.
+
+Stage metadata can be created by using the ``out_metadata`` dictionary **global** variable.
+The ``name`` key must be set. The type of the ``value`` can usually be inferred, but
+can be set to one of ``integer``, ``nonNegativeInteger``, ``double``, ``bounds``,
+``boolean``, ``spatialreference``, ``uuid`` or ``string``.
+
+Children may be set using the ``children`` key whose value is a list of dictionaries.
 
 .. code-block:: python
 
    def myfunc(ins,outs):
-     global metadata
-     metadata = {'name': 'root', 'value': 'a string', 'type': 'string', 'description': 'a description', 'children': [{'name': 'filters.python', 'value': 52, 'type': 'integer', 'description': 'a filter description', 'children': []}, {'name': 'readers.faux', 'value': 'another string', 'type': 'string', 'description': 'a reader description', 'children': []}]}
+     global out_metadata
+     out_metadata = {'name': 'root', 'value': 'a string', 'type': 'string', 'description': 'a description', 'children': [{'name': 'somekey', 'value': 52, 'type': 'integer', 'description': 'a filter description', 'children': []}, {'name': 'readers.faux', 'value': 'another string', 'type': 'string', 'description': 'a reader description', 'children': []}]}
      return True
 
 Passing Python objects

doc/stages/writers.gdal.rst

@@ -188,6 +188,14 @@ override_srs
 default_srs
   Write the raster with the provided SRS if none exists. [Default: None]
 
+metadata:
+  Add or set GDAL metadata to set on the raster, in the form
+  ``NAME=VALUE,NAME2=VALUE2,NAME3=VALUE3`` [Default: None]
+
+pdal_metadata:
+  Write PDAL's pipeline and metadata as base64 to the GDAL PAM metadata [Default: False]
+
+
 .. include:: writer_opts.rst
 
 .. note::

io/GDALReader.cpp

@@ -38,6 +38,7 @@
 
 #include <pdal/PointView.hpp>
 #include <pdal/private/gdal/Raster.hpp>
+#include <pdal/util/Utils.hpp>
 
 namespace pdal
 {
@@ -82,6 +83,8 @@ void GDALReader::initialize()
     m_width = m_raster->width();
     m_height = m_raster->height();
     m_bandTypes = m_raster->getPDALDimensionTypes();
+    m_metadata.add(m_raster->getMetadata());
+
 
     m_dimNames.clear();
     if (m_header.size())

io/GDALReader.hpp

@@ -70,6 +70,7 @@ class PDAL_DLL GDALReader : public Reader , public Streamable
     std::unique_ptr<gdal::Raster> m_raster;
     std::vector<Dimension::Type> m_bandTypes;
     std::vector<Dimension::Id> m_bandIds;
+    pdal::StringList m_GDAL_metadata;
     std::string m_header;
     int m_width;
     int m_height;

io/GDALWriter.cpp

@@ -37,6 +37,7 @@
 
 #include <pdal/PointView.hpp>
 #include <pdal/private/gdal/Raster.hpp>
+#include <pdal/util/Utils.hpp>
 
 #include "private/GDALGrid.hpp"
 
@@ -95,6 +96,10 @@ void GDALWriter::addArgs(ProgramArgs& args)
 
     args.add("default_srs", "Spatial reference to apply to data if one cannot be inferred",
         m_defaultSrs);
+    args.add("metadata", "GDAL metadata to set on the raster, in the form 'NAME=VALUE,NAME2=VALUE2,NAME3=VALUE3'",
+        m_GDAL_metadata);
+    args.add("pdal_metadata", "Write PDAL metadata as to GDAL PAM XML Metadata?",
+        m_writePDALMetadata, decltype(m_writePDALMetadata)(false));
 }
 
 
@@ -161,6 +166,7 @@ void GDALWriter::initialize()
     // don't expand by point if we're running in standard mode.  That's
     // set later in writeView.
     m_expandByPoint = !m_fixedGrid;
+
 }
 
 
@@ -321,6 +327,45 @@ void GDALWriter::doneFile()
         throwError(raster.errorMsg());
 
     getMetadata().addList("filename", m_filename);
+
+    std::vector<std::string> gdalitems = Utils::split(m_GDAL_metadata, ',');
+    for (auto& v: gdalitems)
+    {
+        const std::size_t pos = v.find_first_of("=");
+        if (pos != std::string::npos)
+        {
+            const std::string name = v.substr(0, pos);
+            const std::string value = pos != std::string::npos ? v.substr(pos + 1) : "";
+            raster.addMetadata(name, value);
+        }
+    }
+
+    getMetadata().add(raster.getMetadata());
 }
 
+void GDALWriter::readyTable(PointTableRef table)
+{
+    // Add these as GDAL metadata
+    if(m_writePDALMetadata)
+    {
+        MetadataNode m = table.metadata();
+        std::string json = Utils::toJSON(m);
+        std::vector<uint8_t> metadata(json.begin(), json.end());
+        std::string b64 = Utils::base64_encode(metadata.data(), metadata.size());
+        if (m_GDAL_metadata.size())
+            m_GDAL_metadata += ",";
+
+        m_GDAL_metadata += "pdal_metadata=" + b64;
+
+        std::ostringstream ostr;
+        PipelineWriter::writePipeline(this, ostr);
+        json = ostr.str();
+        std::vector<uint8_t> pipeline(json.begin(), json.end());
+        b64 = Utils::base64_encode(pipeline.data(), pipeline.size());
+        m_GDAL_metadata += ",pdal_pipeline=" + b64;
+    }
+
+}
+
+
 } // namespace pdal

io/GDALWriter.hpp

@@ -68,6 +68,7 @@ class PDAL_DLL GDALWriter : public FlexWriter, public Streamable
     virtual void addArgs(ProgramArgs& args);
     virtual void initialize();
     virtual void prepared(PointTableRef table);
+    virtual void readyTable(PointTableRef table);
     virtual void readyFile(const std::string& filename,
         const SpatialReference& srs);
     virtual void writeView(const PointViewPtr view);
@@ -107,6 +108,8 @@ class PDAL_DLL GDALWriter : public FlexWriter, public Streamable
     bool m_fixedGrid;
     SpatialReference m_defaultSrs;
     SpatialReference m_overrideSrs;
+    std::string m_GDAL_metadata;
+    bool m_writePDALMetadata;
 };
 
 }

pdal/private/gdal/Raster.cpp

@@ -639,5 +639,49 @@ GDALError Raster::statistics(int nBand, double *minimum, double *maximum,
     return GDALError::None;
 }
 
+
+MetadataNode Raster::getMetadata(std::string domain) const
+{
+    char **papszMetadata = NULL;
+
+    MetadataNode output("raster");
+
+    // m_ds owns this
+    papszMetadata = m_ds->GetMetadata(domain.c_str());
+
+    for( int i = 0;
+         papszMetadata != NULL && papszMetadata[i] != NULL;
+         i++ )
+    {
+        std::string v(papszMetadata[i]);
+
+        const std::size_t pos = v.find_first_of("=");
+        if (pos != std::string::npos)
+        {
+            const std::string name = v.substr(0, pos);
+            const std::string value = pos != std::string::npos ? v.substr(pos + 1) : "";
+            output.add(name, value);
+        }
+        else
+        {
+            throw pdal_error("Metadata must be defined in 'key=value,key2=value2' arrangement");
+        }
+
+
+    }
+
+    return output;
+}
+
+GDALError Raster::addMetadata(std::string name,
+                              std::string value,
+                              std::string domain)
+{
+    CPLErr e = m_ds->SetMetadataItem(name.c_str(),
+                                     value.c_str(),
+                                     domain.c_str());
+    return GDALError(e);
+}
+
 } // namespace gdal
 } // namespace pdal

pdal/private/gdal/Raster.hpp

@@ -40,6 +40,7 @@
 #include <pdal/pdal_types.hpp>
 #include <pdal/SpatialReference.hpp>
 #include <pdal/util/Bounds.hpp>
+#include <pdal/Metadata.hpp>
 
 #include "GDALError.hpp"
 
@@ -528,6 +529,9 @@ class PDAL_DLL Raster
     BOX2D bounds() const;
     BOX3D bounds(int nBand) const;
 
+    MetadataNode getMetadata(std::string domain="") const;
+    GDALError addMetadata(std::string name, std::string value, std::string domain="");
+
 private:
     std::string m_filename;
 

test/unit/UtilsTest.cpp

@@ -477,6 +477,8 @@ TEST(UtilsTest, escapeJSON)
     EXPECT_EQ(escaped, "\\u0001\\t\\f\\n\\\\\\\"\\u0016");
 }
 
+// Don't run if we are WIN32
+#if !defined(_WIN32) || defined(_WIN64)
 TEST(UtilsTest, map)
 {
     Support::Tempfile temp;
@@ -536,5 +538,6 @@ TEST(UtilsTest, map)
     EXPECT_EQ(std::string(c), "Another.");
     FileUtils::unmapFile(ctx);
 }
+#endif // guard for 32-bit windows
 
 }

test/unit/io/GDALReaderTest.cpp

@@ -168,6 +168,10 @@ class GDALReaderTypeTest : public ::testing::Test
             EXPECT_DOUBLE_EQ(m_xyzPoints[i].m_y, m_gdalPoints[i].m_y);
             EXPECT_DOUBLE_EQ(m_xyzPoints[i].m_z, m_gdalPoints[i].m_z);
         }
+
+        MetadataNode l = gr.getMetadata().findChild("raster");
+        if (l.empty())
+            FAIL() << "Couldn't find raster metadata";
     }
 
 private:

test/unit/io/GDALWriterTest.cpp

@@ -896,4 +896,48 @@ TEST(GDALWriterTest, srs)
     EXPECT_THROW(test("EPSG:4326", "EPSG:4326", "EPSG:2030", "EPSG:2030"), pdal_error);
 }
 
+
+TEST(GDALWriterTest, testMetadata)
+{
+    std::string infile = Support::datapath("gdal/grid.txt");
+    std::string outfile = Support::temppath("metadata.tif");
+
+    Options wo;
+    wo.add("gdaldriver", "GTiff");
+    wo.add("output_type", "max");
+    wo.add("resolution", 1);
+    wo.add("radius", .7071);
+    wo.add("metadata", "AREA_OR_PIXEL=Pixel,empty=,equals=some_more_equals===");
+    wo.add("filename", outfile);
+    wo.add("window_size", 2);
+
+    const std::string output =
+        "5.000     5.500     7.000     8.000     9.100 "
+        "4.000     4.942     6.000     7.000     8.000 "
+        "3.000     4.000     5.000     6.000     7.000 "
+        "2.000     3.000     4.400     5.400     6.400 "
+        "1.000     2.000     3.000     4.400     5.400 ";
+
+    runGdalWriter(wo, infile, outfile, output);
+
+
+    gdal::Raster raster(outfile);
+    raster.open();
+
+    MetadataNode l = raster.getMetadata().findChild("AREA_OR_PIXEL");
+    if (l.empty())
+        FAIL() << "Couldn't find raster metadata AREA_OR_PIXEL";
+
+    l = raster.getMetadata().findChild("empty");
+    EXPECT_EQ(l.value(), "");
+
+    l = raster.getMetadata().findChild("equals");
+    if (l.empty())
+        FAIL() << "Couldn't find raster metadata equals";
+
+    EXPECT_EQ(l.value(), "some_more_equals===");
+
+
+}
+
 } // namespace pdal