Skip to content

Commit

Permalink
tiledb: use internal array metadata for pipeline serialization (#2838)
Browse files Browse the repository at this point in the history 
* tiledb: use internal array metadata for pipeline serialization

* tiledb: added srs support

* removed unused sidecar reference in reader test

* fixed tests for PR

* include stage metadata

* added json prop check if metadata is an empty array

* fix bug with m_arrayName and TileDB lt 1.7

* tiledb: Fix irrelevant comment

* enable tiledb tests
  • Loading branch information
@normanb
normanb committed Feb 27, 2020
1 parent eefca8c commit 5be7c48
Show file tree
Hide file tree
Showing 7 changed files with 156 additions and 43 deletions.
5 changes: 2 additions & 3 deletions plugins/tiledb/CMakeLists.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ PDAL_ADD_PLUGIN(reader_libname reader tiledb
${TILEDB_LIBRARIES}
INCLUDES
${TILEDB_INCLUDE_DIR}
${NLOHMANN_INCLUDE_DIR}
)

if (WIN32)
Expand All @@ -42,8 +43,6 @@ endif()

if (WITH_TESTS)

if (BUILD_TILEDB_TESTS)

# Need to fix this #2596
IF (NOT APPLE)
PDAL_ADD_TEST(pdal_io_tiledb_writer_test
Expand All @@ -63,6 +62,7 @@ if (BUILD_TILEDB_TESTS)
test/TileDBReaderTest.cpp
LINK_WITH
${reader_libname}
${writer_libname}
${TILEDB_LIBRARIES}
)

Expand All @@ -72,4 +72,3 @@ if (WIN32)
endif()

endif()
endif()
47 changes: 47 additions & 0 deletions plugins/tiledb/io/TileDBReader.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,8 +34,17 @@

#include <algorithm>

#include <nlohmann/json.hpp>

#include "TileDBReader.hpp"


const char pathSeparator =
#ifdef _WIN32
'\\';
#else
'/';
#endif
namespace pdal {

static PluginInfo const s_info
Expand Down Expand Up @@ -283,6 +292,44 @@ void TileDBReader::localReady()
m_query->set_subarray(subarray);
}

// read spatial reference
NL::json meta = nullptr;

#if TILEDB_VERSION_MAJOR >= 1 && TILEDB_VERSION_MINOR >= 7
tiledb_datatype_t v_type = TILEDB_UINT8;
const void* v_r;
uint32_t v_num;
m_array->get_metadata("_pdal", &v_type, &v_num, &v_r);
if (v_r != NULL)
meta = NL::json::parse(static_cast<const char*>(v_r));
#endif

if (meta == nullptr)
{
tiledb::VFS vfs(*m_ctx, m_ctx->config());
tiledb::VFS::filebuf fbuf(vfs);
std::string metaFName = m_filename + pathSeparator + "pdal.json";

if (vfs.is_dir(m_filename))
{
auto nBytes = vfs.file_size(metaFName);
tiledb::VFS::filebuf fbuf(vfs);
fbuf.open(metaFName, std::ios::in);
std::istream is(&fbuf);
std::string s { std::istreambuf_iterator<char>(is), std::istreambuf_iterator<char>() };
fbuf.close();
meta = NL::json::parse(s);
}
}

if ((meta != nullptr) &&
(meta.count("writers.tiledb") > 0) &&
(meta["writers.tiledb"].count("spatialreference") > 0))
{
SpatialReference ref(meta["writers.tiledb"]["spatialreference"]);
setSpatialReference(ref);
}

// initialize read buffer variables
m_offset = 0;
m_resultSize = 0;
Expand Down
67 changes: 34 additions & 33 deletions plugins/tiledb/io/TileDBWriter.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -445,7 +445,6 @@ bool TileDBWriter::processOne(PointRef& point)
m_coords.push_back(x);
m_coords.push_back(y);
m_coords.push_back(z);
m_bbox.grow(x, y, z);

if (++m_current_idx == m_args->m_cache_size)
{
Expand Down Expand Up @@ -474,45 +473,47 @@ void TileDBWriter::done(PointTableRef table)
{
if (flushCache(m_current_idx))
{
tiledb::VFS vfs(*m_ctx, m_ctx->config());

// write pipeline metadata sidecar inside array
MetadataNode anon;
MetadataNode meta("pipeline");
anon.addList(meta);
// set output to tiledb reader
meta.add("type", "readers.tiledb");
if (!getSpatialReference().empty() && table.spatialReferenceUnique())
if (!m_args->m_append)
{
// The point view takes on the spatial reference of that stage,
// if it had one.
anon.add("spatialreference",
Utils::toString(getSpatialReference()));
}
anon.add("bounds", pdal::Utils::toString(m_bbox));
// write pipeline metadata sidecar inside array
MetadataNode node = getMetadata();
if (!getSpatialReference().empty() && table.spatialReferenceUnique())
{
// The point view takes on the spatial reference of that stage,
// if it had one.
node.add("spatialreference",
Utils::toString(getSpatialReference()));
}

// serialize metadata
tiledb::VFS::filebuf fbuf(vfs);
// serialize metadata
#if TILEDB_VERSION_MAJOR == 1 && TILEDB_VERSION_MINOR < 7
tiledb::VFS vfs(*m_ctx, m_ctx->config());
tiledb::VFS::filebuf fbuf(vfs);

if (vfs.is_dir(m_args->m_arrayName))
fbuf.open(m_args->m_arrayName + pathSeparator + "pdal.json",
std::ios::out);
else
{
std::string fname = m_args->m_arrayName + "/pdal.json";
vfs.touch(fname);
fbuf.open(fname, std::ios::out);
}
if (vfs.is_dir(m_args->m_arrayName))
fbuf.open(m_args->m_arrayName + pathSeparator + "pdal.json",
std::ios::out);
else
{
std::string fname = m_args->m_arrayName + "/pdal.json";
vfs.touch(fname);
fbuf.open(fname, std::ios::out);
}

std::ostream os(&fbuf);
std::ostream os(&fbuf);

if (!os.good())
throwError("Unable to create sidecar file for " +
m_args->m_arrayName);
if (!os.good())
throwError("Unable to create sidecar file for " +
m_args->m_arrayName);

pdal::Utils::toJSON(anon, os);
pdal::Utils::toJSON(node, os);

fbuf.close();
fbuf.close();
#else
std::string m = pdal::Utils::toJSON(node);
m_array->put_metadata("_pdal", TILEDB_UINT8, m.length() + 1, m.c_str());
#endif
}
m_array->close();
}
else{
Expand Down
1 change: 0 additions & 1 deletion plugins/tiledb/io/TileDBWriter.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -75,7 +75,6 @@ class PDAL_DLL TileDBWriter : public Writer, public Streamable
struct Args;
std::unique_ptr<TileDBWriter::Args> m_args;

BOX3D m_bbox;
size_t m_current_idx;

std::unique_ptr<tiledb::Context> m_ctx;
Expand Down
47 changes: 46 additions & 1 deletion plugins/tiledb/test/TileDBReaderTest.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,15 +40,19 @@
#include <pdal/PointView.hpp>
#include <pdal/PipelineManager.hpp>
#include <pdal/StageFactory.hpp>
#include <io/FauxReader.hpp>

#include "Support.hpp"

#include "../io/TileDBReader.hpp"
#include "../io/TileDBWriter.hpp"


namespace pdal
{

const SpatialReference utm16("EPSG:26916");

class TileDBReaderTest : public ::testing::Test
{
protected:
Expand Down Expand Up @@ -143,7 +147,6 @@ class TileDBReaderTest : public ::testing::Test
Options options;
options.add("array_name", pth);


tiledb::Array array(ctx, pth, TILEDB_READ);
auto domain = array.non_empty_domain<double>();
std::vector<double> subarray;
Expand Down Expand Up @@ -172,6 +175,48 @@ class TileDBReaderTest : public ::testing::Test
c.setInput(reader);
c.prepare(table);
c.execute(table);
// test using a sidecar file
EXPECT_EQ(reader.getSpatialReference(), utm16);
}

#if TILEDB_VERSION_MAJOR >= 1 && TILEDB_VERSION_MINOR >= 7
TEST_F(TileDBReaderTest, spatial_reference)
{
tiledb::Context ctx;
tiledb::VFS vfs(ctx);
std::string pth = Support::temppath("tiledb_test_srs");

Options options;
options.add("array_name", pth);
options.add("chunk_size", 80);

if (vfs.is_dir(pth))
{
vfs.remove_dir(pth);
}

FauxReader reader;
Options reader_options;
reader_options.add("mode", "ramp");
reader_options.add("count", 50);
reader.addOptions(reader_options);

TileDBWriter writer;
writer.setOptions(options);
writer.setInput(reader);
writer.setSpatialReference(utm16);

FixedPointTable table(100);
writer.prepare(table);
writer.execute(table);

TileDBReader rdr;
rdr.setOptions(options);
FixedPointTable table2(100);
rdr.prepare(table2);
rdr.execute(table2);
EXPECT_EQ(rdr.getSpatialReference(), utm16);
}
#endif
}

27 changes: 23 additions & 4 deletions plugins/tiledb/test/TileDBWriterTest.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -103,10 +103,20 @@ namespace pdal
writer.prepare(table);
writer.execute(table);

tiledb::Array array(ctx, pth, TILEDB_READ);

#if TILEDB_VERSION_MAJOR == 1 && TILEDB_VERSION_MINOR < 7
// check the sidecar exists
EXPECT_TRUE(pdal::Utils::fileExists(sidecar));
#else
tiledb_datatype_t v_type = TILEDB_UINT8;
const void* v_r;
uint32_t v_num;
array.get_metadata("_pdal", &v_type, &v_num, &v_r);
NL::json meta = NL::json::parse(static_cast<const char*>(v_r));
EXPECT_TRUE(meta.count("writers.tiledb") > 0);
#endif

tiledb::Array array(ctx, pth, TILEDB_READ);
auto domain = array.non_empty_domain<double>();
std::vector<double> subarray;

Expand Down Expand Up @@ -159,9 +169,6 @@ namespace pdal
writer.prepare(table);
writer.execute(table);

// check the sidecar exists so that the execute has completed
EXPECT_TRUE(pdal::Utils::fileExists(sidecar));

options.add("append", true);
TileDBWriter append_writer;
append_writer.setOptions(options);
Expand All @@ -175,6 +182,18 @@ namespace pdal
auto domain = array.non_empty_domain<double>();
std::vector<double> subarray;

#if TILEDB_VERSION_MAJOR == 1 && TILEDB_VERSION_MINOR < 7
// check the sidecar exists so that the execute has completed
EXPECT_TRUE(pdal::Utils::fileExists(sidecar));
#else
tiledb_datatype_t v_type = TILEDB_UINT8;
const void* v_r;
uint32_t v_num;
array.get_metadata("_pdal", &v_type, &v_num, &v_r);
NL::json meta = NL::json::parse(static_cast<const char*>(v_r));
EXPECT_TRUE(meta.count("writers.tiledb") > 0);
#endif

for (const auto& kv: domain)
{
subarray.push_back(kv.second.first);
Expand Down
5 changes: 4 additions & 1 deletion test/data/tiledb/array/pdal.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,5 +5,8 @@
{
"type": "readers.tiledb"
}
]
],
"writers.tiledb": {
"spatialreference": "EPSG:26916"
}
}

0 comments on commit 5be7c48

Please sign in to comment.