Skip to content

Commit

Permalink
Working-ish add-on writer.
Browse files Browse the repository at this point in the history
  • Loading branch information
@abellgithub
abellgithub committed May 11, 2020
1 parent c3c61ac commit 962bef3
Show file tree
Hide file tree
Showing 16 changed files with 257 additions and 140 deletions.
147 changes: 82 additions & 65 deletions io/EptAddonWriter.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,11 +36,14 @@

#include <cassert>

#include <arbiter/arbiter.hpp>
#include <nlohmann/json.hpp>

#include <pdal/Metadata.hpp>

#include "private/ept/Addon.hpp"
#include "private/ept/Connector.hpp"
#include "private/ept/Key.hpp"
#include "private/ept/Overlap.hpp"
#include "private/ept/Pool.hpp"
#include "private/ept/EptInfo.hpp"

Expand Down Expand Up @@ -91,8 +94,7 @@ void EptAddonWriter::addDimensions(PointLayoutPtr layout)

void EptAddonWriter::prepared(PointTableRef table)
{
m_arbiter.reset(new arbiter::Arbiter());

std::cerr << "Prepared!\n";
const std::size_t threads(std::max<std::size_t>(m_args->m_numThreads, 4));
if (threads > 100)
{
Expand All @@ -101,14 +103,20 @@ void EptAddonWriter::prepared(PointTableRef table)
}
m_pool.reset(new Pool(threads));

const PointLayout& layout(*table.layout());
m_addons = Addon::load(*m_connector, m_args->m_addons, false);
//ABELL
// Perhaps we need to wait to load addons until we have the
// headers/query from metadata.
m_connector.reset(new Connector());
std::cerr << "About to store!\n";
m_addons = Addon::store(*m_connector, m_args->m_addons, *(table.layout()));
std::cerr << "Done addon store!\n";
}

void EptAddonWriter::ready(PointTableRef table)
{
MetadataNode meta(table.privateMetadata("ept"));

std::cerr << "ready!\n";
if (meta.findChild("info").value().empty())
{
throwError(
Expand All @@ -118,12 +126,24 @@ void EptAddonWriter::ready(PointTableRef table)

try
{
m_info.reset(new EptInfo(meta.findChild("info").value<std::string>()));
const auto keys(parse(meta.findChild("keys").value<std::string>()));
MetadataNode sNode = meta.findChild("step");
m_hierarchyStep = sNode.value<uint64_t>();

m_hierarchyStep = meta.findChild("step").value<uint64_t>();
MetadataNode iNode = meta.findChild("info");
m_info.reset(new EptInfo(iNode.value<std::string>()));

MetadataNode kNode = meta.findChild("keys");
const auto keys(NL::json::parse(kNode.value<std::string>()));
for (auto el : keys.items())
m_hierarchy[Key(el.key())] = el.value().get<uint64_t>();
{
const std::string& key = el.key();
uint64_t count = el.value()[0].get<uint64_t>();
uint64_t nodeId = el.value()[1].get<uint64_t>();

using Entry = std::pair<const std::string, Overlap>;
Entry e {key, {key, count, nodeId}};
m_overlaps.insert(e);
}
}
catch (std::exception& e)
{
Expand All @@ -133,33 +153,32 @@ void EptAddonWriter::ready(PointTableRef table)

void EptAddonWriter::write(const PointViewPtr view)
{
std::cerr << "Write!\n";
for (const auto& addon : m_addons)
{
//ABELL
/**
log()->get(LogLevel::Debug) << "Writing addon dimension " <<
addon->name() << " to " << addon->ep().prefixedRoot() << std::endl;
**/
addon.name() << " to " << addon.filename() << std::endl;

writeOne(view, *addon);
writeOne(view, addon);

log()->get(LogLevel::Debug) << "\tWritten" << std::endl;
}
}

void EptAddonWriter::writeOne(const PointViewPtr view, const Addon& addon) const
{
std::vector<std::vector<char>> buffers;
buffers.reserve(m_hierarchy.size());
std::cerr << "Write one: " << addon.name() << "!\n";
std::vector<std::vector<char>> buffers(m_overlaps.size());

// Create an addon buffer for each node we're going to write.
//ABELL
/**
for (const auto& p : m_hierarchy)

size_t itemSize = Dimension::size(addon.type());
for (const auto& entry : m_overlaps)
{
buffers.emplace_back(p.second * addon.size(), 0);
const Overlap& overlap = entry.second;
std::vector<char>& b = buffers[overlap.m_nodeId - 1];
b.resize(overlap.m_count * itemSize);
}
**/

// Fill in our buffers with the data from the view.
PointRef pr(*view);
Expand All @@ -172,99 +191,97 @@ void EptAddonWriter::writeOne(const PointViewPtr view, const Addon& addon) const

// Node IDs are 1-based to distinguish points that do not come from the
// EPT reader.
if (!nodeId) continue;
if (!nodeId)
continue;

nodeId -= 1;
pointId = pr.getFieldAs<uint64_t>(m_pointIdDim);

/**
auto& buffer(buffers.at(nodeId));
assert(pointId * addon.size() + addon.size() <= buffer.size());
char* dst = buffer.data() + pointId * addon.size();
pr.getField(dst, addon.id(), addon.type());
**/
auto& buffer(buffers.at(nodeId - 1));
assert(pointId * itemSize + itemSize <= buffer.size());
char* dst = buffer.data() + pointId * itemSize;
pr.getField(dst, addon.dstId(), addon.type());
}

/**
const arbiter::Endpoint& ep(addon.ep());
std::string dataDir = addon.dataDir();

const arbiter::Endpoint dataEp(ep.getSubEndpoint("ept-data"));
const arbiter::Endpoint hierEp(ep.getSubEndpoint("ept-hierarchy"));
if (ep.isLocal())
{
arbiter::mkdirp(dataEp.root());
arbiter::mkdirp(hierEp.root());
}
m_connector->makeDir(dataDir);

// Write the binary dimension data for the addon.
nodeId = 0;
for (const auto& p : m_hierarchy)
for (const auto& p : m_overlaps)
{
const Key key(p.first);
const Overlap& overlap = p.second;

m_pool->add([&dataEp, &buffers, key, nodeId]()
std::vector<char>& buffer = buffers.at(overlap.m_nodeId - 1);
std::string filename = dataDir + overlap.m_key.toString() + ".bin";
std::cerr << "About to write to " << filename << "!\n";
m_pool->add([this, filename, &buffer]()
{
dataEp.put(key.toString() + ".bin", buffers.at(nodeId));
m_connector->put(filename, buffer);
});
++nodeId;
}

m_pool->await();
**/

// Write the addon hierarchy data.
/**
NL::json h;
Key key;
key.b = m_info->bounds();
writeHierarchy(h, key, hierEp);
hierEp.put(key.toString() + ".json", h.dump());

std::string hierarchyDir = addon.hierarchyDir();
m_connector->makeDir(hierarchyDir);

writeHierarchy(hierarchyDir, h, key);
std::string filename = hierarchyDir + key.toString() + ".json";
m_connector->put(filename, h.dump());

m_pool->await();

// Write the top-level addon metadata.
NL::json meta;
meta["type"] = getTypeString(addon.type());
meta["size"] = addon.size();
meta["size"] = Dimension::size(addon.type());
meta["version"] = "1.0.0";
meta["dataType"] = "binary";

ep.put("ept-addon.json", meta.dump());
**/
m_connector->put("ept-addon.json", meta.dump());
}

void EptAddonWriter::writeHierarchy(NL::json& curr, const Key& key,
const arbiter::Endpoint& hierEp) const
void EptAddonWriter::writeHierarchy(const std::string& directory,
NL::json& curr, const Key& key) const
{
const std::string keyName(key.toString());
if (!m_hierarchy.count(keyName)) return;
std::string keyName = key.toString();

auto it = m_overlaps.find(keyName);
if (it == m_overlaps.end())
return;

const uint64_t np(m_hierarchy.at(keyName));
if (!np) return;
const Overlap& overlap = it->second;;
if (!overlap.m_count)
return;

if (m_hierarchyStep && key.d && (key.d % m_hierarchyStep == 0))
{
curr[keyName] = -1;

// Create a new hierarchy subtree.
NL::json next {{ keyName, np }};
NL::json next {{ keyName, overlap.m_count }};

for (uint64_t dir(0); dir < 8; ++dir)
writeHierarchy(next, key.bisect(dir), hierEp);
writeHierarchy(directory, next, key.bisect(dir));

m_pool->add([&hierEp, keyName, next]()
std::string filename = directory + keyName + ".json";
std::string data = next.dump();
m_pool->add([this, filename, data]()
{
hierEp.put(keyName + ".json", next.dump());
m_connector->put(filename, data);
});
}
else
{
curr[keyName] = np;
curr[keyName] = overlap.m_count;
for (uint64_t dir(0); dir < 8; ++dir)
{
writeHierarchy(curr, key.bisect(dir), hierEp);
writeHierarchy(directory, curr, key.bisect(dir));
}
}
}
Expand Down
21 changes: 9 additions & 12 deletions io/EptAddonWriter.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,20 +37,17 @@
#include <cstddef>
#include <memory>
#include <vector>
#include <unordered_map>

#include <pdal/JsonFwd.hpp>
#include <pdal/Writer.hpp>

namespace pdal
{
#include "private/ept/Addon.hpp"

namespace arbiter
namespace pdal
{
class Arbiter;
class Endpoint;
}

class Addon;
class Connector;
class EptInfo;
class Key;
class Pool;
Expand All @@ -74,18 +71,18 @@ class PDAL_DLL EptAddonWriter : public Writer
virtual void write(const PointViewPtr view) override;

void writeOne(const PointViewPtr view, const Addon& addon) const;
void writeHierarchy(NL::json& hier, const Key& key,
const arbiter::Endpoint& hierEp) const;
void writeHierarchy(const std::string& hierarchyDir, NL::json& hier,
const Key& key) const;
std::string getTypeString(Dimension::Type t) const;

Dimension::Id m_nodeIdDim = Dimension::Id::Unknown;
Dimension::Id m_pointIdDim = Dimension::Id::Unknown;

std::unique_ptr<arbiter::Arbiter> m_arbiter;
std::unique_ptr<Connector> m_connector;
std::unique_ptr<Pool> m_pool;
std::unique_ptr<EptInfo> m_info;
std::vector<std::unique_ptr<Addon>> m_addons;
std::map<Key, uint64_t> m_hierarchy;
AddonList m_addons;
std::unordered_map<std::string, Overlap> m_overlaps;
uint64_t m_hierarchyStep = 0;
};

Expand Down

0 comments on commit 962bef3

Please sign in to comment.