Skip to content

Commit

Permalink
refactor for buffer mgmt, serialization
Browse files Browse the repository at this point in the history
  • Loading branch information
@mpgerlek
mpgerlek committed May 7, 2015
1 parent 350a713 commit a360830
Show file tree
Hide file tree
Showing 7 changed files with 144 additions and 111 deletions.
12 changes: 6 additions & 6 deletions plugins/rialto/io/RialtoDb.cpp
View file
Expand Up @@ -346,15 +346,13 @@ RialtoDb::TileInfo RialtoDb::getTileInfo(uint32_t tileId, bool withPoints)
info.x = boost::lexical_cast<double>(r->at(2).data);
info.y = boost::lexical_cast<double>(r->at(3).data);

info.patch.buf.clear();
if (withPoints)
{
info.patch = new Patch(); // TODO: delete
const uint32_t blobLen = r->at(4).blobLen;
const std::vector<uint8_t>& blobBuf = r->at(4).blobBuf;
const unsigned char *pos = (const unsigned char *)&(blobBuf[0]);
info.patch->putBytes(pos, blobLen);
} else {
info.patch = NULL;
info.patch.putBytes(pos, blobLen);
}

assert(!m_session->next());
Expand Down Expand Up @@ -485,8 +483,10 @@ void RialtoDb::addDimensions(uint32_t tileSetId,
}


uint32_t RialtoDb::addTile(const RialtoDb::TileInfo& data, char* buf, uint32_t buflen)
uint32_t RialtoDb::addTile(const RialtoDb::TileInfo& data)
{
unsigned char* buf = (unsigned char*)&data.patch.buf[0]; // TODO
uint32_t buflen = data.patch.buf.size();
assert(buf);
assert(buflen);

Expand All @@ -502,7 +502,7 @@ uint32_t RialtoDb::addTile(const RialtoDb::TileInfo& data, char* buf, uint32_t b
r.push_back(column(data.level));
r.push_back(column(data.x));
r.push_back(column(data.y));
r.push_back(blob(buf, buflen));
r.push_back(blob((char*)buf, (size_t)buflen));
rs.push_back(r);

m_session->insert(oss.str(), rs);
Expand Down
8 changes: 4 additions & 4 deletions plugins/rialto/io/RialtoDb.hpp
View file
Expand Up @@ -40,11 +40,12 @@
#include <cstdint>
#include <string>

// TODO: just used for Patch? (SQLite can be fwd declared)
#include <../plugins/sqlite/io/SQLiteCommon.hpp> // TODO: fix path

namespace pdal {
class Log;
class SQLite;
class Patch;
}

namespace rialtosupport
Expand Down Expand Up @@ -92,15 +93,14 @@ class PDAL_DLL RialtoDb
double maxx;
double maxy;
uint32_t numDimensions;
std::vector<DimensionInfo> dimensions;
};

struct TileInfo {
uint32_t tileSetId;
uint32_t level;
uint32_t x; // col
uint32_t y; // row
Patch* patch;
Patch patch;
};

RialtoDb(const std::string& connection);
Expand All @@ -118,7 +118,7 @@ class PDAL_DLL RialtoDb
uint32_t addTileSet(const RialtoDb::TileSetInfo& data);

// returns id of new tile
uint32_t addTile(const RialtoDb::TileInfo& data, char* buf, uint32_t buflen);
uint32_t addTile(const RialtoDb::TileInfo& data);

// add all the dimensions of the tile set
void addDimensions(uint32_t tileSetId,
Expand Down
163 changes: 98 additions & 65 deletions plugins/rialto/io/RialtoDbWriter.cpp
View file
Expand Up @@ -61,91 +61,124 @@ CREATE_SHARED_PLUGIN(1, 0, RialtoDbWriter, Writer, s_info)

namespace
{
} // anonymous namespace


void RialtoDbWriter::writeHeader(MetadataNode tileSetNode,
PointLayoutPtr layout)
{
log()->get(LogLevel::Debug) << "RialtoDbWriter::writeHeader()" << std::endl;

RialtoDb::TileSetInfo data;
data.name = "a.las";

MetadataNode headerNode = tileSetNode.findChild("header");
assert(headerNode.valid());
data.maxLevel = getMetadataU32(headerNode, "maxLevel");
static void serializeToTileSetInfo(MetadataNode tileSetNode,
PointLayoutPtr layout,
RialtoDb::TileSetInfo& tileSetInfo)
{
tileSetInfo.name = "a.las"; // TODO

MetadataNode headerNode = tileSetNode.findChild("header");
assert(headerNode.valid());
tileSetInfo.maxLevel = RialtoDbWriter::getMetadataU32(headerNode, "maxLevel");

tileSetInfo.numCols = RialtoDbWriter::getMetadataU32(headerNode, "numCols");
tileSetInfo.numRows = RialtoDbWriter::getMetadataU32(headerNode, "numRows");
assert(tileSetInfo.numCols == 2 && tileSetInfo.numRows == 1);

tileSetInfo.minx = RialtoDbWriter::getMetadataF64(headerNode, "minX");
tileSetInfo.miny = RialtoDbWriter::getMetadataF64(headerNode, "minY");
tileSetInfo.maxx = RialtoDbWriter::getMetadataF64(headerNode, "maxX");
tileSetInfo.maxy = RialtoDbWriter::getMetadataF64(headerNode, "maxY");
assert(tileSetInfo.minx==-180.0 && tileSetInfo.miny==-90.0 && tileSetInfo.maxx==180.0 && tileSetInfo.maxy==90.0);

tileSetInfo.numDimensions = layout->dims().size();
}

data.numCols = getMetadataU32(headerNode, "numCols");
data.numRows = getMetadataU32(headerNode, "numRows");
assert(data.numCols == 2 && data.numRows == 1);
static void serializeToDimensionInfo(MetadataNode tileSetNode,
PointLayoutPtr layout,
std::vector<RialtoDb::DimensionInfo>& infoList)
{
const uint32_t numDims = layout->dims().size();

infoList.clear();
infoList.resize(numDims);

//log()->get(LogLevel::Debug1) << "num dims: " << infoList.size() << std::endl;

size_t i = 0;
for (const auto& dim : layout->dims())
{
const std::string name = Dimension::name(dim);
const Dimension::Type::Enum dataType = layout->dimType(dim);

double minimum, mean, maximum;
RialtoDbWriter::extractStatistics(tileSetNode, name, minimum, mean, maximum);

RialtoDb::DimensionInfo& info = infoList[i];
info.name = name;
info.position = i;
info.dataType = (RialtoDb::DataType)(uint32_t)dataType; // TODO: enum-to-enum
info.minimum = minimum;
info.mean = mean;
info.maximum = maximum;

++i;
}
}

data.minx = getMetadataF64(headerNode, "minX");
data.miny = getMetadataF64(headerNode, "minY");
data.maxx = getMetadataF64(headerNode, "maxX");
data.maxy = getMetadataF64(headerNode, "maxY");
assert(data.minx==-180.0 && data.miny==-90.0 && data.maxx==180.0 && data.maxy==90.0);

data.numDimensions = layout->dims().size();
static void serializeToPatch(PointView* view, Patch& patch)
{
if (!view)
{
patch.buf.clear();
} else {
size_t len = 0;
unsigned char* buf = RialtoWriter::createBlob(view, len); // TODO
patch.putBytes(buf, len);
delete[] buf; // TODO
}
}

const uint32_t id = m_rialtoDb->addTileSet(data);

data.dimensions.clear();
data.dimensions.resize(data.numDimensions);
static void serializeToTile(MetadataNode tileNode, PointView* view, RialtoDb::TileInfo& tileInfo)
{
tileInfo.tileSetId = 0; // not used in writing

tileInfo.level = RialtoWriter::getMetadataU32(tileNode, "level");
tileInfo.x = RialtoWriter::getMetadataU32(tileNode, "tileX");
tileInfo.y = RialtoWriter::getMetadataU32(tileNode, "tileY");
//const uint32_t mask = getMetadataU32(tileNode, "mask");

//log()->get(LogLevel::Debug) << "RialtoDbWriter::writeTile for "
// << tileInfo.level << "," << info.x << "," << info.y << " "
// << (view==0 ? "no" : "yes")
// << std::endl;

Patch& patch = tileInfo.patch;
serializeToPatch(view, patch);
}
} // anonymous namespace

log()->get(LogLevel::Debug1) << "num dims: " << data.dimensions.size() << std::endl;

size_t i = 0;
for (const auto& dim : layout->dims())
{
const std::string name = Dimension::name(dim);
const Dimension::Type::Enum dataType = layout->dimType(dim);

double minimum, mean, maximum;
extractStatistics(tileSetNode, name, minimum, mean, maximum);
void RialtoDbWriter::writeHeader(MetadataNode tileSetNode,
PointLayoutPtr layout)
{
log()->get(LogLevel::Debug) << "RialtoDbWriter::writeHeader()" << std::endl;

RialtoDb::DimensionInfo& dimData = data.dimensions[i];
dimData.name = name;
dimData.position = i;
dimData.dataType = (RialtoDb::DataType)(uint32_t)dataType; // TODO: enum-to-enum
dimData.minimum = minimum;
dimData.mean = mean;
dimData.maximum = maximum;
RialtoDb::TileSetInfo tileSetInfo;
serializeToTileSetInfo(tileSetNode, layout, tileSetInfo);

++i;
}
std::vector<RialtoDb::DimensionInfo> dimsInfo;
serializeToDimensionInfo(tileSetNode, layout, dimsInfo);

const uint32_t id = m_rialtoDb->addTileSet(tileSetInfo);

m_rialtoDb->addDimensions(id, data.dimensions);
m_rialtoDb->addDimensions(id, dimsInfo);
}


void RialtoDbWriter::writeTile(MetadataNode tileNode, PointView* view)
{
log()->get(LogLevel::Debug1) << "RialtoDbWriter::writeTile()" << std::endl;

RialtoDb::TileInfo data;
data.tileSetId = 0;
data.level = getMetadataU32(tileNode, "level");
data.x = getMetadataU32(tileNode, "tileX");
data.y = getMetadataU32(tileNode, "tileY");
const uint32_t mask = getMetadataU32(tileNode, "mask");

log()->get(LogLevel::Debug) << "RialtoDbWriter::writeTile for "
<< data.level << "," << data.x << "," << data.y << " "
<< (view==0 ? "no" : "yes")
<< std::endl;

char* buf = NULL;
size_t bufsiz = 0;
if (view)
{
buf = createBlob(view, bufsiz);
}
RialtoDb::TileInfo tileInfo;
serializeToTile(tileNode, view, tileInfo);

if (buf)
if (tileInfo.patch.buf.size())
{
uint32_t id = m_rialtoDb->addTile(data, buf, (uint32_t)bufsiz);
delete[] buf; // TODO
uint32_t id = m_rialtoDb->addTile(tileInfo);
}
}

Expand Down
2 changes: 1 addition & 1 deletion plugins/rialto/io/RialtoFileWriter.cpp
View file
Expand Up @@ -146,7 +146,7 @@ void RialtoFileWriter::writeTile(MetadataNode tileNode, PointView* view)
if (view)
{
size_t bufsiz;
char* buf = createBlob(view, bufsiz);
unsigned char* buf = createBlob(view, bufsiz);
fwrite(buf, bufsiz, 1, fp);
delete[] buf; // TODO
}
Expand Down
12 changes: 6 additions & 6 deletions plugins/rialto/io/RialtoWriter.cpp
View file
Expand Up @@ -51,10 +51,10 @@ namespace pdal

namespace
{
static void fillBufferWithPoint(const PointView* view, const PointId& idx, char* buf)
static void fillBufferWithPoint(const PointView* view, const PointId& idx, unsigned char* buf)
{
char* p = buf;
char* q = p;
unsigned char* p = buf;
unsigned char* q = p;

for (const auto& dim : view->dims())
{
Expand All @@ -63,7 +63,7 @@ namespace
}
}

static void fillBufferWithPointView(const PointView* view, char* buf)
static void fillBufferWithPointView(const PointView* view, unsigned char* buf)
{
for (size_t i=0; i<view->size(); ++i)
{
Expand All @@ -77,12 +77,12 @@ namespace


// caller responisble for deleting the buffer
char* RialtoWriter::createBlob(PointView* view, size_t& buflen)
unsigned char* RialtoWriter::createBlob(PointView* view, size_t& buflen)
{
const uint32_t pointSize = view->pointSize();
const uint32_t numPoints = view->size();
buflen = pointSize * numPoints;
char* buf = new char[buflen];
unsigned char* buf = new unsigned char[buflen];

fillBufferWithPointView(view, buf);

Expand Down
14 changes: 7 additions & 7 deletions plugins/rialto/io/RialtoWriter.hpp
View file
Expand Up @@ -59,6 +59,13 @@ class PDAL_DLL RialtoWriter : public Writer
virtual std::string getName() const=0;
virtual Options getDefaultOptions()=0;

// helper functions
static uint32_t getMetadataU32(const MetadataNode& parent, const std::string& name);
static double getMetadataF64(const MetadataNode& parent, const std::string& name);
static void extractStatistics(MetadataNode& tileSetNode, const std::string& dimName,
double& minimum, double& mean, double& maximum);
static unsigned char* createBlob(PointView* view, size_t& buflen);

protected:

// implemented in derived classes
Expand All @@ -71,13 +78,6 @@ class PDAL_DLL RialtoWriter : public Writer
virtual void writeTile(MetadataNode, PointView*) = 0;
virtual void localFinish() = 0;

// helper functions
static uint32_t getMetadataU32(const MetadataNode& parent, const std::string& name);
static double getMetadataF64(const MetadataNode& parent, const std::string& name);
static void extractStatistics(MetadataNode& tileSetNode, const std::string& dimName,
double& minimum, double& mean, double& maximum);
static char* createBlob(PointView* view, size_t& buflen);

private:
virtual void ready(PointTableRef table);
virtual void write(const PointViewPtr view);
Expand Down

0 comments on commit a360830

Please sign in to comment.