From 350a71328c4876d022eeff099ab1ddfc69c9339e Mon Sep 17 00:00:00 2001 From: "Michael P. Gerlek" Date: Thu, 7 May 2015 08:08:50 -0400 Subject: [PATCH] form last night --- plugins/rialto/io/RialtoDb.cpp | 209 ++++++++++----------- plugins/rialto/io/RialtoDb.hpp | 28 ++- plugins/rialto/io/RialtoDbWriter.cpp | 2 +- plugins/rialto/test/RialtoDbWriterTest.cpp | 192 +++++++------------ 4 files changed, 188 insertions(+), 243 deletions(-) diff --git a/plugins/rialto/io/RialtoDb.cpp b/plugins/rialto/io/RialtoDb.cpp index 25d9605af9..443b4da5ca 100644 --- a/plugins/rialto/io/RialtoDb.cpp +++ b/plugins/rialto/io/RialtoDb.cpp @@ -79,7 +79,7 @@ // minimum // mean // maximum - + namespace rialtosupport { @@ -92,6 +92,7 @@ namespace RialtoDb::RialtoDb(const std::string& connection) : m_connection(connection), + m_session(NULL), m_srid(4326) { } @@ -103,6 +104,39 @@ RialtoDb::~RialtoDb() } +void RialtoDb::create() +{ + m_log = std::shared_ptr(new pdal::Log("RialtoDB", "stdout")); + m_log->setLevel(LogLevel::Debug); + + log()->get(LogLevel::Debug) << "RialtoDB::open()" << std::endl; + + if (FileUtils::fileExists(m_connection)) + { + throw pdal_error("RialtoDb: database already exists"); + } + + m_session = new SQLite(m_connection, m_log); + m_session->connect(true); + +#if 0 + m_session->spatialite(); + + const bool hasSpatialite = m_session->doesTableExist("geometry_columns"); + if (!hasSpatialite) + { + std::ostringstream oss; + oss << "SELECT InitSpatialMetadata()"; + m_session->execute(oss.str()); + } +#endif + + createTileSetsTable(); + createTilesTable(); + createDimensionsTable(); +} + + void RialtoDb::open(bool writable) { m_log = std::shared_ptr(new pdal::Log("RialtoDB", "stdout")); @@ -110,7 +144,12 @@ void RialtoDb::open(bool writable) log()->get(LogLevel::Debug) << "RialtoDB::open()" << std::endl; - m_session = std::unique_ptr(new SQLite(m_connection, m_log)); + if (!FileUtils::fileExists(m_connection)) + { + throw pdal_error("RialtoDb: database not found"); + } + + m_session = new SQLite(m_connection, m_log); m_session->connect(writable); #if 0 @@ -125,24 +164,19 @@ void RialtoDb::open(bool writable) } #endif - if (writable) { - createTileSetsTable(); - createTilesTable(); - createDimensionsTable(); - } else { - if (!m_session->doesTableExist("TileSets")) - throw pdal_error("RialtoDb: required table 'TileSets' not found"); - if (!m_session->doesTableExist("Tiles")) - throw pdal_error("RialtoDb: required table 'Tiles' not found"); - if (!m_session->doesTableExist("Dimensions")) - throw pdal_error("RialtoDb: required table 'Dimensions' not found"); - } + if (!m_session->doesTableExist("TileSets")) + throw pdal_error("RialtoDb: required table 'TileSets' not found"); + if (!m_session->doesTableExist("Tiles")) + throw pdal_error("RialtoDb: required table 'Tiles' not found"); + if (!m_session->doesTableExist("Dimensions")) + throw pdal_error("RialtoDb: required table 'Dimensions' not found"); } void RialtoDb::close() { - // TODO + delete m_session; + m_session = NULL; // TODO: assert not null on all public function } @@ -156,11 +190,13 @@ void RialtoDb::createTileSetsTable() oss1 << "CREATE TABLE TileSets(" << "tile_set_id INTEGER PRIMARY KEY AUTOINCREMENT," << "name VARCHAR(64)," // TODO + << "maxLevel INTEGER," + << "numCols INTEGER," + << "numRows INTEGER," << "minx DOUBLE," << "miny DOUBLE," << "maxx DOUBLE," << "maxy DOUBLE," - << "maxLevel INTEGER," << "numDims INTEGER" << ")"; @@ -212,7 +248,7 @@ void RialtoDb::createDimensionsTable() oss1 << "CREATE TABLE Dimensions(" << "tile_set_id INTEGER," - << "name VARCHAR(64)," // TODO + << "name VARCHAR(256)," // TODO << "position INTEGER," << "dataType INTEGER," << "minimum DOUBLE," @@ -224,36 +260,29 @@ void RialtoDb::createDimensionsTable() m_session->execute(oss1.str()); } + std::vector RialtoDb::getTileSetIds() { std::vector ids; - + std::ostringstream oss; oss << "SELECT tile_set_id FROM TileSets"; - + log()->get(LogLevel::Debug1) << "SELECT for tile set ids" << std::endl; m_session->query(oss.str()); - + do { const row* r = m_session->get(); if (!r) break; - int i=0; - for (auto c: *r) - { - log()->get(LogLevel::Debug1) << " col " << i << ": " << c.data << std::endl; - //column const& c = r->at(0); - ++i; - } - column const& c = r->at(0); const uint32_t id = boost::lexical_cast(c.data); log()->get(LogLevel::Debug1) << " got id: " << id << std::endl; ids.push_back(id); - + } while (m_session->next()); - + return ids; } @@ -263,7 +292,7 @@ RialtoDb::TileSetInfo RialtoDb::getTileSetInfo(uint32_t tileSetId) TileSetInfo info; std::ostringstream oss; - oss << "SELECT tile_set_id,name,minx,miny,maxx,maxy,maxLevel,numDims " + oss << "SELECT tile_set_id,name,maxLevel,numCols,numRows,minx,miny,maxx,maxy,numDims " << "FROM TileSets WHERE tile_set_id=" << tileSetId; log()->get(LogLevel::Debug) << "SELECT for tile set" << std::endl; @@ -276,30 +305,33 @@ RialtoDb::TileSetInfo RialtoDb::getTileSetInfo(uint32_t tileSetId) assert(tileSetId == boost::lexical_cast(r->at(0).data)); info.name = r->at(1).data; - info.minx = boost::lexical_cast(r->at(2).data); - info.miny = boost::lexical_cast(r->at(3).data); - info.maxx = boost::lexical_cast(r->at(4).data); - info.maxy = boost::lexical_cast(r->at(5).data); - info.maxLevel = boost::lexical_cast(r->at(6).data); - info.numDimensions = boost::lexical_cast(r->at(7).data); - - info.numCols = 2; - info.numRows = 1; - + info.maxLevel = boost::lexical_cast(r->at(2).data); + info.numCols = boost::lexical_cast(r->at(3).data); + info.numRows = boost::lexical_cast(r->at(4).data); + info.minx = boost::lexical_cast(r->at(5).data); + info.miny = boost::lexical_cast(r->at(6).data); + info.maxx = boost::lexical_cast(r->at(7).data); + info.maxy = boost::lexical_cast(r->at(8).data); + info.numDimensions = boost::lexical_cast(r->at(9).data); + + assert(info.numCols == 2); + assert(info.numRows == 1); + assert(!m_session->next()); return info; } -RialtoDb::TileInfo RialtoDb::getTileInfo(uint32_t tileSetId, uint32_t tileId) +RialtoDb::TileInfo RialtoDb::getTileInfo(uint32_t tileId, bool withPoints) { TileInfo info; - + std::ostringstream oss; - oss << "SELECT tile_id,tile_set_id,level,x,y " - << "FROM Tiles WHERE tile_set_id=" << tileSetId - << " AND tile_id=" << tileId; + oss << "SELECT tile_id,level,x,y" + << (withPoints ? ",points " : " ") + << "FROM Tiles " + << "WHERE tile_id=" << tileId; log()->get(LogLevel::Debug) << "SELECT for tile" << std::endl; @@ -310,11 +342,21 @@ RialtoDb::TileInfo RialtoDb::getTileInfo(uint32_t tileSetId, uint32_t tileId) assert(r); assert(tileId == boost::lexical_cast(r->at(0).data)); - assert(tileSetId == boost::lexical_cast(r->at(1).data)); - info.level = boost::lexical_cast(r->at(2).data); - info.x = boost::lexical_cast(r->at(3).data); - info.y = boost::lexical_cast(r->at(4).data); - + info.level = boost::lexical_cast(r->at(1).data); + info.x = boost::lexical_cast(r->at(2).data); + info.y = boost::lexical_cast(r->at(3).data); + + if (withPoints) + { + info.patch = new Patch(); // TODO: delete + const uint32_t blobLen = r->at(4).blobLen; + const std::vector& blobBuf = r->at(4).blobBuf; + const unsigned char *pos = (const unsigned char *)&(blobBuf[0]); + info.patch->putBytes(pos, blobLen); + } else { + info.patch = NULL; + } + assert(!m_session->next()); return info; @@ -347,49 +389,6 @@ std::vector RialtoDb::getTileIdsAtLevel(uint32_t tileSetId, uint32_t l } -void RialtoDb::getTileData(uint32_t tileId, char*& buf, uint32_t& bufLen) -{ - buf = NULL; - bufLen = 0; - - std::ostringstream oss; - oss << "SELECT points " - << "FROM Tiles " - << "WHERE tile_id=" << tileId; - - log()->get(LogLevel::Debug) << "SELECT for data for tile: " << tileId << std::endl; - - m_session->query(oss.str()); - - // should get exactly one row back - const row* r = m_session->get(); - assert(r); - - double* x = (double*)(&r->at(0).blobBuf[0]); - double* y = (double*)(&r->at(0).blobBuf[8]); - double* z = (double*)(&r->at(0).blobBuf[16]); - log()->get(LogLevel::Debug) << " x: " << *x << std::endl; - log()->get(LogLevel::Debug) << " y: " << *y << std::endl; - log()->get(LogLevel::Debug) << " z: " << *z << std::endl; - - char* p = (char*)(&r->at(0).blobBuf[0]); - - bufLen = (r->at(0)).blobLen; - buf = new char[bufLen]; - for (uint32_t i=0; iget(LogLevel::Debug) << " xx: " << *xx << std::endl; - log()->get(LogLevel::Debug) << " yy: " << *yy << std::endl; - log()->get(LogLevel::Debug) << " zz: " << *zz << std::endl; - assert(buf); - assert(bufLen); -} - - RialtoDb::DimensionInfo RialtoDb::getDimensionInfo(uint32_t tileSetId, uint32_t position) { DimensionInfo info; @@ -415,7 +414,7 @@ RialtoDb::DimensionInfo RialtoDb::getDimensionInfo(uint32_t tileSetId, uint32_t info.minimum = boost::lexical_cast(r->at(4).data); info.mean = boost::lexical_cast(r->at(5).data); info.maximum = boost::lexical_cast(r->at(6).data); - + assert(!m_session->next()); return info; @@ -424,32 +423,32 @@ RialtoDb::DimensionInfo RialtoDb::getDimensionInfo(uint32_t tileSetId, uint32_t uint32_t RialtoDb::addTileSet(const RialtoDb::TileSetInfo& data) { + log()->get(LogLevel::Debug) << "RialtoDb::addTileSet()" << std::endl; + std::ostringstream oss; oss << "INSERT INTO TileSets " - << "(name, minx, miny, maxx, maxy, maxLevel, numDims) " - << "VALUES (?, ?, ?, ?, ?, ?, ?)"; + << "(name, maxLevel, numCols, numRows, minx, miny, maxx, maxy, numDims) " + << "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)"; records rs; row r; r.push_back(column(data.name)); + r.push_back(column(data.maxLevel)); + r.push_back(column(data.numCols)); + r.push_back(column(data.numRows)); r.push_back(column(data.minx)); r.push_back(column(data.miny)); r.push_back(column(data.maxx)); r.push_back(column(data.maxy)); - r.push_back(column(data.maxLevel)); r.push_back(column(data.numDimensions)); rs.push_back(r); m_session->insert(oss.str(), rs); long id = m_session->last_row_id(); - log()->get(LogLevel::Debug1) << "inserted TileSet, id=" << id << std::endl; - - //log()->get(LogLevel::Debug) << "dimensions count: " - // << data.numDimensions << " " << data.dimensions.size() << std::endl; - //assert(data.numDimensions == data.dimensions.size()); - //addDimensions(id, data.dimensions); + log()->get(LogLevel::Debug) << "inserted TileSet, id=" << id << std::endl; + log()->get(LogLevel::Debug) << " " << data.numCols << data.numRows << std::endl; return id; } @@ -490,7 +489,7 @@ uint32_t RialtoDb::addTile(const RialtoDb::TileInfo& data, char* buf, uint32_t b { assert(buf); assert(buflen); - + std::ostringstream oss; oss << "INSERT INTO Tiles " << "(tile_set_id, level, x, y, points) " diff --git a/plugins/rialto/io/RialtoDb.hpp b/plugins/rialto/io/RialtoDb.hpp index d237d46da2..c6e920434e 100644 --- a/plugins/rialto/io/RialtoDb.hpp +++ b/plugins/rialto/io/RialtoDb.hpp @@ -42,9 +42,9 @@ namespace pdal { -class Log; -class SQLite; - + class Log; + class SQLite; + class Patch; } namespace rialtosupport @@ -57,6 +57,7 @@ using namespace pdal; class PDAL_DLL RialtoDb { public: + // values match PDAL's enum DataType { Invalid = 0, Uint8 = 0x200 | 1, @@ -99,12 +100,14 @@ class PDAL_DLL RialtoDb uint32_t level; uint32_t x; // col uint32_t y; // row + Patch* patch; }; RialtoDb(const std::string& connection); ~RialtoDb(); + void create(); void open(bool writable=false); void close(); @@ -114,12 +117,13 @@ class PDAL_DLL RialtoDb // returns id of new data set uint32_t addTileSet(const RialtoDb::TileSetInfo& data); + // returns id of new tile + uint32_t addTile(const RialtoDb::TileInfo& data, char* buf, uint32_t buflen); + + // add all the dimensions of the tile set void addDimensions(uint32_t tileSetId, const std::vector& dimensions); - // remove a tile set from the database - void deleteTileSet(uint32_t tileSetId); - // get list all the tile sets in the database, as a list of its std::vector getTileSetIds(); @@ -129,18 +133,12 @@ class PDAL_DLL RialtoDb // get info about one of the dimensions of a tile set DimensionInfo getDimensionInfo(uint32_t tileSetId, uint32_t dimension); - // returns id of new tile - uint32_t addTile(const RialtoDb::TileInfo& data, char* buf, uint32_t buflen); - // get info about a tile - TileInfo getTileInfo(uint32_t tileSetId, uint32_t tileId); + TileInfo getTileInfo(uint32_t tileId, bool withPoints); - // get the raw buffer with the points - void getTileData(uint32_t tileId, char*& buf, uint32_t& buflen); - // use with caution for levels greater than 16 or so std::vector getTileIdsAtLevel(uint32_t tileSetId, uint32_t level); - + // query for all the points of a tile set, bounded by bbox region // returns a pipeline made up of a BufferReader and a CropFilter Stage* query(uint32_t tileSetId, @@ -164,7 +162,7 @@ class PDAL_DLL RialtoDb LogPtr log() const { return m_log; } std::string m_connection; - std::unique_ptr m_session; + SQLite* m_session; LogPtr m_log; int m_srid; diff --git a/plugins/rialto/io/RialtoDbWriter.cpp b/plugins/rialto/io/RialtoDbWriter.cpp index 9a1451c96c..1c9e1dc9a9 100644 --- a/plugins/rialto/io/RialtoDbWriter.cpp +++ b/plugins/rialto/io/RialtoDbWriter.cpp @@ -183,7 +183,7 @@ void RialtoDbWriter::localStart() FileUtils::deleteFile(m_connection); m_rialtoDb = new RialtoDb(m_connection); - m_rialtoDb->open(true); + m_rialtoDb->create(); } diff --git a/plugins/rialto/test/RialtoDbWriterTest.cpp b/plugins/rialto/test/RialtoDbWriterTest.cpp index 12ba352505..889037ede7 100644 --- a/plugins/rialto/test/RialtoDbWriterTest.cpp +++ b/plugins/rialto/test/RialtoDbWriterTest.cpp @@ -48,7 +48,8 @@ #include -#include // TODO: fix path +#include <../plugins/rialto/io/RialtoDb.hpp> // TODO: fix path +#include <../plugins/sqlite/io/SQLiteCommon.hpp> // TODO: fix path using namespace pdal; @@ -91,66 +92,26 @@ const struct Data { }; -static void readBytes(const std::string& filename, uint8_t* buf, uint32_t len) -{ - const char* name = Support::temppath(filename).c_str(); - FILE* fp = fopen(name, "rb"); - EXPECT_TRUE(fp != NULL); - size_t cnt = fread(buf, 1, len, fp); - EXPECT_EQ(cnt, len); - fclose(fp); -} - - -static void verifyBytes(char* actualData, +static void verifyBytes(std::vector& buf, const Data* expectedData) { - EXPECT_EQ(*(double*)(actualData+0), expectedData->x); - EXPECT_EQ(*(double*)(actualData+8), expectedData->y); - EXPECT_EQ(*(double*)(actualData+16), expectedData->z); -} - - -static void verify(const std::string& filename, - const Data* expectedData, - uint8_t expectedMask) -{ - uint32_t actualSize = FileUtils::fileSize(Support::temppath(filename)); - - if (expectedData) { - EXPECT_EQ(actualSize, 25u); - union { - uint8_t buf[25]; - struct { - double x; - double y; - double z; - uint8_t m; - } actualData; - } u; - readBytes(filename, u.buf, 25); - EXPECT_EQ(u.actualData.x, expectedData->x); - EXPECT_EQ(u.actualData.y, expectedData->y); - EXPECT_EQ(u.actualData.z, expectedData->z); - EXPECT_EQ(u.actualData.m, expectedMask); - } else { - EXPECT_EQ(actualSize, 1u); - uint8_t actualMask; - readBytes(filename, &actualMask, 1); - EXPECT_EQ(actualMask, expectedMask); - } -} - - -static void verifyDirectorySize(const std::string& path, uint32_t expectedSize) -{ - boost::filesystem::directory_iterator iter(Support::temppath(path)); - uint32_t cnt = 0; - while (iter != boost::filesystem::directory_iterator()) { - ++cnt; - ++iter; - } - EXPECT_EQ(cnt, expectedSize); + const unsigned char* p = &buf[0]; + + union U { + uint8_t buf[24]; + struct S { + double x; + double y; + double z; + } s; + } u; + + for (int i=0; i<24; i++) + u.buf[i] = buf[i]; + + EXPECT_EQ(u.s.x, expectedData->x); + EXPECT_EQ(u.s.y, expectedData->y); + EXPECT_EQ(u.s.z, expectedData->z); } @@ -191,7 +152,6 @@ TEST(RialtoDbWriterTest, testWriter) tiler.setOptions(tilerOptions); tiler.setInput(stats); - //RialtoFileWriter writer; Options writerOptions; writerOptions.add("filename", Support::temppath("rialto2.sqlite")); writerOptions.add("overwrite", true); @@ -201,7 +161,7 @@ TEST(RialtoDbWriterTest, testWriter) writer->setOptions(writerOptions); writer->setInput(tiler); - // execution + // execution: write to database writer->prepare(table); PointViewSet outputViews = writer->execute(table); delete writer; @@ -209,10 +169,10 @@ TEST(RialtoDbWriterTest, testWriter) // verification rialtosupport::RialtoDb db(Support::temppath("rialto2.sqlite")); db.open(false); - + std::vector tileSetIds = db.getTileSetIds(); EXPECT_EQ(tileSetIds.size(), 1u); - + rialtosupport::RialtoDb::TileSetInfo tileSetInfo = db.getTileSetInfo(tileSetIds[0]); EXPECT_DOUBLE_EQ(tileSetInfo.minx, -180.0); EXPECT_DOUBLE_EQ(tileSetInfo.miny, -90.0); @@ -251,72 +211,60 @@ TEST(RialtoDbWriterTest, testWriter) EXPECT_EQ(tilesAt2.size(), 8u); std::vector tilesAt3 = db.getTileIdsAtLevel(0, 3); EXPECT_EQ(tilesAt3.size(), 0u); - - char* buf = NULL; - uint32_t bufLen = 0; - + + rialtosupport::RialtoDb::TileInfo info; + { - db.getTileData(tilesAt0[0], buf, bufLen); - EXPECT_EQ(bufLen, 24u); - EXPECT_TRUE(buf != NULL); - verifyBytes(buf, &data[0]); + info = db.getTileInfo(tilesAt0[0], true); + EXPECT_EQ(info.patch->buf.size(), 24u); + verifyBytes(info.patch->buf, &data[0]); } - { + { // TODO: these two are order-dependent - db.getTileData(tilesAt1[0], buf, bufLen); - EXPECT_EQ(bufLen, 24u); - EXPECT_TRUE(buf != NULL); - verifyBytes(buf, &data[0]); - - db.getTileData(tilesAt1[1], buf, bufLen); - EXPECT_EQ(bufLen, 24u); - EXPECT_TRUE(buf != NULL); - verifyBytes(buf, &data[4]); + info = db.getTileInfo(tilesAt1[0], true); + EXPECT_EQ(info.patch->buf.size(), 24u); + verifyBytes(info.patch->buf, &data[0]); + + info = db.getTileInfo(tilesAt1[1], true); + EXPECT_EQ(info.patch->buf.size(), 24u); + verifyBytes(info.patch->buf, &data[4]); } - { + { // TODO: these eight are order-dependent - db.getTileData(tilesAt2[0], buf, bufLen); - EXPECT_EQ(bufLen, 24u); - EXPECT_TRUE(buf != NULL); - verifyBytes(buf, &data[0]); - - db.getTileData(tilesAt2[1], buf, bufLen); - EXPECT_EQ(bufLen, 24u); - EXPECT_TRUE(buf != NULL); - verifyBytes(buf, &data[1]); - - db.getTileData(tilesAt2[2], buf, bufLen); - EXPECT_EQ(bufLen, 24u); - EXPECT_TRUE(buf != NULL); - verifyBytes(buf, &data[2]); - - db.getTileData(tilesAt2[3], buf, bufLen); - EXPECT_EQ(bufLen, 24u); - EXPECT_TRUE(buf != NULL); - verifyBytes(buf, &data[3]); - - db.getTileData(tilesAt2[4], buf, bufLen); - EXPECT_EQ(bufLen, 24u); - EXPECT_TRUE(buf != NULL); - verifyBytes(buf, &data[4]); - - db.getTileData(tilesAt2[5], buf, bufLen); - EXPECT_EQ(bufLen, 24u); - EXPECT_TRUE(buf != NULL); - verifyBytes(buf, &data[5]); - - db.getTileData(tilesAt2[6], buf, bufLen); - EXPECT_EQ(bufLen, 24u); - EXPECT_TRUE(buf != NULL); - verifyBytes(buf, &data[6]); - - db.getTileData(tilesAt2[7], buf, bufLen); - EXPECT_EQ(bufLen, 24u); - EXPECT_TRUE(buf != NULL); - verifyBytes(buf, &data[7]); + info = db.getTileInfo(tilesAt2[0], true); + EXPECT_EQ(info.patch->buf.size(), 24u); + verifyBytes(info.patch->buf, &data[0]); + + info = db.getTileInfo(tilesAt2[1], true); + EXPECT_EQ(info.patch->buf.size(), 24u); + verifyBytes(info.patch->buf, &data[1]); + + info = db.getTileInfo(tilesAt2[2], true); + EXPECT_EQ(info.patch->buf.size(), 24u); + verifyBytes(info.patch->buf, &data[2]); + + info = db.getTileInfo(tilesAt2[3], true); + EXPECT_EQ(info.patch->buf.size(), 24u); + verifyBytes(info.patch->buf, &data[3]); + + info = db.getTileInfo(tilesAt2[4], true); + EXPECT_EQ(info.patch->buf.size(), 24u); + verifyBytes(info.patch->buf, &data[4]); + + info = db.getTileInfo(tilesAt2[5], true); + EXPECT_EQ(info.patch->buf.size(), 24u); + verifyBytes(info.patch->buf, &data[5]); + + info = db.getTileInfo(tilesAt2[6], true); + EXPECT_EQ(info.patch->buf.size(), 24u); + verifyBytes(info.patch->buf, &data[6]); + + info = db.getTileInfo(tilesAt2[7], true); + EXPECT_EQ(info.patch->buf.size(), 24u); + verifyBytes(info.patch->buf, &data[7]); } - + //FileUtils::deleteDirectory(Support::temppath("rialto2.sqlite")); }