From 3e345bd0ae34b1eb66062e9622d5242bcdc586ac Mon Sep 17 00:00:00 2001 From: "Michael P. Gerlek" Date: Wed, 6 May 2015 14:49:06 -0400 Subject: [PATCH] end-to-end passing validation --- plugins/rialto/io/RialtoDb.cpp | 108 +++++++++++++++++- plugins/rialto/io/RialtoDb.hpp | 10 +- plugins/rialto/io/RialtoDbWriter.cpp | 23 ++-- plugins/rialto/test/RialtoDbWriterTest.cpp | 127 ++++++++++++++------- 4 files changed, 206 insertions(+), 62 deletions(-) diff --git a/plugins/rialto/io/RialtoDb.cpp b/plugins/rialto/io/RialtoDb.cpp index 949149cd12..25d9605af9 100644 --- a/plugins/rialto/io/RialtoDb.cpp +++ b/plugins/rialto/io/RialtoDb.cpp @@ -292,6 +292,104 @@ RialtoDb::TileSetInfo RialtoDb::getTileSetInfo(uint32_t tileSetId) } +RialtoDb::TileInfo RialtoDb::getTileInfo(uint32_t tileSetId, uint32_t tileId) +{ + TileInfo info; + + std::ostringstream oss; + oss << "SELECT tile_id,tile_set_id,level,x,y " + << "FROM Tiles WHERE tile_set_id=" << tileSetId + << " AND tile_id=" << tileId; + + log()->get(LogLevel::Debug) << "SELECT for tile" << std::endl; + + m_session->query(oss.str()); + + // should get exactly one row back + const row* r = m_session->get(); + assert(r); + + assert(tileId == boost::lexical_cast(r->at(0).data)); + assert(tileSetId == boost::lexical_cast(r->at(1).data)); + info.level = boost::lexical_cast(r->at(2).data); + info.x = boost::lexical_cast(r->at(3).data); + info.y = boost::lexical_cast(r->at(4).data); + + assert(!m_session->next()); + + return info; +} + + +std::vector RialtoDb::getTileIdsAtLevel(uint32_t tileSetId, uint32_t level) +{ + std::vector ids; + + std::ostringstream oss; + oss << "SELECT tile_id FROM Tiles" + << " WHERE tile_set_id=" << tileSetId + << " AND level=" << level; + + log()->get(LogLevel::Debug) << "SELECT for tile ids at level: " << level << std::endl; + + m_session->query(oss.str()); + + do { + const row* r = m_session->get(); + if (!r) break; + + uint32_t id = boost::lexical_cast(r->at(0).data); + log()->get(LogLevel::Debug) << " got tile id=" << id << std::endl; + ids.push_back(id); + } while (m_session->next()); + + return ids; +} + + +void RialtoDb::getTileData(uint32_t tileId, char*& buf, uint32_t& bufLen) +{ + buf = NULL; + bufLen = 0; + + std::ostringstream oss; + oss << "SELECT points " + << "FROM Tiles " + << "WHERE tile_id=" << tileId; + + log()->get(LogLevel::Debug) << "SELECT for data for tile: " << tileId << std::endl; + + m_session->query(oss.str()); + + // should get exactly one row back + const row* r = m_session->get(); + assert(r); + + double* x = (double*)(&r->at(0).blobBuf[0]); + double* y = (double*)(&r->at(0).blobBuf[8]); + double* z = (double*)(&r->at(0).blobBuf[16]); + log()->get(LogLevel::Debug) << " x: " << *x << std::endl; + log()->get(LogLevel::Debug) << " y: " << *y << std::endl; + log()->get(LogLevel::Debug) << " z: " << *z << std::endl; + + char* p = (char*)(&r->at(0).blobBuf[0]); + + bufLen = (r->at(0)).blobLen; + buf = new char[bufLen]; + for (uint32_t i=0; iget(LogLevel::Debug) << " xx: " << *xx << std::endl; + log()->get(LogLevel::Debug) << " yy: " << *yy << std::endl; + log()->get(LogLevel::Debug) << " zz: " << *zz << std::endl; + assert(buf); + assert(bufLen); +} + + RialtoDb::DimensionInfo RialtoDb::getDimensionInfo(uint32_t tileSetId, uint32_t position) { DimensionInfo info; @@ -390,6 +488,9 @@ void RialtoDb::addDimensions(uint32_t tileSetId, uint32_t RialtoDb::addTile(const RialtoDb::TileInfo& data, char* buf, uint32_t buflen) { + assert(buf); + assert(buflen); + std::ostringstream oss; oss << "INSERT INTO Tiles " << "(tile_set_id, level, x, y, points) " @@ -402,16 +503,13 @@ uint32_t RialtoDb::addTile(const RialtoDb::TileInfo& data, char* buf, uint32_t b r.push_back(column(data.level)); r.push_back(column(data.x)); r.push_back(column(data.y)); - if (buf) - r.push_back(blob(buf, buflen)); - else - r.push_back(column('NULL')); + r.push_back(blob(buf, buflen)); rs.push_back(r); m_session->insert(oss.str(), rs); long id = m_session->last_row_id(); - log()->get(LogLevel::Debug1) << "inserted Tile, id=" << id << std::endl; + log()->get(LogLevel::Debug) << "inserted Tile, id=" << id << std::endl; return id; } diff --git a/plugins/rialto/io/RialtoDb.hpp b/plugins/rialto/io/RialtoDb.hpp index c2cc873410..d237d46da2 100644 --- a/plugins/rialto/io/RialtoDb.hpp +++ b/plugins/rialto/io/RialtoDb.hpp @@ -133,8 +133,14 @@ class PDAL_DLL RialtoDb uint32_t addTile(const RialtoDb::TileInfo& data, char* buf, uint32_t buflen); // get info about a tile - TileInfo getTileInfo(uint32_t tileId); - + TileInfo getTileInfo(uint32_t tileSetId, uint32_t tileId); + + // get the raw buffer with the points + void getTileData(uint32_t tileId, char*& buf, uint32_t& buflen); + + // use with caution for levels greater than 16 or so + std::vector getTileIdsAtLevel(uint32_t tileSetId, uint32_t level); + // query for all the points of a tile set, bounded by bbox region // returns a pipeline made up of a BufferReader and a CropFilter Stage* query(uint32_t tileSetId, diff --git a/plugins/rialto/io/RialtoDbWriter.cpp b/plugins/rialto/io/RialtoDbWriter.cpp index 13aa66396f..9a1451c96c 100644 --- a/plugins/rialto/io/RialtoDbWriter.cpp +++ b/plugins/rialto/io/RialtoDbWriter.cpp @@ -130,28 +130,23 @@ void RialtoDbWriter::writeTile(MetadataNode tileNode, PointView* view) data.y = getMetadataU32(tileNode, "tileY"); const uint32_t mask = getMetadataU32(tileNode, "mask"); + log()->get(LogLevel::Debug) << "RialtoDbWriter::writeTile for " + << data.level << "," << data.x << "," << data.y << " " + << (view==0 ? "no" : "yes") + << std::endl; + char* buf = NULL; size_t bufsiz = 0; if (view) { - char* buf = createBlob(view, bufsiz); + buf = createBlob(view, bufsiz); } - uint32_t id = m_rialtoDb->addTile(data, buf, (uint32_t)bufsiz); - delete[] buf; // TODO - -/* - if (view) + if (buf) { - size_t bufsiz; - char* buf = createBlob(view, bufsiz); - fwrite(buf, bufsiz, 1, fp); + uint32_t id = m_rialtoDb->addTile(data, buf, (uint32_t)bufsiz); + delete[] buf; // TODO } - - uint8_t mask8 = mask; - fwrite(&mask8, 1, 1, fp); -*/ - } diff --git a/plugins/rialto/test/RialtoDbWriterTest.cpp b/plugins/rialto/test/RialtoDbWriterTest.cpp index 8570d512ef..12ba352505 100644 --- a/plugins/rialto/test/RialtoDbWriterTest.cpp +++ b/plugins/rialto/test/RialtoDbWriterTest.cpp @@ -102,6 +102,15 @@ static void readBytes(const std::string& filename, uint8_t* buf, uint32_t len) } +static void verifyBytes(char* actualData, + const Data* expectedData) +{ + EXPECT_EQ(*(double*)(actualData+0), expectedData->x); + EXPECT_EQ(*(double*)(actualData+8), expectedData->y); + EXPECT_EQ(*(double*)(actualData+16), expectedData->z); +} + + static void verify(const std::string& filename, const Data* expectedData, uint8_t expectedMask) @@ -233,45 +242,81 @@ TEST(RialtoDbWriterTest, testWriter) EXPECT_DOUBLE_EQ(dimensionInfo.minimum, 0.0); EXPECT_DOUBLE_EQ(dimensionInfo.mean+100.0, 38.5+100.0); // TODO EXPECT_DOUBLE_EQ(dimensionInfo.maximum, 77.0); -return; - - verify("rialto1/0/0/0.ria-db", &data[0], 15); - verify("rialto1/0/1/0.ria-db", NULL, 15); - - verify("rialto1/1/0/0.ria-db", &data[0], 8); - verify("rialto1/1/0/1.ria-db", NULL, 1); - verify("rialto1/1/1/0.ria-db", NULL, 4); - verify("rialto1/1/1/1.ria-db", NULL, 2); - verify("rialto1/1/2/0.ria-db", &data[4], 2); - verify("rialto1/1/2/1.ria-db", NULL, 4); - verify("rialto1/1/3/0.ria-db", NULL, 1); - verify("rialto1/1/3/1.ria-db", NULL, 8); - - verify("rialto1/2/0/0.ria-db", &data[0], 0); - verify("rialto1/2/0/3.ria-db", &data[2], 0); - verify("rialto1/2/3/0.ria-db", &data[1], 0); - verify("rialto1/2/3/3.ria-db", &data[3], 0); - verify("rialto1/2/5/1.ria-db", &data[4], 0); - verify("rialto1/2/5/2.ria-db", &data[6], 0); - verify("rialto1/2/6/1.ria-db", &data[5], 0); - verify("rialto1/2/6/2.ria-db", &data[7], 0); - - verifyDirectorySize("rialto1", 4); - verifyDirectorySize("rialto1/0", 2); - verifyDirectorySize("rialto1/0/0", 1); - verifyDirectorySize("rialto1/0/1", 1); - verifyDirectorySize("rialto1/1", 4); - verifyDirectorySize("rialto1/1/0", 2); - verifyDirectorySize("rialto1/1/1", 2); - verifyDirectorySize("rialto1/1/2", 2); - verifyDirectorySize("rialto1/1/3", 2); - verifyDirectorySize("rialto1/2", 4); - verifyDirectorySize("rialto1/2/0", 2); - verifyDirectorySize("rialto1/2/3", 2); - verifyDirectorySize("rialto1/2/5", 2); - verifyDirectorySize("rialto1/2/6", 2); - - //if (ok) { - //FileUtils::deleteDirectory(Support::temppath("rialto2.sqlite")); - //} + + std::vector tilesAt0 = db.getTileIdsAtLevel(0, 0); + EXPECT_EQ(tilesAt0.size(), 1u); + std::vector tilesAt1 = db.getTileIdsAtLevel(0, 1); + EXPECT_EQ(tilesAt1.size(), 2u); + std::vector tilesAt2 = db.getTileIdsAtLevel(0, 2); + EXPECT_EQ(tilesAt2.size(), 8u); + std::vector tilesAt3 = db.getTileIdsAtLevel(0, 3); + EXPECT_EQ(tilesAt3.size(), 0u); + + char* buf = NULL; + uint32_t bufLen = 0; + + { + db.getTileData(tilesAt0[0], buf, bufLen); + EXPECT_EQ(bufLen, 24u); + EXPECT_TRUE(buf != NULL); + verifyBytes(buf, &data[0]); + } + + { + // TODO: these two are order-dependent + db.getTileData(tilesAt1[0], buf, bufLen); + EXPECT_EQ(bufLen, 24u); + EXPECT_TRUE(buf != NULL); + verifyBytes(buf, &data[0]); + + db.getTileData(tilesAt1[1], buf, bufLen); + EXPECT_EQ(bufLen, 24u); + EXPECT_TRUE(buf != NULL); + verifyBytes(buf, &data[4]); + } + + { + // TODO: these eight are order-dependent + db.getTileData(tilesAt2[0], buf, bufLen); + EXPECT_EQ(bufLen, 24u); + EXPECT_TRUE(buf != NULL); + verifyBytes(buf, &data[0]); + + db.getTileData(tilesAt2[1], buf, bufLen); + EXPECT_EQ(bufLen, 24u); + EXPECT_TRUE(buf != NULL); + verifyBytes(buf, &data[1]); + + db.getTileData(tilesAt2[2], buf, bufLen); + EXPECT_EQ(bufLen, 24u); + EXPECT_TRUE(buf != NULL); + verifyBytes(buf, &data[2]); + + db.getTileData(tilesAt2[3], buf, bufLen); + EXPECT_EQ(bufLen, 24u); + EXPECT_TRUE(buf != NULL); + verifyBytes(buf, &data[3]); + + db.getTileData(tilesAt2[4], buf, bufLen); + EXPECT_EQ(bufLen, 24u); + EXPECT_TRUE(buf != NULL); + verifyBytes(buf, &data[4]); + + db.getTileData(tilesAt2[5], buf, bufLen); + EXPECT_EQ(bufLen, 24u); + EXPECT_TRUE(buf != NULL); + verifyBytes(buf, &data[5]); + + db.getTileData(tilesAt2[6], buf, bufLen); + EXPECT_EQ(bufLen, 24u); + EXPECT_TRUE(buf != NULL); + verifyBytes(buf, &data[6]); + + db.getTileData(tilesAt2[7], buf, bufLen); + EXPECT_EQ(bufLen, 24u); + EXPECT_TRUE(buf != NULL); + verifyBytes(buf, &data[7]); + } + + //FileUtils::deleteDirectory(Support::temppath("rialto2.sqlite")); }