From d34b3f46c9ec165cee6d60d04babb0ab70716896 Mon Sep 17 00:00:00 2001 From: "Michael P. Gerlek" Date: Thu, 21 May 2015 13:40:03 -0400 Subject: [PATCH] w00t! Rialto is now a GeoPackage --- plugins/rialto/io/RialtoDb.cpp | 451 ++++----------------- plugins/rialto/io/RialtoDb.hpp | 28 +- plugins/rialto/io/RialtoDbReader.cpp | 27 +- plugins/rialto/io/RialtoDbReader.hpp | 2 +- plugins/rialto/io/RialtoDbWriter.cpp | 12 +- plugins/rialto/io/RialtoDbWriter.hpp | 4 +- plugins/rialto/io/RialtoFileWriter.cpp | 8 +- plugins/rialto/io/RialtoFileWriter.hpp | 4 +- plugins/rialto/io/RialtoWriter.cpp | 12 +- plugins/rialto/io/RialtoWriter.hpp | 8 +- plugins/rialto/test/RialtoDbWriterTest.cpp | 56 ++- 11 files changed, 155 insertions(+), 457 deletions(-) diff --git a/plugins/rialto/io/RialtoDb.cpp b/plugins/rialto/io/RialtoDb.cpp index db9f6ad361..026c0a4a48 100644 --- a/plugins/rialto/io/RialtoDb.cpp +++ b/plugins/rialto/io/RialtoDb.cpp @@ -49,32 +49,6 @@ #include <../plugins/sqlite/io/SQLiteCommon.hpp> // TODO: fix path -// A Rialto database contains these tables: -// -// TileSets -// tile_set_id (PK) -// name -// maxLevel -// numDims -// -// Tiles -// tile_id (PK) -// tile_set_id (FK) -// level -// column -// row -// points -// -// Dimensions -// tile_set_id (FK) -// name -// position -// datatype -// minimum -// mean -// maximum - - namespace pdal { @@ -133,9 +107,6 @@ void RialtoDb::create() m_sqlite->connect(true); e_creation.start(); - createTileSetsTable(); - createTilesTable(); - createDimensionsTable(); createGpkgId(); createTableGpkgSpatialRefSys(); @@ -153,6 +124,15 @@ void RialtoDb::create() } +void RialtoDb::verifyTableExists(std::string const& name) const +{ + if (!m_sqlite->doesTableExist(name)) + { + throw pdal_error("RialtoDb: required table '" + name + "' not found"); + } +} + + void RialtoDb::open(bool writable) { if (m_sqlite) @@ -170,12 +150,13 @@ void RialtoDb::open(bool writable) m_sqlite = std::unique_ptr(new SQLite(m_connection, m_log)); m_sqlite->connect(writable); - if (!m_sqlite->doesTableExist("TileSets")) - throw pdal_error("RialtoDb: required table 'TileSets' not found"); - if (!m_sqlite->doesTableExist("Tiles")) - throw pdal_error("RialtoDb: required table 'Tiles' not found"); - if (!m_sqlite->doesTableExist("Dimensions")) - throw pdal_error("RialtoDb: required table 'Dimensions' not found"); + verifyTableExists("gpkg_spatial_ref_sys"); + verifyTableExists("gpkg_contents"); + verifyTableExists("gpkg_pctile_matrix"); + verifyTableExists("gpkg_pctile_matrix_set"); + verifyTableExists("gpkg_extensions"); + verifyTableExists("gpkg_metadata"); + verifyTableExists("gpkg_metadata_reference"); } @@ -195,6 +176,8 @@ void RialtoDb::close() m_sqlite->commit(); } +#if 0 +// TODO std::ostringstream oss2; oss2 << "CREATE INDEX index_name ON Tiles(column,row)"; m_sqlite->execute(oss2.str()); @@ -202,6 +185,7 @@ void RialtoDb::close() std::ostringstream oss3; oss3 << "CREATE INDEX index_name2 ON Tiles(level)"; m_sqlite->execute(oss3.str()); +#endif e_indexCreation.stop(); } @@ -281,26 +265,6 @@ void RialtoDb::createTableGpkgSpatialRefSys() } -void RialtoDb::createTileSetsTable() -{ - if (m_sqlite->doesTableExist("TileSets")) - { - throw pdal_error("RialtoDB: invalid state (table 'TileSets' already exists)"); - } - - std::ostringstream oss1; - - oss1 << "CREATE TABLE TileSets(" - << "tile_set_id INTEGER PRIMARY KEY AUTOINCREMENT," - << "name VARCHAR(64) NOT NULL," // TODO - << "maxLevel INTEGER NOT NULL," - << "numDims INTEGER NOT NULL" - << ")"; - - m_sqlite->execute(oss1.str()); -} - - void RialtoDb::createTableGpkgContents() { if (m_sqlite->doesTableExist("gpkg_contents")) @@ -526,109 +490,36 @@ void RialtoDb::createTablePctilesDimensionSet() } -void RialtoDb::createTilesTable() -{ - if (m_sqlite->doesTableExist("Tiles")) - { - throw pdal_error("RialtoDB: invalid state (table 'Tiles' already exists)"); - } - - std::ostringstream oss1; - oss1 << "CREATE TABLE Tiles(" - << "tile_id INTEGER PRIMARY KEY AUTOINCREMENT," - << " tile_set_id INTEGER NOT NULL," - << " level INTEGER NOT NULL," - << " column INTEGER NOT NULL," - << " row INTEGER NOT NULL," - << " numPoints INTEGER NOT NULL," - << " points BLOB NOT NULL," - << " FOREIGN KEY(tile_set_id) REFERENCES TileSets(tile_set_id)" -// << " UNIQUE(level,column,row)" - << ")"; - - m_sqlite->execute(oss1.str()); -} - - -void RialtoDb::createDimensionsTable() -{ - if (m_sqlite->doesTableExist("Dimensions")) - { - throw pdal_error("RialtoDB: invalid state (table 'Dimensions' already exists)"); - } - - std::ostringstream oss1; - std::ostringstream oss2; - - oss1 << "CREATE TABLE Dimensions(" - << "tile_set_id INTEGER NOT NULL," - << "name VARCHAR(256) NOT NULL," // TODO - << "position INTEGER NOT NULL," - << "dataType VARCHAR(256) NOT NULL," - << "minimum DOUBLE NOT NULL," - << "mean DOUBLE NOT NULL," - << "maximum DOUBLE NOT NULL," - << "FOREIGN KEY(tile_set_id) REFERENCES TileSets(tile_set_id)" - << ")"; - - m_sqlite->execute(oss1.str()); -} - - -void RialtoDb::readTileSetIds(std::vector& ids, std::vector& names) const +void RialtoDb::readTileSetIds(std::vector& names) const { if (!m_sqlite) { throw pdal_error("RialtoDB: invalid state (session does exist)"); } - - { - ids.clear(); - - std::ostringstream oss; - oss << "SELECT tile_set_id FROM TileSets"; - - log()->get(LogLevel::Debug1) << "SELECT for tile set ids" << std::endl; - - m_sqlite->query(oss.str()); - - do { - const row* r = m_sqlite->get(); - if (!r) break; - - column const& c = r->at(0); - const uint32_t id = boost::lexical_cast(c.data); - log()->get(LogLevel::Debug1) << " got id: " << id << std::endl; - ids.push_back(id); - - } while (m_sqlite->next()); - } - { - names.clear(); + names.clear(); - std::ostringstream oss; - oss << "SELECT table_name FROM gpkg_contents"; + std::ostringstream oss; + oss << "SELECT table_name FROM gpkg_contents"; - log()->get(LogLevel::Debug1) << "SELECT for tile set ids" << std::endl; + log()->get(LogLevel::Debug1) << "SELECT for tile set ids" << std::endl; - m_sqlite->query(oss.str()); + m_sqlite->query(oss.str()); - do { - const row* r = m_sqlite->get(); - if (!r) break; + do { + const row* r = m_sqlite->get(); + if (!r) break; - column const& c = r->at(0); - const std::string name = c.data; - log()->get(LogLevel::Debug1) << " got name: " << name << std::endl; - names.push_back(name); + column const& c = r->at(0); + const std::string name = c.data; + log()->get(LogLevel::Debug1) << " got name: " << name << std::endl; + names.push_back(name); - } while (m_sqlite->next()); - } -} + } while (m_sqlite->next()); +} -void RialtoDb::readTileSetInfo(uint32_t tileSetId, std::string const& name, TileSetInfo& info) const +void RialtoDb::readTileSetInfo(std::string const& name, TileSetInfo& info) const { if (!m_sqlite) { @@ -637,33 +528,8 @@ void RialtoDb::readTileSetInfo(uint32_t tileSetId, std::string const& name, Tile e_tileSetsRead.start(); - { - std::ostringstream oss; - oss << "SELECT tile_set_id,name,maxLevel,numDims " - << "FROM TileSets WHERE tile_set_id='" << tileSetId << "'"; - - log()->get(LogLevel::Debug) << "SELECT for tile set" << std::endl; - - m_sqlite->query(oss.str()); - - // should get exactly one row back - const row* r = m_sqlite->get(); - assert(r); - - assert(tileSetId == boost::lexical_cast(r->at(0).data)); - info.name = r->at(1).data; - info.maxLevel = boost::lexical_cast(r->at(2).data); - info.numDimensions = boost::lexical_cast(r->at(3).data); - - assert(!m_sqlite->next()); - - info.dimensions.clear(); - info.dimensions.resize(info.numDimensions); - readDimensionsInfo(tileSetId, name, info.dimensions); - } + info.name = name; - TileSetInfo info2; - info2.name = name; { std::ostringstream oss; oss << "SELECT table_name " @@ -690,7 +556,7 @@ void RialtoDb::readTileSetInfo(uint32_t tileSetId, std::string const& name, Tile // should get exactly one row back const row* r = m_sqlite->get(); assert(r); - info2.maxLevel = boost::lexical_cast(r->at(0).data); + info.maxLevel = boost::lexical_cast(r->at(0).data); assert(!m_sqlite->next()); } @@ -704,19 +570,19 @@ void RialtoDb::readTileSetInfo(uint32_t tileSetId, std::string const& name, Tile // should get exactly one row back const row* r = m_sqlite->get(); assert(r); - info2.numDimensions = boost::lexical_cast(r->at(0).data); + info.numDimensions = boost::lexical_cast(r->at(0).data); + assert(info.numDimensions != 0); assert(!m_sqlite->next()); } - assert(info.name == info2.name); - assert(info.maxLevel == info2.maxLevel); - assert(info.numDimensions == info2.numDimensions); - + readDimensionsInfo(info.name, info.dimensions); + assert(info.dimensions.size() == info.numDimensions); + e_tileSetsRead.stop(); } -void RialtoDb::readTileInfo(uint32_t tileId, bool withPoints, RialtoDb::TileInfo& info) const +void RialtoDb::readTileInfo(std::string const& name, uint32_t tileId, bool withPoints, RialtoDb::TileInfo& info) const { if (!m_sqlite) { @@ -726,10 +592,10 @@ void RialtoDb::readTileInfo(uint32_t tileId, bool withPoints, RialtoDb::TileInfo e_tilesRead.start(); std::ostringstream oss; - oss << "SELECT tile_id,tile_set_id,level,column,row,numPoints" - << (withPoints ? ",points " : " ") - << "FROM Tiles " - << "WHERE tile_id=" << tileId; + oss << "SELECT zoom_level,tile_column,tile_row,num_points" + << (withPoints ? ",tile_data " : " ") + << "FROM '" << name << "'" + << "WHERE id=" << tileId; //log()->get(LogLevel::Debug) << "SELECT for tile" << std::endl; @@ -739,18 +605,16 @@ void RialtoDb::readTileInfo(uint32_t tileId, bool withPoints, RialtoDb::TileInfo const row* r = m_sqlite->get(); assert(r); - assert(tileId == boost::lexical_cast(r->at(0).data)); - info.tileSetId = boost::lexical_cast(r->at(1).data); - info.level = boost::lexical_cast(r->at(2).data); - info.column = boost::lexical_cast(r->at(3).data); - info.row = boost::lexical_cast(r->at(4).data); - info.numPoints = boost::lexical_cast(r->at(5).data); + info.level = boost::lexical_cast(r->at(0).data); + info.column = boost::lexical_cast(r->at(1).data); + info.row = boost::lexical_cast(r->at(2).data); + info.numPoints = boost::lexical_cast(r->at(3).data); info.patch.buf.clear(); if (withPoints) { - const uint32_t blobLen = r->at(6).blobLen; - const std::vector& blobBuf = r->at(6).blobBuf; + const uint32_t blobLen = r->at(4).blobLen; + const std::vector& blobBuf = r->at(4).blobBuf; const unsigned char *pos = (const unsigned char *)&(blobBuf[0]); info.patch.putBytes(pos, blobLen); @@ -763,7 +627,7 @@ void RialtoDb::readTileInfo(uint32_t tileId, bool withPoints, RialtoDb::TileInfo } -void RialtoDb::readTileIdsAtLevel(uint32_t tileSetId, uint32_t level, std::vector& ids) const +void RialtoDb::readTileIdsAtLevel(std::string const& name, uint32_t level, std::vector& ids) const { if (!m_sqlite) { @@ -773,9 +637,8 @@ void RialtoDb::readTileIdsAtLevel(uint32_t tileSetId, uint32_t level, std::vecto ids.clear(); std::ostringstream oss; - oss << "SELECT tile_id FROM Tiles" - << " WHERE tile_set_id=" << tileSetId - << " AND level=" << level; + oss << "SELECT id FROM '" << name << "'" + << " WHERE zoom_level=" << level; log()->get(LogLevel::Debug) << "SELECT for tile ids at level: " << level << std::endl; @@ -792,7 +655,7 @@ void RialtoDb::readTileIdsAtLevel(uint32_t tileSetId, uint32_t level, std::vecto } -void RialtoDb::readDimensionsInfo(uint32_t tileSetId, std::string const& name, std::vector& dimensionsInfo) const +void RialtoDb::readDimensionsInfo(std::string const& name, std::vector& dimensionsInfo) const { if (!m_sqlite) { @@ -801,84 +664,37 @@ void RialtoDb::readDimensionsInfo(uint32_t tileSetId, std::string const& name, s dimensionsInfo.clear(); - { - std::ostringstream oss; - oss << "SELECT tile_set_id,name,position,dataType,minimum,mean,maximum " - << "FROM Dimensions " - << "WHERE tile_set_id=" << tileSetId; - - log()->get(LogLevel::Debug) << "SELECT for dim info of tile set " << tileSetId << std::endl; - - m_sqlite->query(oss.str()); - - int i = 0; - do { - const row* r = m_sqlite->get(); - if (!r) break; - - DimensionInfo info; - - assert(tileSetId == boost::lexical_cast(r->at(0).data)); - info.name = r->at(1).data; - info.position = boost::lexical_cast(r->at(2).data); - info.dataType = r->at(3).data; - info.minimum = boost::lexical_cast(r->at(4).data); - info.mean = boost::lexical_cast(r->at(5).data); - info.maximum = boost::lexical_cast(r->at(6).data); - - log()->get(LogLevel::Debug1) << "read dim: " << info.name << std::endl; - - ++i; - - dimensionsInfo.push_back(info); - } while (m_sqlite->next()); - } - - std::vector dimensionsInfo2; - { - std::ostringstream oss; - oss << "SELECT ordinal_position, dimension_name, data_type, description, minimum, mean, maximum " - << "FROM pctiles_dimension_set WHERE table_name='" << name << "'"; + std::ostringstream oss; + oss << "SELECT ordinal_position, dimension_name, data_type, description, minimum, mean, maximum " + << "FROM pctiles_dimension_set WHERE table_name='" << name << "'"; - m_sqlite->query(oss.str()); + m_sqlite->query(oss.str()); - int i = 0; - do { - const row* r = m_sqlite->get(); - if (!r) break; + int i = 0; + do { + const row* r = m_sqlite->get(); + if (!r) break; - DimensionInfo info; + DimensionInfo info; - info.position = boost::lexical_cast(r->at(0).data); - info.name = r->at(1).data; - info.dataType = r->at(2).data; - info.description = r->at(3).data; - info.minimum = boost::lexical_cast(r->at(4).data); - info.mean = boost::lexical_cast(r->at(5).data); - info.maximum = boost::lexical_cast(r->at(6).data); + info.position = boost::lexical_cast(r->at(0).data); + info.name = r->at(1).data; + info.dataType = r->at(2).data; + info.description = r->at(3).data; + info.minimum = boost::lexical_cast(r->at(4).data); + info.mean = boost::lexical_cast(r->at(5).data); + info.maximum = boost::lexical_cast(r->at(6).data); - log()->get(LogLevel::Debug1) << "read dim: " << info.name << std::endl; + log()->get(LogLevel::Debug1) << "read dim: " << info.name << std::endl; - ++i; - - dimensionsInfo2.push_back(info); - } while (m_sqlite->next()); - } - - assert(dimensionsInfo.size() == dimensionsInfo2.size()); - for (uint32_t i=0; inext()); } -uint32_t RialtoDb::writeTileSet(const RialtoDb::TileSetInfo& data) +void RialtoDb::writeTileSet(const RialtoDb::TileSetInfo& data) { if (!m_sqlite) { @@ -889,29 +705,6 @@ uint32_t RialtoDb::writeTileSet(const RialtoDb::TileSetInfo& data) e_tileSetsWritten.start(); - uint32_t tile_set_id; - { - std::ostringstream oss; - oss << "INSERT INTO TileSets " - << "(name, maxLevel, numDims) " - << "VALUES (?, ?, ?)"; - - records rs; - row r; - - r.push_back(column(data.name)); - r.push_back(column(data.maxLevel)); - r.push_back(column(data.numDimensions)); - rs.push_back(r); - - m_sqlite->insert(oss.str(), rs); - - tile_set_id = m_sqlite->last_row_id(); - log()->get(LogLevel::Debug) << "inserted TileSet, id=" << tile_set_id << std::endl; - - writeDimensions(tile_set_id, data.dimensions); - } - { createTableTilePyramidUserData(data.name); } @@ -1039,8 +832,6 @@ uint32_t RialtoDb::writeTileSet(const RialtoDb::TileSetInfo& data) } e_tileSetsWritten.stop(); - - return tile_set_id; } @@ -1078,7 +869,7 @@ void RialtoDb::writeDimensions(uint32_t tileSetId, } -uint32_t RialtoDb::writeTile(const std::string& tileSetName, const RialtoDb::TileInfo& data) +void RialtoDb::writeTile(const std::string& tileSetName, const RialtoDb::TileInfo& data) { if (!m_sqlite) { @@ -1099,36 +890,6 @@ uint32_t RialtoDb::writeTile(const std::string& tileSetName, const RialtoDb::Til assert(buf); assert(buflen); - uint32_t tile_id; - { - // note we don't use 'mask' in the database version of tiles - const std::string sql = - "INSERT INTO Tiles " - "(tile_set_id, level, column, row, numPoints, points) " - "VALUES (?, ?, ?, ?, ?, ?)"; - - records rs; - row r; - - r.push_back(column(data.tileSetId)); - r.push_back(column(data.level)); - r.push_back(column(data.column)); - r.push_back(column(data.row)); - r.push_back(column(data.numPoints)); - r.push_back(blob((char*)buf, (size_t)buflen)); - rs.push_back(r); - - m_sqlite->insert(sql, rs); - - tile_id = m_sqlite->last_row_id(); - //log()->get(LogLevel::Debug) << "inserted for tile set " << data.tileSetId - // << ": tile id " << tile_id - // << "(" << data.column - // << ", " << data.row - // << ", " << data.level - // << ")" << std::endl; - } - if (!m_haveWrittenMatrixAtLevel[data.level]) { const std::string sql = @@ -1177,8 +938,6 @@ uint32_t RialtoDb::writeTile(const std::string& tileSetName, const RialtoDb::Til e_tilesWritten.stop(); m_numPointsWritten += data.numPoints; - - return tile_id; } @@ -1220,7 +979,7 @@ void RialtoDb::matrixSizeAtLevel(uint32_t level, uint32_t& numCols, uint32_t& nu } -void RialtoDb::queryForTileIds(uint32_t tileSetId, +void RialtoDb::queryForTileIds(std::string const& name, double minx, double miny, double maxx, double maxy, uint32_t level, @@ -1231,7 +990,7 @@ void RialtoDb::queryForTileIds(uint32_t tileSetId, throw pdal_error("RialtoDB: invalid state (session does exist)"); } - log()->get(LogLevel::Debug) << "Querying tile set " << tileSetId + log()->get(LogLevel::Debug) << "Querying tile set " << name << " for some tile ids" << std::endl; ids.clear(); @@ -1253,13 +1012,12 @@ void RialtoDb::queryForTileIds(uint32_t tileSetId, assert(minrow <= maxrow); std::ostringstream oss; - oss << "SELECT tile_id FROM Tiles" - << " WHERE tile_set_id=" << tileSetId - << " AND level=" << level - << " AND column >= " << mincol - << " AND column <= " << maxcol - << " AND row >= " << minrow - << " AND row <= " << maxrow; + oss << "SELECT id FROM '" << name << "'" + << " WHERE zoom_level=" << level + << " AND tile_column >= " << mincol + << " AND tile_column <= " << maxcol + << " AND tile_row >= " << minrow + << " AND tile_row <= " << maxrow; m_sqlite->query(oss.str()); @@ -1276,7 +1034,7 @@ void RialtoDb::queryForTileIds(uint32_t tileSetId, } -void RialtoDb::queryForTileInfosBegin(uint32_t tileSetId, std::string const& name, +void RialtoDb::queryForTileInfosBegin(std::string const& name, double minx, double miny, double maxx, double maxy, uint32_t level) @@ -1288,7 +1046,7 @@ void RialtoDb::queryForTileInfosBegin(uint32_t tileSetId, std::string const& nam e_tilesRead.start(); - log()->get(LogLevel::Debug) << "Querying tile set " << tileSetId + log()->get(LogLevel::Debug) << "Querying tile set " << name << " for some tile infos" << std::endl; assert(minx <= maxx); @@ -1306,16 +1064,6 @@ void RialtoDb::queryForTileInfosBegin(uint32_t tileSetId, std::string const& nam assert(minrow <= maxrow); std::ostringstream oss; -#if 0 - oss << "SELECT tile_id,tile_set_id,level,column,row,numPoints,points" - << " FROM Tiles " - << " WHERE tile_set_id=" << tileSetId - << " AND level=" << level - << " AND column >= " << mincol - << " AND column <= " << maxcol - << " AND row >= " << minrow - << " AND row <= " << maxrow; -#else oss << "SELECT zoom_level,tile_column,tile_row,num_points,tile_data" << " FROM '" << name << "'" << " WHERE zoom_level=" << level @@ -1323,7 +1071,6 @@ void RialtoDb::queryForTileInfosBegin(uint32_t tileSetId, std::string const& nam << " AND tile_column <= " << maxcol << " AND tile_row >= " << minrow << " AND tile_row <= " << maxrow; -#endif m_sqlite->query(oss.str()); @@ -1343,23 +1090,6 @@ bool RialtoDb::queryForTileInfos(TileInfo& info) return false; } -#if 0 - //assert(tileId == boost::lexical_cast(r->at(0).data)); - info.tileSetId = boost::lexical_cast(r->at(1).data); - info.level = boost::lexical_cast(r->at(2).data); - info.column = boost::lexical_cast(r->at(3).data); - info.row = boost::lexical_cast(r->at(4).data); - info.numPoints = boost::lexical_cast(r->at(5).data); - - // this query always reads the points - info.patch.buf.clear(); - { - const uint32_t blobLen = r->at(6).blobLen; - const std::vector& blobBuf = r->at(6).blobBuf; - const unsigned char *pos = (const unsigned char *)&(blobBuf[0]); - info.patch.putBytes(pos, blobLen); - } -#else info.level = boost::lexical_cast(r->at(0).data); info.column = boost::lexical_cast(r->at(1).data); info.row = boost::lexical_cast(r->at(2).data); @@ -1373,7 +1103,6 @@ bool RialtoDb::queryForTileInfos(TileInfo& info) const unsigned char *pos = (const unsigned char *)&(blobBuf[0]); info.patch.putBytes(pos, blobLen); } -#endif e_tilesRead.stop(); diff --git a/plugins/rialto/io/RialtoDb.hpp b/plugins/rialto/io/RialtoDb.hpp index 0424d0992c..b09af47255 100644 --- a/plugins/rialto/io/RialtoDb.hpp +++ b/plugins/rialto/io/RialtoDb.hpp @@ -112,32 +112,33 @@ class PDAL_DLL RialtoDb // adds a tile set to the database, including its dimensions // // returns id of new data set - uint32_t writeTileSet(const RialtoDb::TileSetInfo& data); + void writeTileSet(const RialtoDb::TileSetInfo& data); // returns id of new tile - uint32_t writeTile(const std::string& tileSetName, const RialtoDb::TileInfo& data); + void writeTile(const std::string& tileSetName, const RialtoDb::TileInfo& data); // get list all the tile sets in the database, as a list of its - void readTileSetIds(std::vector&, std::vector& names) const; + void readTileSetIds(std::vector& names) const; // get info about a specific tile set (including its dimensions) - void readTileSetInfo(uint32_t tileSetId, std::string const& name, TileSetInfo& info) const; + void readTileSetInfo(std::string const& name, TileSetInfo& info) const; // get info about a tile - void readTileInfo(uint32_t tileId, bool withPoints, TileInfo& tileInfo) const; + void readTileInfo(std::string const& name, uint32_t tileId, bool withPoints, TileInfo& tileInfo) const; // use with caution for levels greater than 16 or so - void readTileIdsAtLevel(uint32_t tileSetId, uint32_t level, std::vector& tileIds) const; + // DANGER: this assumes only one tile set per database, use only for testing + void readTileIdsAtLevel(std::string const& name, uint32_t level, std::vector& tileIds) const; // query for all the tiles of a tile set, bounded by bbox region - void queryForTileIds(uint32_t tileSetId, + void queryForTileIds(std::string const& name, double minx, double miny, double maxx, double maxy, uint32_t level, std::vector& ids) const; // combines query-for-tile-ids with query-for-tile-info - void queryForTileInfosBegin(uint32_t tileSetId, std::string const& name, + void queryForTileInfosBegin(std::string const& name, double minx, double miny, double maxx, double maxy, uint32_t level); @@ -158,14 +159,11 @@ class PDAL_DLL RialtoDb void dumpStats() const; private: - // create the req'd tables in the db - void createTileSetsTable(); - void createTableGpkgContents(); - void createTilesTable(); - void createDimensionsTable(); - + void verifyTableExists(std::string const& name) const; + void createGpkgId(); void createTableGpkgSpatialRefSys(); + void createTableGpkgContents(); void createTableGpkgPctileMatrixSet(); void createTableGpkgPctileMatrix(); void createTableTilePyramidUserData(const std::string& table_name); @@ -179,7 +177,7 @@ class PDAL_DLL RialtoDb const std::vector& dimensions); // get info about one of the dimensions of a tile set - void readDimensionsInfo(uint32_t tileSetId, std::string const& name, std::vector&) const; + void readDimensionsInfo(std::string const& name, std::vector&) const; void matrixSizeAtLevel(uint32_t level, uint32_t& numCols, uint32_t& numRows) const; diff --git a/plugins/rialto/io/RialtoDbReader.cpp b/plugins/rialto/io/RialtoDbReader.cpp index d105515265..8021eaf8fe 100644 --- a/plugins/rialto/io/RialtoDbReader.cpp +++ b/plugins/rialto/io/RialtoDbReader.cpp @@ -63,16 +63,14 @@ void RialtoDbReader::initialize() m_db = std::unique_ptr(new RialtoDb(m_filename, log())); m_db->open(false); - std::vector ids; std::vector names; - m_db->readTileSetIds(ids, names); - assert(ids.size()==1); // TODO: always take the first one for now - m_tileSetId = ids[0]; + m_db->readTileSetIds(names); + m_tileSetName = names[0]; assert(names.size()==1); // TODO: always take the first one for now m_tileSetInfo = std::unique_ptr(new RialtoDb::TileSetInfo()); - m_db->readTileSetInfo(m_tileSetId, names[0], *m_tileSetInfo); + m_db->readTileSetInfo(m_tileSetName, *m_tileSetInfo); } } @@ -142,23 +140,7 @@ point_count_t RialtoDbReader::read(PointViewPtr view, point_count_t count) maxLevel = m_tileSetInfo->maxLevel; } -#if 0 - std::vector ids; - m_db->queryForTileIds(m_tileSetId, minx, miny, maxx, maxy, maxLevel, ids); - - for (auto id: ids) - { - RialtoDb::TileInfo info; - m_db->readTileInfo(id, true, info); - - log()->get(LogLevel::Debug) << " got some points: " << info.numPoints << std::endl; - - m_db->serializeToPointView(info, view); - - log()->get(LogLevel::Debug) << " view now has this many: " << view->size() << std::endl; - } -#else - m_db->queryForTileInfosBegin(m_tileSetId, "myunnamedlasfile", minx, miny, maxx, maxy, maxLevel); + m_db->queryForTileInfosBegin(m_tileSetName, minx, miny, maxx, maxy, maxLevel); RialtoDb::TileInfo info; @@ -184,7 +166,6 @@ point_count_t RialtoDbReader::read(PointViewPtr view, point_count_t count) log()->get(LogLevel::Debug) << " view now has this many: " << view->size() << std::endl; } while (m_db->queryForTileInfosNext()); -#endif return view->size(); } diff --git a/plugins/rialto/io/RialtoDbReader.hpp b/plugins/rialto/io/RialtoDbReader.hpp index c43693ddc7..b4c5311a9b 100644 --- a/plugins/rialto/io/RialtoDbReader.hpp +++ b/plugins/rialto/io/RialtoDbReader.hpp @@ -64,7 +64,7 @@ class PDAL_DLL RialtoDbReader : public Reader private: std::unique_ptr m_db; - uint32_t m_tileSetId; + std::string m_tileSetName; uint32_t m_level; std::unique_ptr m_tileSetInfo; BOX3D m_query; diff --git a/plugins/rialto/io/RialtoDbWriter.cpp b/plugins/rialto/io/RialtoDbWriter.cpp index 44ea86d152..e7a19cbb24 100644 --- a/plugins/rialto/io/RialtoDbWriter.cpp +++ b/plugins/rialto/io/RialtoDbWriter.cpp @@ -65,7 +65,7 @@ namespace -uint32_t RialtoDbWriter::writeHeader(const std::string& tileSetName, +void RialtoDbWriter::writeHeader(const std::string& tileSetName, MetadataNode tileSetNode, PointLayoutPtr layout) { @@ -74,24 +74,22 @@ uint32_t RialtoDbWriter::writeHeader(const std::string& tileSetName, RialtoDb::TileSetInfo tileSetInfo; serializeToTileSetInfo(tileSetName, tileSetNode, layout, tileSetInfo); - const uint32_t tileSetId = m_rialtoDb->writeTileSet(tileSetInfo); - - return tileSetId; + m_rialtoDb->writeTileSet(tileSetInfo); } -void RialtoDbWriter::writeTile(uint32_t tileSetId, const std::string& tileSetName, PointView* view, uint32_t level, uint32_t col, uint32_t row, uint32_t mask) +void RialtoDbWriter::writeTile(const std::string& tileSetName, PointView* view, uint32_t level, uint32_t col, uint32_t row, uint32_t mask) { //log()->get(LogLevel::Debug1) << "RialtoDbWriter::writeTile()" << std::endl; //printf("writing tile %d/%d/%d\n", level, col, row); RialtoDb::TileInfo tileInfo; - serializeToTileInfo(tileSetId, view, tileInfo, level, col, row, mask); + serializeToTileInfo(view, tileInfo, level, col, row, mask); if (tileInfo.patch.buf.size()) { - uint32_t id = m_rialtoDb->writeTile(tileSetName, tileInfo); + m_rialtoDb->writeTile(tileSetName, tileInfo); } } diff --git a/plugins/rialto/io/RialtoDbWriter.hpp b/plugins/rialto/io/RialtoDbWriter.hpp index dafac77553..5835992cc7 100644 --- a/plugins/rialto/io/RialtoDbWriter.hpp +++ b/plugins/rialto/io/RialtoDbWriter.hpp @@ -62,10 +62,10 @@ class PDAL_DLL RialtoDbWriter : public RialtoWriter Options getDefaultOptions(); virtual void localStart(); - virtual uint32_t writeHeader(const std::string& tileSetName, + virtual void writeHeader(const std::string& tileSetName, MetadataNode tileSetNode, PointLayoutPtr layout); - virtual void writeTile(uint32_t tileSetId, const std::string& tileSetName, PointView*, uint32_t level, uint32_t col, uint32_t row, uint32_t mask); + virtual void writeTile(const std::string& tileSetName, PointView*, uint32_t level, uint32_t col, uint32_t row, uint32_t mask); virtual void localFinish(); private: diff --git a/plugins/rialto/io/RialtoFileWriter.cpp b/plugins/rialto/io/RialtoFileWriter.cpp index ca474060e1..20d63540c4 100644 --- a/plugins/rialto/io/RialtoFileWriter.cpp +++ b/plugins/rialto/io/RialtoFileWriter.cpp @@ -61,7 +61,7 @@ namespace } // anonymous namespace -uint32_t RialtoFileWriter::writeHeader(const std::string& tileSetName, +void RialtoFileWriter::writeHeader(const std::string& tileSetName, MetadataNode tileSetNode, PointLayoutPtr layout) { @@ -98,16 +98,14 @@ uint32_t RialtoFileWriter::writeHeader(const std::string& tileSetName, fprintf(fp, "}\n"); fclose(fp); - - return 0; // tile set id not used for files } -void RialtoFileWriter::writeTile(uint32_t tileSetId, const std::string& tileSetName, PointView* view, uint32_t level, uint32_t col, uint32_t row, uint32_t mask) +void RialtoFileWriter::writeTile(const std::string& tileSetName, PointView* view, uint32_t level, uint32_t col, uint32_t row, uint32_t mask) { log()->get(LogLevel::Debug) << "RialtoFileWriter::writeTile()" << std::endl; RialtoDb::TileInfo tileInfo; - serializeToTileInfo(tileSetId, view, tileInfo, level, col, row, mask); + serializeToTileInfo(view, tileInfo, level, col, row, mask); std::ostringstream os; diff --git a/plugins/rialto/io/RialtoFileWriter.hpp b/plugins/rialto/io/RialtoFileWriter.hpp index 621e7c57eb..20bbea8168 100644 --- a/plugins/rialto/io/RialtoFileWriter.hpp +++ b/plugins/rialto/io/RialtoFileWriter.hpp @@ -61,10 +61,10 @@ class PDAL_DLL RialtoFileWriter : public RialtoWriter Options getDefaultOptions(); virtual void localStart(); - virtual uint32_t writeHeader(const std::string& tileSetName, + virtual void writeHeader(const std::string& tileSetName, MetadataNode tileSetNode, PointLayoutPtr layout); - virtual void writeTile(uint32_t tileSetId, const std::string& tileSetName, PointView*, uint32_t level, uint32_t col, uint32_t row, uint32_t mask); + virtual void writeTile(const std::string& tileSetName, PointView*, uint32_t level, uint32_t col, uint32_t row, uint32_t mask); virtual void localFinish(); private: diff --git a/plugins/rialto/io/RialtoWriter.cpp b/plugins/rialto/io/RialtoWriter.cpp index 71b1e805a7..b6e60d10d5 100644 --- a/plugins/rialto/io/RialtoWriter.cpp +++ b/plugins/rialto/io/RialtoWriter.cpp @@ -148,11 +148,9 @@ void RialtoWriter::serializeToPatch(const PointView& view, Patch& patch) } -void RialtoWriter::serializeToTileInfo(uint32_t tileSetId, PointView* view, RialtoDb::TileInfo& tileInfo, +void RialtoWriter::serializeToTileInfo(PointView* view, RialtoDb::TileInfo& tileInfo, uint32_t level, uint32_t col, uint32_t row, uint32_t mask) -{ - tileInfo.tileSetId = tileSetId; - +{ tileInfo.level = level; tileInfo.column = col; tileInfo.row = row; @@ -233,7 +231,7 @@ void RialtoWriter::ready(PointTableRef table) throw pdal_error("RialtoWriter: \"filters.tiler\" metadata not found"); } - m_tileSetId = writeHeader(m_tileSetName, tileSetNode, m_table->layout()); + writeHeader(m_tileSetName, tileSetNode, m_table->layout()); makePointViewMap(tileSetNode); } @@ -254,7 +252,7 @@ void RialtoWriter::write(const PointViewPtr viewPtr) uint32_t pvid = m_tileMetadata[idx+4]; assert(pvid == 0xffffffff || pvid == (uint32_t)viewPtr->id()); - writeTile(m_tileSetId, m_tileSetName, view, level, col, row, mask); + writeTile(m_tileSetName, view, level, col, row, mask); } @@ -317,7 +315,7 @@ void RialtoWriter::writeEmptyTiles() if (pvid == 0xffffffff) { - writeTile(m_tileSetId, m_tileSetName, NULL, level, col, row, mask); + writeTile(m_tileSetName, NULL, level, col, row, mask); } } } diff --git a/plugins/rialto/io/RialtoWriter.hpp b/plugins/rialto/io/RialtoWriter.hpp index 8d70b4fb32..a59d9b6a7b 100644 --- a/plugins/rialto/io/RialtoWriter.hpp +++ b/plugins/rialto/io/RialtoWriter.hpp @@ -71,7 +71,7 @@ class PDAL_DLL RialtoWriter : public Writer PointLayoutPtr layout, std::vector& infoList); static void serializeToPatch(const PointView& view, Patch& patch); - static void serializeToTileInfo(uint32_t tileSetId, PointView* view, RialtoDb::TileInfo& tileInfo, + static void serializeToTileInfo(PointView* view, RialtoDb::TileInfo& tileInfo, uint32_t level, uint32_t col, uint32_t row, uint32_t mask); static uint32_t getMetadataU32(const MetadataNode& parent, const std::string& name); @@ -89,10 +89,10 @@ class PDAL_DLL RialtoWriter : public Writer // implemented in derived classes virtual void localStart() = 0; - virtual uint32_t writeHeader(const std::string& tileSetName, + virtual void writeHeader(const std::string& tileSetName, MetadataNode tileSetNode, PointLayoutPtr layout) = 0; - virtual void writeTile(uint32_t tileSetId, const std::string& tileSetName, PointView*, uint32_t level, uint32_t col, uint32_t row, uint32_t mask) = 0; + virtual void writeTile(const std::string& tileSetName, PointView*, uint32_t level, uint32_t col, uint32_t row, uint32_t mask) = 0; virtual void localFinish() = 0; std::string m_tileSetName; @@ -109,8 +109,6 @@ class PDAL_DLL RialtoWriter : public Writer BasePointTable *m_table; std::string m_directory; - uint32_t m_tileSetId; - std::map m_pointViewMap2; // PV id to array index uint32_t* m_tileMetadata; uint32_t m_numTiles; diff --git a/plugins/rialto/test/RialtoDbWriterTest.cpp b/plugins/rialto/test/RialtoDbWriterTest.cpp index 74e58d6aee..f07f1fa6a7 100644 --- a/plugins/rialto/test/RialtoDbWriterTest.cpp +++ b/plugins/rialto/test/RialtoDbWriterTest.cpp @@ -74,13 +74,12 @@ void verifyDatabase(const std::string& filename, RialtoTest::Data* actualData) RialtoDb db(filename, log); db.open(false); - std::vector tileSetIds; std::vector names; - db.readTileSetIds(tileSetIds, names); - EXPECT_EQ(tileSetIds.size(), 1u); + db.readTileSetIds(names); + EXPECT_EQ(names.size(), 1u); RialtoDb::TileSetInfo tileSetInfo; - db.readTileSetInfo(tileSetIds[0], names[0], tileSetInfo); + db.readTileSetInfo(names[0], tileSetInfo); EXPECT_EQ(tileSetInfo.maxLevel, 2u); EXPECT_EQ(tileSetInfo.numDimensions, 3u); @@ -102,22 +101,22 @@ void verifyDatabase(const std::string& filename, RialtoTest::Data* actualData) EXPECT_DOUBLE_EQ(dimensionsInfo[2].maximum, 77.0); std::vector tilesAt0; - db.readTileIdsAtLevel(tileSetIds[0], 0, tilesAt0); + db.readTileIdsAtLevel(names[0], 0, tilesAt0); EXPECT_EQ(tilesAt0.size(), 1u); std::vector tilesAt1; - db.readTileIdsAtLevel(tileSetIds[0], 1, tilesAt1); + db.readTileIdsAtLevel(names[0], 1, tilesAt1); EXPECT_EQ(tilesAt1.size(), 2u); std::vector tilesAt2; - db.readTileIdsAtLevel(tileSetIds[0], 2, tilesAt2); + db.readTileIdsAtLevel(names[0], 2, tilesAt2); EXPECT_EQ(tilesAt2.size(), 8u); std::vector tilesAt3; - db.readTileIdsAtLevel(tileSetIds[0], 3, tilesAt3); + db.readTileIdsAtLevel(names[0], 3, tilesAt3); EXPECT_EQ(tilesAt3.size(), 0u); RialtoDb::TileInfo info; { - db.readTileInfo(tilesAt0[0], true, info); + db.readTileInfo(names[0], tilesAt0[0], true, info); EXPECT_EQ(info.numPoints, 1u); EXPECT_EQ(info.patch.buf.size(), 24u); RialtoTest::verifyPointFromBuffer(info.patch.buf, actualData[0]); @@ -125,12 +124,12 @@ void verifyDatabase(const std::string& filename, RialtoTest::Data* actualData) { // TODO: these two are order-dependent - db.readTileInfo(tilesAt1[0], true, info); + db.readTileInfo(names[0], tilesAt1[0], true, info); EXPECT_EQ(info.numPoints, 1u); EXPECT_EQ(info.patch.buf.size(), 24u); RialtoTest::verifyPointFromBuffer(info.patch.buf, actualData[0]); - db.readTileInfo(tilesAt1[1], true, info); + db.readTileInfo(names[0], tilesAt1[1], true, info); EXPECT_EQ(info.numPoints, 1u); EXPECT_EQ(info.patch.buf.size(), 24u); RialtoTest::verifyPointFromBuffer(info.patch.buf, actualData[4]); @@ -138,42 +137,42 @@ void verifyDatabase(const std::string& filename, RialtoTest::Data* actualData) { // TODO: these eight are order-dependent - db.readTileInfo(tilesAt2[0], true, info); + db.readTileInfo(names[0], tilesAt2[0], true, info); EXPECT_EQ(info.numPoints, 1u); EXPECT_EQ(info.patch.buf.size(), 24u); RialtoTest::verifyPointFromBuffer(info.patch.buf, actualData[0]); - db.readTileInfo(tilesAt2[1], true, info); + db.readTileInfo(names[0], tilesAt2[1], true, info); EXPECT_EQ(info.numPoints, 1u); EXPECT_EQ(info.patch.buf.size(), 24u); - RialtoTest::verifyPointFromBuffer(info.patch.buf, actualData[1]); + RialtoTest::verifyPointFromBuffer(info.patch.buf, actualData[2]); - db.readTileInfo(tilesAt2[2], true, info); + db.readTileInfo(names[0], tilesAt2[2], true, info); EXPECT_EQ(info.numPoints, 1u); EXPECT_EQ(info.patch.buf.size(), 24u); - RialtoTest::verifyPointFromBuffer(info.patch.buf, actualData[2]); + RialtoTest::verifyPointFromBuffer(info.patch.buf, actualData[1]); - db.readTileInfo(tilesAt2[3], true, info); + db.readTileInfo(names[0], tilesAt2[3], true, info); EXPECT_EQ(info.numPoints, 1u); EXPECT_EQ(info.patch.buf.size(), 24u); RialtoTest::verifyPointFromBuffer(info.patch.buf, actualData[3]); - db.readTileInfo(tilesAt2[4], true, info); + db.readTileInfo(names[0], tilesAt2[4], true, info); EXPECT_EQ(info.numPoints, 1u); EXPECT_EQ(info.patch.buf.size(), 24u); RialtoTest::verifyPointFromBuffer(info.patch.buf, actualData[4]); - db.readTileInfo(tilesAt2[5], true, info); + db.readTileInfo(names[0], tilesAt2[5], true, info); EXPECT_EQ(info.numPoints, 1u); EXPECT_EQ(info.patch.buf.size(), 24u); - RialtoTest::verifyPointFromBuffer(info.patch.buf, actualData[5]); + RialtoTest::verifyPointFromBuffer(info.patch.buf, actualData[6]); - db.readTileInfo(tilesAt2[6], true, info); + db.readTileInfo(names[0], tilesAt2[6], true, info); EXPECT_EQ(info.numPoints, 1u); EXPECT_EQ(info.patch.buf.size(), 24u); - RialtoTest::verifyPointFromBuffer(info.patch.buf, actualData[6]); + RialtoTest::verifyPointFromBuffer(info.patch.buf, actualData[5]); - db.readTileInfo(tilesAt2[7], true, info); + db.readTileInfo(names[0], tilesAt2[7], true, info); EXPECT_EQ(info.numPoints, 1u); EXPECT_EQ(info.patch.buf.size(), 24u); RialtoTest::verifyPointFromBuffer(info.patch.buf, actualData[7]); @@ -287,22 +286,21 @@ TEST(RialtoDbWriterTest, testWriter) RialtoDb db(filename, log); db.open(false); - std::vector tileSetIds; std::vector names; - db.readTileSetIds(tileSetIds, names); - uint32_t tileSetId = tileSetIds[0]; + db.readTileSetIds(names); + std::string tileSetName = names[0]; { std::vector ids; - db.queryForTileIds(tileSetId, 0.1, 0.1, 179.9, 89.9, 0, ids); + db.queryForTileIds(tileSetName, 0.1, 0.1, 179.9, 89.9, 0, ids); EXPECT_EQ(ids.size(), 0u); - db.queryForTileIds(tileSetId, 0.1, 0.1, 179.9, 89.9, 1, ids); + db.queryForTileIds(tileSetName, 0.1, 0.1, 179.9, 89.9, 1, ids); EXPECT_EQ(ids.size(), 1u); EXPECT_EQ(ids[0], 7u); - db.queryForTileIds(tileSetId, 0.1, 0.1, 179.9, 89.9, 2, ids); + db.queryForTileIds(tileSetName, 0.1, 0.1, 179.9, 89.9, 2, ids); EXPECT_EQ(ids.size(), 2u); EXPECT_EQ(ids[0], 8u); EXPECT_EQ(ids[1], 9u);