From d03a6344230cfb99a4e25145c8865413d7d3251e Mon Sep 17 00:00:00 2001 From: "Michael P. Gerlek" Date: Fri, 22 May 2015 15:14:50 -0400 Subject: [PATCH] set SRS and data bounds and tile bounds properly --- plugins/rialto/io/RialtoDb.cpp | 75 ++++++++++++++++++---- plugins/rialto/io/RialtoDb.hpp | 3 + plugins/rialto/io/RialtoDbReader.cpp | 3 + plugins/rialto/io/RialtoDbWriter.cpp | 11 +++- plugins/rialto/io/RialtoDbWriter.hpp | 3 +- plugins/rialto/io/RialtoFileWriter.cpp | 9 +-- plugins/rialto/io/RialtoFileWriter.hpp | 3 +- plugins/rialto/io/RialtoSupport.cpp | 24 +++++-- plugins/rialto/io/RialtoSupport.hpp | 13 ++-- plugins/rialto/test/RialtoDbReaderTest.cpp | 10 +++ plugins/rialto/test/RialtoDbWriterTest.cpp | 9 +++ plugins/rialto/test/RialtoTest.hpp | 6 +- 12 files changed, 132 insertions(+), 37 deletions(-) diff --git a/plugins/rialto/io/RialtoDb.cpp b/plugins/rialto/io/RialtoDb.cpp index 43ab60d617..1fd7421908 100644 --- a/plugins/rialto/io/RialtoDb.cpp +++ b/plugins/rialto/io/RialtoDb.cpp @@ -221,12 +221,12 @@ void RialtoDb::createTableGpkgSpatialRefSys() row r2; row r3; - + const std::string wkt4326 = SpatialReference("EPSG:4326").getWKT(SpatialReference::eCompoundOK); r1.push_back(column("EPSG:4326")); r1.push_back(column(4326)); r1.push_back(column("EPSG")); r1.push_back(column(4326)); - r1.push_back(column("EPSG:4326")); + r1.push_back(column(wkt4326)); r1.push_back(column("EPSG:4326")); rs1.push_back(r1); m_sqlite->insert(data, rs1); @@ -515,10 +515,11 @@ void RialtoDb::readTileTable(std::string const& name, TileTableInfo& info) const e_tileTablesRead.start(); std::string datetime; + uint32_t srs_id; double data_min_x, data_min_y, data_max_x, data_max_y; { std::ostringstream oss; - oss << "SELECT last_change,data_min_x, data_min_y, data_max_x, data_max_y " + oss << "SELECT last_change, data_min_x, data_min_y, data_max_x, data_max_y, srs_id " << "FROM gpkg_contents WHERE table_name='" << name << "'"; log()->get(LogLevel::Debug) << "SELECT for tile set" << std::endl; @@ -533,6 +534,7 @@ void RialtoDb::readTileTable(std::string const& name, TileTableInfo& info) const data_min_y = boost::lexical_cast(r->at(2).data); data_max_x = boost::lexical_cast(r->at(3).data); data_max_y = boost::lexical_cast(r->at(4).data); + srs_id = boost::lexical_cast(r->at(5).data); assert(!m_sqlite->next()); } @@ -556,6 +558,8 @@ void RialtoDb::readTileTable(std::string const& name, TileTableInfo& info) const assert(!m_sqlite->next()); } + std::string wkt = querySrsWkt(srs_id); + uint32_t maxLevel; { std::ostringstream oss; @@ -587,7 +591,7 @@ void RialtoDb::readTileTable(std::string const& name, TileTableInfo& info) const assert(!m_sqlite->next()); } - info.set(datetime, name, maxLevel, numDimensions, + info.set(datetime, name, maxLevel, numDimensions, wkt, data_min_x, data_min_y, data_max_x, data_max_y, tmset_min_x, tmset_min_y, tmset_max_x, tmset_max_y); @@ -724,11 +728,14 @@ void RialtoDb::writeTileTable(const TileTableInfo& data) createTableGpkgPctile(data.getName()); } + + const uint32_t srs_id = querySrsId(data.getWkt()); + { const std::string sql = "INSERT INTO gpkg_contents" - " (table_name, data_type, identifier, description, last_change," - " data_min_x, data_min_y, data_max_x, data_max_y, srs_id)" + " (table_name, data_type, identifier, description, last_change, srs_id," + " data_min_x, data_min_y, data_max_x, data_max_y)" " VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"; records rs; @@ -739,7 +746,7 @@ void RialtoDb::writeTileTable(const TileTableInfo& data) r.push_back(column(data.getName())); // identifier r.push_back(column(data.getName())); // description r.push_back(column(data.getDateTime())); // last_change - r.push_back(column(4326)); // srs_id + r.push_back(column(srs_id)); r.push_back(column(data.getDataMinX())); r.push_back(column(data.getDataMinY())); r.push_back(column(data.getDataMaxX())); @@ -761,11 +768,11 @@ void RialtoDb::writeTileTable(const TileTableInfo& data) row r; r.push_back(column(data.getName())); // table_name - r.push_back(column(4326)); // srs_id - r.push_back(column(data.getTmsetMinX())); - r.push_back(column(data.getTmsetMinX())); - r.push_back(column(data.getTmsetMinX())); + r.push_back(column(srs_id)); r.push_back(column(data.getTmsetMinX())); + r.push_back(column(data.getTmsetMinY())); + r.push_back(column(data.getTmsetMaxX())); + r.push_back(column(data.getTmsetMaxY())); rs.push_back(r); @@ -775,7 +782,7 @@ void RialtoDb::writeTileTable(const TileTableInfo& data) writeDimensions(data); writeMetadata(data); - + e_tileTablesWritten.stop(); } @@ -937,6 +944,50 @@ void RialtoDb::writeTile(const std::string& tileTableName, const TileInfo& data) } +uint32_t RialtoDb::querySrsId(const std::string& wkt) const +{ + if (!m_sqlite) + { + throw pdal_error("RialtoDB: invalid state (session does exist)"); + } + + // TODO: how better to look up SRS than text match of WKT? + + std::ostringstream oss; + oss << "SELECT srs_id FROM gpkg_spatial_ref_sys" + << " WHERE definition='" << wkt << "'"; + + m_sqlite->query(oss.str()); + + const row* r = m_sqlite->get(); + if (!r) return 0; + + // take the first one, ignore any others + const uint32_t srs_id = boost::lexical_cast(r->at(0).data); + + return srs_id; +} + + +std::string RialtoDb::querySrsWkt(uint32_t srs_id) const +{ + std::ostringstream oss; + oss << "SELECT definition " + << "FROM gpkg_spatial_ref_sys WHERE srs_id=" << srs_id; + + log()->get(LogLevel::Debug) << "SELECT for tile set" << std::endl; + + m_sqlite->query(oss.str()); + + // should get exactly one row back + const row* r = m_sqlite->get(); + assert(r); + const std::string wkt = r->at(0).data; + assert(!m_sqlite->next()); + return wkt; +} + + void RialtoDb::xyPointToTileColRow(double x, double y, uint32_t level, uint32_t& col, uint32_t& row) { if (x>=180.0) x = -180.0; diff --git a/plugins/rialto/io/RialtoDb.hpp b/plugins/rialto/io/RialtoDb.hpp index 3086717687..7a07d073ef 100644 --- a/plugins/rialto/io/RialtoDb.hpp +++ b/plugins/rialto/io/RialtoDb.hpp @@ -100,6 +100,9 @@ class PDAL_DLL RialtoDb bool queryForTiles_step(TileInfo& tileInfo); bool queryForTiles_next(); + uint32_t querySrsId(const std::string& wkt) const; + std::string querySrsWkt(uint32_t srs_id) const; + // fills in the dimensions of an otherwise empty layout with // the dimension information from the tile set void setupLayout(const TileTableInfo& tileTableInfo, PointLayoutPtr layout) const; diff --git a/plugins/rialto/io/RialtoDbReader.cpp b/plugins/rialto/io/RialtoDbReader.cpp index 533c830eda..5b4d0476d8 100644 --- a/plugins/rialto/io/RialtoDbReader.cpp +++ b/plugins/rialto/io/RialtoDbReader.cpp @@ -85,6 +85,9 @@ void RialtoDbReader::initialize() m_tileTableInfo = std::unique_ptr(new TileTableInfo()); m_db->readTileTable(m_tileTableName, *m_tileTableInfo); + + SpatialReference srs(m_tileTableInfo->getWkt()); // TODO: or do I need to call setWKT()? + setSpatialReference(srs); } } diff --git a/plugins/rialto/io/RialtoDbWriter.cpp b/plugins/rialto/io/RialtoDbWriter.cpp index 68b1f0f3b1..4252a586d7 100644 --- a/plugins/rialto/io/RialtoDbWriter.cpp +++ b/plugins/rialto/io/RialtoDbWriter.cpp @@ -65,7 +65,11 @@ void RialtoDbWriter::ready(PointTableRef table) m_assister.m_rialtoDb = m_rialtoDb; - m_assister.ready(table); + const SpatialReference& srs = getSpatialReference().empty() ? + table.spatialRef() : getSpatialReference(); + setSpatialReference(srs); + + m_assister.ready(table, getSpatialReference()); } @@ -115,9 +119,10 @@ Options RialtoDbWriter::getDefaultOptions() void DbWriterAssister::writeHeader(const std::string& tileTableName, MetadataNode tileTableNode, - PointLayoutPtr layout, const std::string& datetime) + PointLayoutPtr layout, const std::string& datetime, + const SpatialReference& srs) { - const TileTableInfo tileTableInfo(tileTableName, tileTableNode, layout, datetime); + const TileTableInfo tileTableInfo(tileTableName, tileTableNode, layout, datetime, srs); m_rialtoDb->writeTileTable(tileTableInfo); } diff --git a/plugins/rialto/io/RialtoDbWriter.hpp b/plugins/rialto/io/RialtoDbWriter.hpp index 6fbb899dbc..1659e48655 100644 --- a/plugins/rialto/io/RialtoDbWriter.hpp +++ b/plugins/rialto/io/RialtoDbWriter.hpp @@ -57,7 +57,8 @@ class DbWriterAssister: public WriterAssister virtual void writeHeader(const std::string& tileTableName, MetadataNode tileTableNode, PointLayoutPtr layout, - const std::string& datetime); + const std::string& datetime, + const SpatialReference& srs); virtual void writeTile(const std::string& tileTableName, PointView*, uint32_t level, uint32_t col, uint32_t row, uint32_t mask); }; diff --git a/plugins/rialto/io/RialtoFileWriter.cpp b/plugins/rialto/io/RialtoFileWriter.cpp index 4b6752b990..bf7a2ef76d 100644 --- a/plugins/rialto/io/RialtoFileWriter.cpp +++ b/plugins/rialto/io/RialtoFileWriter.cpp @@ -68,7 +68,7 @@ void RialtoFileWriter::ready(PointTableRef table) m_assister.m_directory = m_filename; - m_assister.ready(table); + m_assister.ready(table, getSpatialReference()); } @@ -115,9 +115,10 @@ Options RialtoFileWriter::getDefaultOptions() void FileWriterAssister::writeHeader(const std::string& tileTableName, MetadataNode tileTableNode, PointLayoutPtr layout, - const std::string& datetime) -{ - const TileTableInfo tileTableInfo(tileTableName, tileTableNode, layout, datetime); + const std::string& datetime, + const SpatialReference& srs) +{ + const TileTableInfo tileTableInfo(tileTableName, tileTableNode, layout, datetime, srs); const std::string filename(m_directory + "/header.json"); FILE* fp = fopen(filename.c_str(), "wt"); diff --git a/plugins/rialto/io/RialtoFileWriter.hpp b/plugins/rialto/io/RialtoFileWriter.hpp index 24c988ef63..90d9394d3c 100644 --- a/plugins/rialto/io/RialtoFileWriter.hpp +++ b/plugins/rialto/io/RialtoFileWriter.hpp @@ -55,7 +55,8 @@ class FileWriterAssister: public WriterAssister virtual void writeHeader(const std::string& tileTableName, MetadataNode tileTableNode, PointLayoutPtr layout, - const std::string& datetime); + const std::string& datetime, + const SpatialReference& srs); virtual void writeTile(const std::string& tileTableName, PointView*, uint32_t level, uint32_t col, uint32_t row, uint32_t mask); }; diff --git a/plugins/rialto/io/RialtoSupport.cpp b/plugins/rialto/io/RialtoSupport.cpp index e197b78bf4..45c5ff4e40 100644 --- a/plugins/rialto/io/RialtoSupport.cpp +++ b/plugins/rialto/io/RialtoSupport.cpp @@ -93,7 +93,8 @@ static void extractStatistics(MetadataNode& tileTableNode, const std::string& di TileTableInfo::TileTableInfo(const std::string& tileTableName, MetadataNode tileTableNode, PointLayoutPtr layout, - const std::string& datetime) + const std::string& datetime, + const SpatialReference& srs) { m_name = tileTableName; @@ -104,15 +105,22 @@ TileTableInfo::TileTableInfo(const std::string& tileTableName, m_maxLevel = getMetadataU32(headerNode, "maxLevel"); m_numDimensions = layout->dims().size(); + m_wkt = srs.getWKT(SpatialReference::eCompoundOK); + m_tmset_min_x = -180.0; m_tmset_min_y = -90.0; m_tmset_max_x = 180.0; m_tmset_max_y = 90.0; - m_data_min_x = -189.0; // TODO - m_data_min_y = -89.0; - m_data_max_x = 179.0; - m_data_max_y = 89.0; + double statMinX, statMeanX, statMaxX; + double statMinY, statMeanY, statMaxY; + extractStatistics(tileTableNode, "X", statMinX, statMeanX, statMaxX); + extractStatistics(tileTableNode, "Y", statMinY, statMeanY, statMaxY); + + m_data_min_x = statMinX; // TODO + m_data_min_y = statMinY; + m_data_max_x = statMaxX; + m_data_max_y = statMaxY; DimensionInfo::importVector(tileTableNode, layout, m_dimensions); } @@ -122,6 +130,7 @@ void TileTableInfo::set(const std::string& datetime, const std::string& name, uint32_t maxLevel, uint32_t numDimensions, + const std::string& wkt, double data_min_x, double data_min_y, double data_max_x, @@ -135,6 +144,7 @@ void TileTableInfo::set(const std::string& datetime, m_name = name; m_maxLevel = maxLevel; m_numDimensions = numDimensions; + m_wkt = wkt; m_data_min_x = data_min_x; m_data_min_y = data_min_y; m_data_max_x = data_max_x; @@ -312,7 +322,7 @@ void WriterAssister::setTileTableName(const std::string& tileTableName) } -void WriterAssister::ready(PointTableRef table) +void WriterAssister::ready(PointTableRef table, const SpatialReference& srs) { m_tileTableNode = table.metadata().findChild("filters.tiler"); if (!m_tileTableNode.valid()) { @@ -325,7 +335,7 @@ void WriterAssister::ready(PointTableRef table) // TODO: this produces "ss", not "ss.sss" as the gpkg spec implies is required strftime(buf, sizeof(buf), "%FT%TZ", gmtime(&now)); std::string datetime(buf); - writeHeader(m_tileTableName, m_tileTableNode, table.layout(), datetime); + writeHeader(m_tileTableName, m_tileTableNode, table.layout(), datetime, srs); makePointViewMap(); } diff --git a/plugins/rialto/io/RialtoSupport.hpp b/plugins/rialto/io/RialtoSupport.hpp index 95a9ca5037..fec3954ccc 100644 --- a/plugins/rialto/io/RialtoSupport.hpp +++ b/plugins/rialto/io/RialtoSupport.hpp @@ -109,12 +109,14 @@ class TileTableInfo TileTableInfo(const std::string& tileTableName, MetadataNode tileTableNode, PointLayoutPtr layout, - const std::string& datetime); + const std::string& datetime, + const SpatialReference& srs); void set(const std::string& datetime, const std::string& name, uint32_t maxLevel, uint32_t numDimensions, + const std::string& wkt, double data_min_x, double data_min_y, double data_max_x, @@ -130,7 +132,8 @@ class TileTableInfo uint32_t getNumDimensions() const { return m_numDimensions; } const std::vector& getDimensions() const { return m_dimensions; }; std::vector& getDimensionsRef() { return m_dimensions; }; - + const std::string getWkt() const { return m_wkt; } + double getDataMinX() const { return m_data_min_x; } // data extents double getDataMinY() const { return m_data_min_y; } double getDataMaxX() const { return m_data_max_x; } @@ -147,6 +150,7 @@ class TileTableInfo uint32_t m_maxLevel; uint32_t m_numDimensions; std::vector m_dimensions; + std::string m_wkt; // the srs double m_data_min_x; // data extents double m_data_min_y; double m_data_max_x; @@ -195,14 +199,15 @@ class WriterAssister void setTileTableName(const std::string&); void write(const PointViewPtr viewPtr); - void ready(PointTableRef table); + void ready(PointTableRef table, const SpatialReference& srs); void done(); protected: virtual void writeHeader(const std::string& tileTableName, MetadataNode tileTableNode, PointLayoutPtr layout, - const std::string& datetime)=0; + const std::string& datetime, + const SpatialReference& srs)=0; virtual void writeTile(const std::string& tileTableName, PointView*, uint32_t level, uint32_t col, uint32_t row, uint32_t mask)=0; diff --git a/plugins/rialto/test/RialtoDbReaderTest.cpp b/plugins/rialto/test/RialtoDbReaderTest.cpp index 690e98352a..1c87c50c4e 100644 --- a/plugins/rialto/test/RialtoDbReaderTest.cpp +++ b/plugins/rialto/test/RialtoDbReaderTest.cpp @@ -73,6 +73,16 @@ TEST(RialtoDbReaderTest, test) options.add("filename", filename); //options.add("verbose", LogLevel::Debug); reader.setOptions(options); + + { + PointTable table; + reader.prepare(table); + const SpatialReference& srs = reader.getSpatialReference(); + const std::string& wkt = srs.getWKT(); + const SpatialReference srs4326("EPSG:4326"); + const std::string wkt4326 = srs4326.getWKT(); + EXPECT_EQ(wkt, wkt4326); + } { PointTable table; diff --git a/plugins/rialto/test/RialtoDbWriterTest.cpp b/plugins/rialto/test/RialtoDbWriterTest.cpp index f181305c58..a287e52f93 100644 --- a/plugins/rialto/test/RialtoDbWriterTest.cpp +++ b/plugins/rialto/test/RialtoDbWriterTest.cpp @@ -83,6 +83,15 @@ void verifyDatabase(const std::string& filename, RialtoTest::Data* actualData) EXPECT_EQ(tileTableInfo.getMaxLevel(), 2u); EXPECT_EQ(tileTableInfo.getNumDimensions(), 3u); + EXPECT_DOUBLE_EQ(tileTableInfo.getDataMinX(), -179.0); + EXPECT_DOUBLE_EQ(tileTableInfo.getDataMinY(), -89.0); + EXPECT_DOUBLE_EQ(tileTableInfo.getDataMaxX(), 91.0); + EXPECT_DOUBLE_EQ(tileTableInfo.getDataMaxY(), 89.0); + EXPECT_DOUBLE_EQ(tileTableInfo.getTmsetMinX(), -180.0); + EXPECT_DOUBLE_EQ(tileTableInfo.getTmsetMinY(), -90.0); + EXPECT_DOUBLE_EQ(tileTableInfo.getTmsetMaxX(), 180.0); + EXPECT_DOUBLE_EQ(tileTableInfo.getTmsetMaxY(), 90.0); + const std::vector& dimensionsInfo = tileTableInfo.getDimensions(); EXPECT_EQ(dimensionsInfo[0].getName(), "X"); EXPECT_EQ(dimensionsInfo[0].getDataType(), "double"); diff --git a/plugins/rialto/test/RialtoTest.hpp b/plugins/rialto/test/RialtoTest.hpp index b501f9bc7c..0d11dba3e6 100644 --- a/plugins/rialto/test/RialtoTest.hpp +++ b/plugins/rialto/test/RialtoTest.hpp @@ -177,7 +177,7 @@ void RialtoTest::createDatabase(pdal::PointTable& table, pdal::PointViewPtr view, const std::string& filename, uint32_t maxLevel) -{ +{ pdal::Options readerOptions; pdal::BufferReader reader; reader.setOptions(readerOptions); @@ -197,14 +197,10 @@ void RialtoTest::createDatabase(pdal::PointTable& table, pdal::StageFactory f; pdal::Options writerOptions; -#if 1 writerOptions.add("filename", filename); writerOptions.add("overwrite", true); //writerOptions.add("verbose", LogLevel::Debug); pdal::Stage* writer = f.createStage("writers.rialtodb"); -#else - pdal::Stage* writer = f.createStage("writers.null"); -#endif writer->setOptions(writerOptions); writer->setInput(tiler);