From 018cee1cbdb1d204c5adf7ff4f57cb441d2632a5 Mon Sep 17 00:00:00 2001 From: Howard Butler Date: Sun, 27 Jul 2014 12:12:10 -0500 Subject: [PATCH] working drivers.sqlite.reader with tests --- include/pdal/drivers/sqlite/SQLiteCommon.hpp | 27 ++++-- src/drivers/sqlite/SQLiteIterator.cpp | 8 +- test/data/drivers/sqlite-reader.xml | 22 +++++ test/data/drivers/sqlite-writer.xml | 61 ++++++++++++ test/unit/CMakeLists.txt | 2 +- .../{SQLiteWriterTest.cpp => SQLiteTest.cpp} | 92 +++++++++---------- 6 files changed, 152 insertions(+), 60 deletions(-) create mode 100644 test/data/drivers/sqlite-reader.xml create mode 100644 test/data/drivers/sqlite-writer.xml rename test/unit/drivers/sqlite/{SQLiteWriterTest.cpp => SQLiteTest.cpp} (69%) diff --git a/include/pdal/drivers/sqlite/SQLiteCommon.hpp b/include/pdal/drivers/sqlite/SQLiteCommon.hpp index 3b612a2daf..e46c2d7e0d 100644 --- a/include/pdal/drivers/sqlite/SQLiteCommon.hpp +++ b/include/pdal/drivers/sqlite/SQLiteCommon.hpp @@ -91,7 +91,7 @@ class column std::string data; bool null; - const char * blobBuf; + std::vector blobBuf; std::size_t blobLen; }; @@ -100,7 +100,8 @@ class blob : public column public: blob(const char* buffer, std::size_t size) : column() { - blobBuf = buffer; + blobBuf.resize(size); + std::copy(buffer, buffer+size, blobBuf.begin()); blobLen = size; null = false; @@ -121,7 +122,7 @@ class Patch point_count_t remaining; size_t byte_size; - const char* bytes; + std::vector bytes; }; @@ -147,7 +148,13 @@ class SQLite { if (m_session) + { +#ifdef sqlite3_close_v2 sqlite3_close_v2(m_session); +#else + sqlite3_close(m_session); +#endif + } sqlite3_shutdown(); } @@ -225,7 +232,6 @@ class SQLite m_columns.clear(); m_data.clear(); sqlite3_reset(m_statement); - m_log->get(logDEBUG3) << "reset sqlite3_reset" << std::endl; m_log->get(logDEBUG3) << "Querying '" << query.c_str() <<"'"<< std::endl; @@ -274,8 +280,11 @@ class SQLite if (sqlite3_column_type(m_statement, v) == SQLITE_BLOB) { - c.blobLen = sqlite3_column_bytes(m_statement, v); - c.blobBuf = (char*) sqlite3_column_blob(m_statement, v); + int len = sqlite3_column_bytes(m_statement, v); + const char* buf = (char*) sqlite3_column_blob(m_statement, v); + c.blobLen = len; + c.blobBuf.resize(len); + std::copy(buf, buf+len, c.blobBuf.begin()); } else if (sqlite3_column_type(m_statement, v) == SQLITE_NULL) { c.null = true; @@ -368,7 +377,7 @@ class SQLite else if (c.blobLen != 0) { didBind = sqlite3_bind_blob(m_statement, pos+1, - c.blobBuf, + &(c.blobBuf.front()), static_cast(c.blobLen), SQLITE_STATIC); } @@ -424,6 +433,10 @@ class SQLite #ifdef _WIN32 so_extension = "dll"; +#endif + +#ifndef sqlite3_enable_load_extension +#error "sqlite3_enable_load_extension and spatialite is required for sqlite PDAL support" #endif int code = sqlite3_enable_load_extension(m_session, 1); if (code != SQLITE_OK) diff --git a/src/drivers/sqlite/SQLiteIterator.cpp b/src/drivers/sqlite/SQLiteIterator.cpp index e7f3dcc51a..91c9d68e26 100644 --- a/src/drivers/sqlite/SQLiteIterator.cpp +++ b/src/drivers/sqlite/SQLiteIterator.cpp @@ -338,7 +338,7 @@ point_count_t SQLiteIterator::readPatch(PointBuffer& buffer, point_count_t numPt size_t size = (*r)[position].blobLen; position = columns.find("NUM_POINTS")->second; int32_t count = boost::lexical_cast((*r)[position].data); - m_reader.log()->get(logDEBUG) << "fetched patch with " << count + m_reader.log()->get(logDEBUG4) << "fetched patch with " << count << " points and " << size << " bytes bytesize: " << size << std::endl; m_patch->remaining = count; m_patch->count = count; @@ -350,7 +350,7 @@ point_count_t SQLiteIterator::readPatch(PointBuffer& buffer, point_count_t numPt point_count_t numRead = 0; size_t offset = ((m_patch->count - m_patch->remaining) * m_point_size); - uint8_t *pos = (uint8_t*)m_patch->bytes + offset; + uint8_t *pos = &(m_patch->bytes.front()) + offset; assert(offset <= m_patch->byte_size); while (numRead < numPts && numRemaining > 0) { @@ -373,7 +373,7 @@ point_count_t SQLiteIterator::readImpl(PointBuffer& buffer, point_count_t count) if (atEndImpl()) return 0; - m_reader.log()->get(logDEBUG) << "readBufferImpl called with " + m_reader.log()->get(logDEBUG4) << "readBufferImpl called with " "PointBuffer filled to " << buffer.size() << " points" << std::endl; @@ -398,8 +398,6 @@ point_count_t SQLiteIterator::readImpl(PointBuffer& buffer, point_count_t count) } } PointId bufBegin = buffer.size(); - if (patch_count >= 4 && patch_count < 7) - std::cout << buffer << std::endl; point_count_t numRead = readPatch(buffer, count - totalNumRead); PointId bufEnd = bufBegin + numRead; totalNumRead += numRead; diff --git a/test/data/drivers/sqlite-reader.xml b/test/data/drivers/sqlite-reader.xml new file mode 100644 index 0000000000..2c4a75ef30 --- /dev/null +++ b/test/data/drivers/sqlite-reader.xml @@ -0,0 +1,22 @@ + + + + + + + + + + + diff --git a/test/data/drivers/sqlite-writer.xml b/test/data/drivers/sqlite-writer.xml new file mode 100644 index 0000000000..ff77f001d3 --- /dev/null +++ b/test/data/drivers/sqlite-writer.xml @@ -0,0 +1,61 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/test/unit/CMakeLists.txt b/test/unit/CMakeLists.txt index 458803c362..779e61ce72 100644 --- a/test/unit/CMakeLists.txt +++ b/test/unit/CMakeLists.txt @@ -103,7 +103,7 @@ endif (WITH_ORACLE) if (WITH_SQLITE) if (NOT USE_PDAL_PLUGIN_SQLITE) - set(PDAL_SQLITE_TEST_CPP drivers/sqlite/SqliteWriterTest.cpp) + set(PDAL_SQLITE_TEST_CPP drivers/sqlite/SqliteTest.cpp) FOREACH(file ${PDAL_SQLITE_TEST_CPP}) SET(PDAL_UNITTEST_TEST_SRC "${PDAL_UNITTEST_TEST_SRC};${file}" diff --git a/test/unit/drivers/sqlite/SQLiteWriterTest.cpp b/test/unit/drivers/sqlite/SQLiteTest.cpp similarity index 69% rename from test/unit/drivers/sqlite/SQLiteWriterTest.cpp rename to test/unit/drivers/sqlite/SQLiteTest.cpp index c9eb19d44d..98a0d2edc8 100644 --- a/test/unit/drivers/sqlite/SQLiteWriterTest.cpp +++ b/test/unit/drivers/sqlite/SQLiteTest.cpp @@ -39,8 +39,8 @@ #include #include -#include #include +#include #include #include #include @@ -63,7 +63,7 @@ Options getSQLITEOptions() Option capacity("capacity", chunk_size,"capacity"); options.add(capacity); - Option overwrite("overwrite", false,"overwrite"); + Option overwrite("overwrite", true,"overwrite"); options.add(overwrite); std::string temp_filename(Support::temppath("temp-SqliteWriterTest_test_simple_las.sqlite")); @@ -71,15 +71,15 @@ Options getSQLITEOptions() options.add(connection); Option debug("debug", true, "debug"); - options.add(debug); + // options.add(debug); Option verbose("verbose", 7, "verbose"); - options.add(verbose); + // options.add(verbose); - Option block_table_name("block_table", "PDAL_TEST_BLOCKS", "block_table_name"); + Option block_table_name("block_table_name", "PDAL_TEST_BLOCKS", "block_table_name"); options.add(block_table_name); - Option base_table_name("cloud_table", "PDAL_TEST_BASE" , ""); + Option base_table_name("cloud_table_name", "PDAL_TEST_BASE" , ""); options.add(base_table_name); Option is3d("is3d", false,""); @@ -97,16 +97,12 @@ Options getSQLITEOptions() Option scale_y("scale_y", 0.0000001f, ""); options.add(scale_y); - Option max_cache_blocks("max_cache_blocks", 1, ""); - options.add(max_cache_blocks); - - Option cache_block_size("cache_block_size", capacity.getValue(), ""); - options.add(cache_block_size); - Option filename("filename", Support::datapath("1.2-with-color.las"), ""); options.add(filename); - Option query("query", "SELECT CLOUD FROM PDAL_TEST_BASE where ID=1", ""); + Option query("query", " SELECT b.schema, l.cloud, l.block_id, l.num_points, l.bbox, l.extent, l.points, b.cloud FROM PDAL_TEST_BLOCKS l, PDAL_TEST_BASE b " + "WHERE l.cloud = b.cloud and l.cloud in (1) " + "order by l.cloud", ""); options.add(query); Option a_srs("spatialreference", "EPSG:2926", ""); @@ -114,9 +110,13 @@ Options getSQLITEOptions() Option pack("pack_ignored_fields", true, ""); options.add(pack); - - Option xml_schema_dump("xml_schema_dump", "sqlite-xml-schema-dump.xml", ""); - options.add(xml_schema_dump); + + + Option cloud_column("cloud_column_name", "CLOUD", ""); + options.add(cloud_column); + + // Option xml_schema_dump("xml_schema_dump", "sqlite-xml-schema-dump.xml", ""); + // options.add(xml_schema_dump); Option con_type("type", "sqlite", ""); options.add(con_type); @@ -144,56 +144,54 @@ struct SQLiteTestFixture -BOOST_FIXTURE_TEST_SUITE(SqliteWriterTest, SQLiteTestFixture) +BOOST_FIXTURE_TEST_SUITE(SQLiteTest, SQLiteTestFixture) -BOOST_AUTO_TEST_CASE(SqliteWriterTest_test_simple_las) +BOOST_AUTO_TEST_CASE(SqliteTest_test_simple_las) { #ifdef PDAL_HAVE_SQLITE // remove file from earlier run, if needed std::string temp_filename = getSQLITEOptions().getValueOrThrow("connection"); - FileUtils::deleteFile(temp_filename); pdal::drivers::las::Reader reader(Support::datapath("1.2-with-color.las")); - std::ostream* ofs = FileUtils::createFile(Support::temppath(temp_filename)); - { - pdal::drivers::las::Reader writer_reader(getSQLITEOptions()); - pdal::filters::Chipper writer_chipper(getSQLITEOptions()); - writer_chipper.setInput(&writer_reader); - pdal::filters::InPlaceReprojection writer_reproj(getSQLITEOptions()); - writer_reproj.setInput(&writer_chipper); pdal::drivers::sqlite::SQLiteWriter writer_writer(getSQLITEOptions()); - writer_writer.setInput(&writer_chipper); + writer_writer.setInput(&writer_reader); - // try - // { - PointContext ctx; - writer_writer.prepare(ctx); - boost::uint64_t numPointsToRead = writer_reader.getNumPoints(); - - BOOST_CHECK_EQUAL(numPointsToRead, 1065u); - - writer_writer.execute(ctx); - - // } - // catch (std::runtime_error &) - // { - // FileUtils::closeFile(ofs); - // - // FileUtils::deleteFile(temp_filename); - // return; - // } + PointContext ctx; + writer_writer.prepare(ctx); + boost::uint64_t numPointsToRead = writer_reader.getNumPoints(); + BOOST_CHECK_EQUAL(numPointsToRead, 1065u); + writer_writer.execute(ctx); } + + { + // Read the data - FileUtils::closeFile(ofs); + pdal::drivers::sqlite::SQLiteReader reader(getSQLITEOptions()); + PointContext ctx; + reader.prepare(ctx); + PointBuffer buffer(ctx); - FileUtils::deleteFile(temp_filename); + pdal::StageSequentialIterator* iter = + reader.createSequentialIterator(); + iter->read(buffer); + + Schema *schema = ctx.schema(); + pdal::Dimension const& dimRed = schema->getDimension("Red"); + pdal::Dimension const& dimX = schema->getDimension("X"); + boost::uint16_t r = buffer.getField(dimRed, 0); + BOOST_CHECK_EQUAL(r, 68u); + boost::int32_t x = buffer.getField(dimX, 0); + BOOST_CHECK_EQUAL(x, 63701224); + } + // FileUtils::deleteFile(temp_filename); #endif } + BOOST_AUTO_TEST_SUITE_END()