Skip to content

Commit

Permalink
working drivers.sqlite.reader with tests
Browse files Browse the repository at this point in the history
  • Loading branch information
@hobu
hobu committed Jul 27, 2014
1 parent d9ce4f8 commit 018cee1
Show file tree
Hide file tree
Showing 6 changed files with 152 additions and 60 deletions.
27 changes: 20 additions & 7 deletions include/pdal/drivers/sqlite/SQLiteCommon.hpp
View file
Expand Up @@ -91,7 +91,7 @@ class column

std::string data;
bool null;
const char * blobBuf;
std::vector<uint8_t> blobBuf;
std::size_t blobLen;
};

Expand All @@ -100,7 +100,8 @@ class blob : public column
public:
blob(const char* buffer, std::size_t size) : column()
{
blobBuf = buffer;
blobBuf.resize(size);
std::copy(buffer, buffer+size, blobBuf.begin());
blobLen = size;
null = false;

Expand All @@ -121,7 +122,7 @@ class Patch
point_count_t remaining;

size_t byte_size;
const char* bytes;
std::vector<uint8_t> bytes;

};

Expand All @@ -147,7 +148,13 @@ class SQLite
{

if (m_session)
{
#ifdef sqlite3_close_v2
sqlite3_close_v2(m_session);
#else
sqlite3_close(m_session);
#endif
}
sqlite3_shutdown();

}
Expand Down Expand Up @@ -225,7 +232,6 @@ class SQLite
m_columns.clear();
m_data.clear();
sqlite3_reset(m_statement);
m_log->get(logDEBUG3) << "reset sqlite3_reset" << std::endl;

m_log->get(logDEBUG3) << "Querying '" << query.c_str() <<"'"<< std::endl;

Expand Down Expand Up @@ -274,8 +280,11 @@ class SQLite

if (sqlite3_column_type(m_statement, v) == SQLITE_BLOB)
{
c.blobLen = sqlite3_column_bytes(m_statement, v);
c.blobBuf = (char*) sqlite3_column_blob(m_statement, v);
int len = sqlite3_column_bytes(m_statement, v);
const char* buf = (char*) sqlite3_column_blob(m_statement, v);
c.blobLen = len;
c.blobBuf.resize(len);
std::copy(buf, buf+len, c.blobBuf.begin());
} else if (sqlite3_column_type(m_statement, v) == SQLITE_NULL)
{
c.null = true;
Expand Down Expand Up @@ -368,7 +377,7 @@ class SQLite
else if (c.blobLen != 0)
{
didBind = sqlite3_bind_blob(m_statement, pos+1,
c.blobBuf,
&(c.blobBuf.front()),
static_cast<int>(c.blobLen),
SQLITE_STATIC);
}
Expand Down Expand Up @@ -424,6 +433,10 @@ class SQLite

#ifdef _WIN32
so_extension = "dll";
#endif

#ifndef sqlite3_enable_load_extension
#error "sqlite3_enable_load_extension and spatialite is required for sqlite PDAL support"
#endif
int code = sqlite3_enable_load_extension(m_session, 1);
if (code != SQLITE_OK)
Expand Down
8 changes: 3 additions & 5 deletions src/drivers/sqlite/SQLiteIterator.cpp
View file
Expand Up @@ -338,7 +338,7 @@ point_count_t SQLiteIterator::readPatch(PointBuffer& buffer, point_count_t numPt
size_t size = (*r)[position].blobLen;
position = columns.find("NUM_POINTS")->second;
int32_t count = boost::lexical_cast<int32_t>((*r)[position].data);
m_reader.log()->get(logDEBUG) << "fetched patch with " << count
m_reader.log()->get(logDEBUG4) << "fetched patch with " << count
<< " points and " << size << " bytes bytesize: " << size << std::endl;
m_patch->remaining = count;
m_patch->count = count;
Expand All @@ -350,7 +350,7 @@ point_count_t SQLiteIterator::readPatch(PointBuffer& buffer, point_count_t numPt
point_count_t numRead = 0;

size_t offset = ((m_patch->count - m_patch->remaining) * m_point_size);
uint8_t *pos = (uint8_t*)m_patch->bytes + offset;
uint8_t *pos = &(m_patch->bytes.front()) + offset;
assert(offset <= m_patch->byte_size);
while (numRead < numPts && numRemaining > 0)
{
Expand All @@ -373,7 +373,7 @@ point_count_t SQLiteIterator::readImpl(PointBuffer& buffer, point_count_t count)
if (atEndImpl())
return 0;

m_reader.log()->get(logDEBUG) << "readBufferImpl called with "
m_reader.log()->get(logDEBUG4) << "readBufferImpl called with "
"PointBuffer filled to " << buffer.size() << " points" <<
std::endl;

Expand All @@ -398,8 +398,6 @@ point_count_t SQLiteIterator::readImpl(PointBuffer& buffer, point_count_t count)
}
}
PointId bufBegin = buffer.size();
if (patch_count >= 4 && patch_count < 7)
std::cout << buffer << std::endl;
point_count_t numRead = readPatch(buffer, count - totalNumRead);
PointId bufEnd = bufBegin + numRead;
totalNumRead += numRead;
Expand Down
22 changes: 22 additions & 0 deletions test/data/drivers/sqlite-reader.xml
View file
@@ -0,0 +1,22 @@
<?xml version="1.0" encoding="utf-8"?>
<Pipeline version="1.0">
<Writer type="drivers.las.writer">
<Option name="filename">
written-from-sqlite.las
</Option>
<Option name="compression">
false
</Option>
<Reader type="drivers.sqlite.reader">
<Option name="query">
SELECT b.schema, l.cloud, l.block_id, l.num_points, l.bbox, l.extent, l.points, b.cloud
FROM simple_blocks l, simple_cloud b
WHERE l.cloud = b.cloud and l.cloud in (1)
order by l.cloud
</Option>
<Option name="connection">
/Users/hobu/dev/git/pdal/test/data/autzen-dd.sqlite
</Option>
</Reader>
</Writer>
</Pipeline>
61 changes: 61 additions & 0 deletions test/data/drivers/sqlite-writer.xml
View file
@@ -0,0 +1,61 @@
<?xml version="1.0" encoding="utf-8"?>
<Pipeline version="1.0">
<Writer type="drivers.sqlite.writer">
<Option name="connection">
autzen-dd.sqlite
</Option>
<Option name="cloud_table_name">
SIMPLE_CLOUD
</Option>
<Option name="block_table_name">
SIMPLE_BLOCKS
</Option>

<Option name="cloud_column_name">
CLOUD
</Option>
<Option name="is3d">
false
</Option>
<Option name="overwrite">
true
</Option>
<Option name="srid">
4326
</Option>
<Option name="base_table_boundary_wkt">
</Option>
<Option name="pre_block_sql">
</Option>
<Option name="pre_sql">
</Option>
<Option name="post_block_sql">
</Option>
<Option name="capacity">
50
</Option>
<Option name="stream_output_precision">
8
</Option>
<Option name="pack_ignored_fields">
true
</Option>

<Filter type="filters.chipper">
<Option name="capacity">
50
</Option>
<Option name="max_cache_blocks">
1
</Option>
<Reader type="drivers.las.reader">
<Option name="filename">
../autzen-dd.las
</Option>
<Option name="spatialreference">
EPSG:4326
</Option>
</Reader>
</Filter>
</Writer>
</Pipeline>
2 changes: 1 addition & 1 deletion test/unit/CMakeLists.txt
View file
Expand Up @@ -103,7 +103,7 @@ endif (WITH_ORACLE)

if (WITH_SQLITE)
if (NOT USE_PDAL_PLUGIN_SQLITE)
set(PDAL_SQLITE_TEST_CPP drivers/sqlite/SqliteWriterTest.cpp)
set(PDAL_SQLITE_TEST_CPP drivers/sqlite/SqliteTest.cpp)

FOREACH(file ${PDAL_SQLITE_TEST_CPP})
SET(PDAL_UNITTEST_TEST_SRC "${PDAL_UNITTEST_TEST_SRC};${file}"
Expand Down
92 changes: 45 additions & 47 deletions .../unit/drivers/sqlite/SQLiteWriterTest.cpp → test/unit/drivers/sqlite/SQLiteTest.cpp
View file
Expand Up @@ -39,8 +39,8 @@
#include <boost/concept_check.hpp>

#include <pdal/FileUtils.hpp>
#include <pdal/drivers/faux/Reader.hpp>
#include <pdal/drivers/sqlite/SQLiteWriter.hpp>
#include <pdal/drivers/sqlite/SQLiteReader.hpp>
#include <pdal/drivers/las/Reader.hpp>
#include <pdal/filters/Cache.hpp>
#include <pdal/filters/Chipper.hpp>
Expand All @@ -63,23 +63,23 @@ Options getSQLITEOptions()
Option capacity("capacity", chunk_size,"capacity");
options.add(capacity);

Option overwrite("overwrite", false,"overwrite");
Option overwrite("overwrite", true,"overwrite");
options.add(overwrite);

std::string temp_filename(Support::temppath("temp-SqliteWriterTest_test_simple_las.sqlite"));
Option connection("connection",temp_filename, "connection");
options.add(connection);

Option debug("debug", true, "debug");
options.add(debug);
// options.add(debug);

Option verbose("verbose", 7, "verbose");
options.add(verbose);
// options.add(verbose);

Option block_table_name("block_table", "PDAL_TEST_BLOCKS", "block_table_name");
Option block_table_name("block_table_name", "PDAL_TEST_BLOCKS", "block_table_name");
options.add(block_table_name);

Option base_table_name("cloud_table", "PDAL_TEST_BASE" , "");
Option base_table_name("cloud_table_name", "PDAL_TEST_BASE" , "");
options.add(base_table_name);

Option is3d("is3d", false,"");
Expand All @@ -97,26 +97,26 @@ Options getSQLITEOptions()
Option scale_y("scale_y", 0.0000001f, "");
options.add(scale_y);

Option max_cache_blocks("max_cache_blocks", 1, "");
options.add(max_cache_blocks);

Option cache_block_size("cache_block_size", capacity.getValue<boost::uint32_t>(), "");
options.add(cache_block_size);

Option filename("filename", Support::datapath("1.2-with-color.las"), "");
options.add(filename);

Option query("query", "SELECT CLOUD FROM PDAL_TEST_BASE where ID=1", "");
Option query("query", " SELECT b.schema, l.cloud, l.block_id, l.num_points, l.bbox, l.extent, l.points, b.cloud FROM PDAL_TEST_BLOCKS l, PDAL_TEST_BASE b "
"WHERE l.cloud = b.cloud and l.cloud in (1) "
"order by l.cloud", "");
options.add(query);

Option a_srs("spatialreference", "EPSG:2926", "");
options.add(a_srs);

Option pack("pack_ignored_fields", true, "");
options.add(pack);

Option xml_schema_dump("xml_schema_dump", "sqlite-xml-schema-dump.xml", "");
options.add(xml_schema_dump);


Option cloud_column("cloud_column_name", "CLOUD", "");
options.add(cloud_column);

// Option xml_schema_dump("xml_schema_dump", "sqlite-xml-schema-dump.xml", "");
// options.add(xml_schema_dump);

Option con_type("type", "sqlite", "");
options.add(con_type);
Expand Down Expand Up @@ -144,56 +144,54 @@ struct SQLiteTestFixture



BOOST_FIXTURE_TEST_SUITE(SqliteWriterTest, SQLiteTestFixture)
BOOST_FIXTURE_TEST_SUITE(SQLiteTest, SQLiteTestFixture)

BOOST_AUTO_TEST_CASE(SqliteWriterTest_test_simple_las)
BOOST_AUTO_TEST_CASE(SqliteTest_test_simple_las)
{
#ifdef PDAL_HAVE_SQLITE
// remove file from earlier run, if needed
std::string temp_filename = getSQLITEOptions().getValueOrThrow<std::string>("connection");
FileUtils::deleteFile(temp_filename);

pdal::drivers::las::Reader reader(Support::datapath("1.2-with-color.las"));

std::ostream* ofs = FileUtils::createFile(Support::temppath(temp_filename));

{

pdal::drivers::las::Reader writer_reader(getSQLITEOptions());
pdal::filters::Chipper writer_chipper(getSQLITEOptions());
writer_chipper.setInput(&writer_reader);
pdal::filters::InPlaceReprojection writer_reproj(getSQLITEOptions());
writer_reproj.setInput(&writer_chipper);
pdal::drivers::sqlite::SQLiteWriter writer_writer(getSQLITEOptions());
writer_writer.setInput(&writer_chipper);
writer_writer.setInput(&writer_reader);

// try
// {
PointContext ctx;
writer_writer.prepare(ctx);
boost::uint64_t numPointsToRead = writer_reader.getNumPoints();

BOOST_CHECK_EQUAL(numPointsToRead, 1065u);

writer_writer.execute(ctx);

// }
// catch (std::runtime_error &)
// {
// FileUtils::closeFile(ofs);
//
// FileUtils::deleteFile(temp_filename);
// return;
// }
PointContext ctx;
writer_writer.prepare(ctx);
boost::uint64_t numPointsToRead = writer_reader.getNumPoints();

BOOST_CHECK_EQUAL(numPointsToRead, 1065u);

writer_writer.execute(ctx);
}

{
// Read the data

FileUtils::closeFile(ofs);
pdal::drivers::sqlite::SQLiteReader reader(getSQLITEOptions());
PointContext ctx;
reader.prepare(ctx);
PointBuffer buffer(ctx);

FileUtils::deleteFile(temp_filename);
pdal::StageSequentialIterator* iter =
reader.createSequentialIterator();
iter->read(buffer);

Schema *schema = ctx.schema();
pdal::Dimension const& dimRed = schema->getDimension("Red");
pdal::Dimension const& dimX = schema->getDimension("X");
boost::uint16_t r = buffer.getField<boost::uint16_t>(dimRed, 0);
BOOST_CHECK_EQUAL(r, 68u);
boost::int32_t x = buffer.getField<boost::int32_t>(dimX, 0);
BOOST_CHECK_EQUAL(x, 63701224);
}
// FileUtils::deleteFile(temp_filename);
#endif
}



BOOST_AUTO_TEST_SUITE_END()

0 comments on commit 018cee1

Please sign in to comment.