From 6bf31f7c0f9cd1ee2093c2d8653051c9f804ad51 Mon Sep 17 00:00:00 2001 From: Paul Ramsey Date: Thu, 9 May 2013 06:36:11 -0700 Subject: [PATCH] Working writer for pgpointcloud --- include/pdal/drivers/pgpointcloud/Writer.hpp | 78 +- include/pdal/drivers/pgpointcloud/common.hpp | 111 +-- src/StageFactory.cpp | 4 + src/drivers/pgpointcloud/Writer.cpp | 935 +++++++------------ 4 files changed, 406 insertions(+), 722 deletions(-) diff --git a/include/pdal/drivers/pgpointcloud/Writer.hpp b/include/pdal/drivers/pgpointcloud/Writer.hpp index 7882d9f3d3..5f7c26125b 100644 --- a/include/pdal/drivers/pgpointcloud/Writer.hpp +++ b/include/pdal/drivers/pgpointcloud/Writer.hpp @@ -52,7 +52,7 @@ namespace pgpointcloud class PDAL_DLL Writer : public pdal::Writer { public: - SET_STAGE_NAME("drivers.pgpointcloud.writer", "Database Writer") + SET_STAGE_NAME("drivers.pgpointcloud.writer", "PostgresSQL Pointcloud Database Writer") Writer(Stage& prevStage, const Options&); ~Writer(); @@ -60,12 +60,9 @@ class PDAL_DLL Writer : public pdal::Writer virtual void initialize(); static Options getDefaultOptions(); - - protected: virtual void writeBegin(boost::uint64_t targetNumPointsToWrite); virtual void writeBufferBegin(PointBuffer const&); - virtual boost::uint32_t writeBuffer(const PointBuffer&); virtual void writeEnd(boost::uint64_t actualNumPointsWritten); @@ -74,53 +71,52 @@ class PDAL_DLL Writer : public pdal::Writer Writer& operator=(const Writer&); // not implemented Writer(const Writer&); // not implemented - void CreateBlockTable(std::string const& name, boost::uint32_t srid); - void CreateCloudTable(std::string const& name, boost::uint32_t srid); bool CheckTableExists(std::string const& name); - void DeleteBlockTable(std::string const& cloud_table_name, - std::string const& cloud_column_name, - std::string const& block_table_name); - void DeleteCloudTable(std::string const& cloud_table_name, - std::string const& cloud_column_name); - void CreateIndexes(std::string const& table_name, - std::string const& spatial_column_name, - bool is3d, - bool isBlockTable=true); - void CreateSDOEntry(std::string const& block_table, - boost::uint32_t srid, - pdal::Bounds bounds, - bool is3d); - Schema getPackedSchema( Schema const& schema) const; - bool IsValidGeometryWKT(std::string const& wkt) const; - std::string loadGeometryWKT(std::string const& filename_or_wkt) const; - void CreateCloud(Schema const& buffer_schema); - - void PackPointData( PointBuffer const& buffer, - boost::uint8_t** point_data, - boost::uint32_t& point_data_len, - boost::uint32_t& schema_byte_size); + bool CheckPointCloudExists(); + bool CheckPostGISExists(); + Schema PackSchema(Schema const& schema) const; + boost::uint32_t SetupSchema(Schema const& buffer_schema, boost::uint32_t srid); + + void CreateTable(std::string const& schema_name, + std::string const& table_name, + std::string const& column_name, + boost::uint32_t pcid); + + void DeleteTable(std::string const& schema_name, + std::string const& table_name); + + void CreateIndex(std::string const& schema_name, + std::string const& table_name, + std::string const& column_name); + + void PackPointData(PointBuffer const& buffer, + boost::uint8_t** point_data, + boost::uint32_t& point_data_len, + boost::uint32_t& schema_byte_size); + bool WriteBlock(PointBuffer const& buffer); #ifdef PDAL_HAVE_SOCI ::soci::session* m_session; - ::soci::statement* m_block_statement; #else void* m_session; #endif - DatabaseType m_type; - bool m_doCreateIndex; - pdal::Bounds m_bounds; // Bounds of the entire point cloud + const pdal::Schema &m_pdal_schema; + std::string m_schema_name; + std::string m_table_name; + std::string m_column_name; + CompressionType m_patch_compression_type; + boost::uint32_t m_patch_capacity; + boost::uint32_t m_srid; + boost::uint32_t m_pcid; + bool m_have_postgis; + bool m_create_index; + bool m_overwrite; + + + // lose this bool m_sdo_pc_is_initialized; - std::ostringstream m_block_insert_query; - std::ostringstream m_block_bytes; - std::string m_block_data; - std::string m_extent; - std::string m_bbox; - boost::int32_t m_obj_id; - boost::int32_t m_block_id; - boost::uint32_t m_srid; - boost::int64_t m_num_points; }; } diff --git a/include/pdal/drivers/pgpointcloud/common.hpp b/include/pdal/drivers/pgpointcloud/common.hpp index 25f6f716d2..2cf123b8e3 100644 --- a/include/pdal/drivers/pgpointcloud/common.hpp +++ b/include/pdal/drivers/pgpointcloud/common.hpp @@ -57,85 +57,54 @@ namespace drivers namespace pgpointcloud { - class soci_driver_error : public pdal_error - { - public: - soci_driver_error(std::string const& msg) - : pdal_error(msg) - {} - }; - - class connection_failed : public soci_driver_error - { - public: - connection_failed(std::string const& msg) - : soci_driver_error(msg) - {} - }; - - class buffer_too_small : public soci_driver_error - { - public: - buffer_too_small(std::string const& msg) - : soci_driver_error(msg) - {} - }; - - - enum DatabaseType - { - DATABASE_POSTGRESQL, - DATABASE_ORACLE, - DATABASE_UNKNOWN = 128 - }; - - enum QueryType - { - QUERY_CLOUD = 0, - QUERY_BLOCKS_PLUS_CLOUD_VIEW, - QUERY_UNKNOWN = 512 - }; - - -inline DatabaseType getDatabaseConnectionType(std::string const& connection_type) +class soci_driver_error : public pdal_error { - DatabaseType output; +public: + soci_driver_error(std::string const& msg) + : pdal_error(msg) + {} +}; - if (boost::iequals(connection_type, "oracle")) - output = DATABASE_ORACLE; - else if (boost::iequals(connection_type, "postgresql")) - output = DATABASE_POSTGRESQL; +class connection_failed : public soci_driver_error +{ +public: + connection_failed(std::string const& msg) + : soci_driver_error(msg) + {} +}; + +class buffer_too_small : public soci_driver_error +{ +public: + buffer_too_small(std::string const& msg) + : soci_driver_error(msg) + {} +}; + + +enum CompressionType +{ + COMPRESSION_NONE = 0, + COMPRESSION_GHT = 1, + COMPRESSION_DIMENSIONAL = 2 +}; + + +inline CompressionType getCompressionType(std::string const& compression_type) +{ + CompressionType output; + + if (boost::iequals(compression_type, "dimensional")) + output = COMPRESSION_DIMENSIONAL; + else if (boost::iequals(compression_type, "ght")) + output = COMPRESSION_GHT; else - output = DATABASE_UNKNOWN; + output = COMPRESSION_NONE; return output; } -inline ::soci::session* connectToDataBase(std::string const& connection, DatabaseType dtype) -{ - ::soci::session* output(0); - if (dtype == DATABASE_UNKNOWN) - { - std::stringstream oss; - oss << "Database connection type '" << dtype << "' is unknown or not configured"; - throw soci_driver_error(oss.str()); - } - - try - { - if (dtype == DATABASE_POSTGRESQL) - output = new ::soci::session(::soci::postgresql, connection); - - } catch (::soci::soci_error const& e) - { - std::stringstream oss; - oss << "Unable to connect to database with error '" << e.what() << "'"; - throw connection_failed(oss.str()); - } - - return output; -} } } diff --git a/src/StageFactory.cpp b/src/StageFactory.cpp index 296d8bdde9..65db5f29c5 100644 --- a/src/StageFactory.cpp +++ b/src/StageFactory.cpp @@ -81,6 +81,8 @@ #ifndef USE_PDAL_PLUGIN_SOCI #include #include +/* #include */ +#include #endif #endif @@ -197,6 +199,7 @@ MAKE_WRITER_CREATOR(P2GWriter, pdal::drivers::p2g::Writer) #ifdef PDAL_HAVE_SOCI #ifndef USE_PDAL_PLUGIN_SOCI +MAKE_WRITER_CREATOR(PgPcWriter, pdal::drivers::pgpointcloud::Writer) MAKE_WRITER_CREATOR(SociWriter, pdal::drivers::soci::Writer) #endif #endif @@ -482,6 +485,7 @@ void StageFactory::registerKnownWriters() #ifdef PDAL_HAVE_SOCI #ifndef USE_PDAL_PLUGIN_SOCI REGISTER_WRITER(SociWriter, pdal::drivers::soci::Writer); + REGISTER_WRITER(PgPcWriter, pdal::drivers::pgpointcloud::Writer); #endif #endif diff --git a/src/drivers/pgpointcloud/Writer.cpp b/src/drivers/pgpointcloud/Writer.cpp index adf6eac4e8..050ff1f23d 100644 --- a/src/drivers/pgpointcloud/Writer.cpp +++ b/src/drivers/pgpointcloud/Writer.cpp @@ -38,21 +38,27 @@ #include #include #include +#include #include +#include #include -#ifdef PDAL_HAVE_GDAL -#include -#include -#endif - #ifdef USE_PDAL_PLUGIN_PGPOINTCLOUD MAKE_WRITER_CREATOR(pgpointcloudWriter, pdal::drivers::pgpointcloud::Writer) CREATE_WRITER_PLUGIN(pgpointcloud, pdal::drivers::pgpointcloud::Writer) #endif +// TO DO: +// - PCID / Schema consistency. If a PCID is specified, +// must it be consistent with the buffer schema? Or should +// the writer shove the data into the database schema as best +// it can? +// - Load information table. Should PDAL write into a metadata +// table information about each load? If so, how to distinguish +// between loads? Leave to pre/post SQL? + namespace pdal { @@ -61,20 +67,24 @@ namespace drivers namespace pgpointcloud { - - - Writer::Writer(Stage& prevStage, const Options& options) : pdal::Writer(prevStage, options) , m_session(0) - , m_block_statement(0) - , m_type(DATABASE_UNKNOWN) - , m_doCreateIndex(false) - , m_bounds(Bounds()) + , m_pdal_schema(prevStage.getSchema()) + , m_schema_name("") + , m_table_name("") + , m_column_name("") + , m_patch_compression_type(COMPRESSION_NONE) + , m_patch_capacity(400) + , m_srid(0) + , m_pcid(0) + , m_have_postgis(false) + , m_create_index(true) + , m_overwrite(true) , m_sdo_pc_is_initialized(false) - { + return; } @@ -85,84 +95,110 @@ Writer::~Writer() } +// +// Called from PDAL core during start-up. Do everything +// here that you are going to absolutely require later. +// Optional things you can defer or attempt to initialize +// here. +// void Writer::initialize() { pdal::Writer::initialize(); + // If we don't know the table name, we're SOL + m_table_name = getOptions().getValueOrThrow("table"); + + // Schema and column name can be defaulted safely + m_column_name = getOptions().getValueOrDefault("column", "pa"); + m_schema_name = getOptions().getValueOrDefault("schema", ""); + + // Read compression type and turn into an integer + std::string compression_str = getOptions().getValueOrDefault("compression", "dimensional"); + m_patch_compression_type = getCompressionType(compression_str); + + // Connection string needs to exist and actually work std::string connection = getOptions().getValueOrDefault("connection", ""); - if (!connection.size()) - { - throw soci_driver_error("unable to connect to database, no connection string was given!"); - } - m_type = getDatabaseConnectionType(getOptions().getValueOrThrow("type")); - if (m_type == DATABASE_UNKNOWN) + // No string, nothing we can do + if ( ! connection.size() ) { - std::stringstream oss; - oss << "Database connection type '" - << getOptions().getValueOrThrow("type") - << "' is unknown or not configured"; - throw soci_driver_error(oss.str()); + throw soci_driver_error("unable to connect to database, no connection string was given!"); } + + // Can we connect, using this string? try { - if (m_type == DATABASE_POSTGRESQL) - m_session = new ::soci::session(::soci::postgresql, connection); - + m_session = new ::soci::session(::soci::postgresql, connection); log()->get(logDEBUG) << "Connected to database" << std::endl; - - } catch (::soci::soci_error const& e) + } + catch (::soci::soci_error const &e) { std::stringstream oss; - oss << "Unable to connect to database with error '" << e.what() << "'"; + oss << "Unable to connect '" << connection << "' with error '" << e.what() << "'"; throw pdal_error(oss.str()); } - + + // Direct database log info to the logger m_session->set_log_stream(&(log()->get(logDEBUG2))); - return; -} + // + // Read other preferences + m_overwrite = getOptions().getValueOrDefault("overwrite", true); + m_patch_capacity = getOptions().getValueOrDefault("capacity", 400); + m_srid = getOptions().getValueOrDefault("srid", 4326); + m_pcid = getOptions().getValueOrDefault("pcid", 0); + return; +} +// +// Called from somewhere (?) in PDAL core presumably to provide a user-friendly +// means of editing the reader options. +// Options Writer::getDefaultOptions() { Options options; + Option table("table", "", "table to write to"); + Option schema("schema", "", "schema table resides in"); + Option column("column", "", "column to write to"); + Option compression("compression", "dimensional", "patch compression format to use (none, dimensional, ght)"); + Option overwrite("overwrite", true, "replace any existing table"); + Option capacity("capacity", 400, "how many points to store in each patch"); + Option srid("srid", 4326, "spatial reference id to store data in"); + Option pcid("pcid", 0, "use this existing pointcloud schema id, if it exists"); + Option pre_sql("pre_sql", "", "before the pipeline runs, read and execute this SQL file, or run this SQL command"); + Option post_sql("post_sql", "", "after the pipeline runs, read and execute this SQL file, or run this SQL command"); + + options.add(table); + options.add(schema); + options.add(column); + options.add(compression); + options.add(overwrite); + options.add(capacity); + options.add(srid); + options.add(pcid); + options.add(pre_sql); + options.add(post_sql); return options; } - +// +// Called by PDAL core before the start of the writing process, but +// after the initialization. At this point, the machinery is all set +// up and we can apply actions to the target database, like pre-SQL and +// preparing new tables and/or deleting old ones. +// void Writer::writeBegin(boost::uint64_t /*targetNumPointsToWrite*/) { - std::string block_table = getOptions().getValueOrThrow("block_table"); - std::string cloud_table = getOptions().getValueOrThrow("cloud_table"); - std::string cloud_column = getOptions().getValueOrDefault("cloud_column", "id"); - m_block_insert_query << "INSERT INTO " << boost::to_lower_copy(block_table) - << " ("<< boost::to_lower_copy(cloud_column) <<", block_id, num_points, points, extent, bbox) VALUES (" - << " :obj_id, :block_id, :num_points, decode(:hex, 'hex'), ST_Force_2D(ST_GeometryFromText(:extent,:srid)), :bbox)"; - + // Start up the database connection m_session->begin(); - - bool bHaveBlockTable = CheckTableExists(block_table); - bool bHaveCloudTable = CheckTableExists(cloud_table); - - if (getOptions().getValueOrDefault("overwrite", true)) - { - if (bHaveBlockTable) - { - DeleteBlockTable(cloud_table, cloud_column, block_table); - bHaveBlockTable = false; - } - if (bHaveCloudTable) - { - DeleteCloudTable(cloud_table, cloud_column); - bHaveCloudTable = false; - } - } - + + // Pre-SQL can be *either* a SQL file to execute, *or* a SQL statement + // to execute. We find out which one here. std::string pre_sql = getOptions().getValueOrDefault("pre_sql", ""); if (pre_sql.size()) { @@ -176,324 +212,160 @@ void Writer::writeBegin(boost::uint64_t /*targetNumPointsToWrite*/) } m_session->once << sql; } + + bool bHaveTable = CheckTableExists(m_table_name); - if (!bHaveCloudTable) + // Apply the over-write preference if it is set + if ( m_overwrite && bHaveTable ) { - CreateCloudTable(cloud_table, getOptions().getValueOrDefault("srid", 4326)); + DeleteTable(m_schema_name, m_table_name); + bHaveTable = false; } - - if (!bHaveBlockTable) + + // Read or create a PCID for our new table + m_pcid = SetupSchema(m_pdal_schema, m_srid); + + // Create the table! + if ( ! bHaveTable ) { - m_doCreateIndex = true; - CreateBlockTable(block_table, getOptions().getValueOrDefault("srid", 4326)); + CreateTable(m_schema_name, m_table_name, m_column_name, m_pcid); } return; } -bool Writer::CheckTableExists(std::string const& name) +void Writer::writeEnd(boost::uint64_t /*actualNumPointsWritten*/) { + if ( m_create_index && m_have_postgis ) + { + CreateIndex(m_schema_name, m_table_name, m_column_name); + } - std::ostringstream oss; - - if (m_type == DATABASE_ORACLE) - oss << "select table_name from user_tables"; - else if (m_type == DATABASE_POSTGRESQL) - oss << "SELECT tablename FROM pg_tables"; - - log()->get(logDEBUG) << "checking for " << name << " existence ... " << std::endl; - - ::soci::rowset rs = (m_session->prepare << oss.str()); - - std::ostringstream debug; - for (::soci::rowset::const_iterator it = rs.begin(); it != rs.end(); ++it) + // Post-SQL can be *either* a SQL file to execute, *or* a SQL statement + // to execute. We find out which one here. + std::string post_sql = getOptions().getValueOrDefault("post_sql", ""); + if (post_sql.size()) { - debug << ", " << *it; - if (boost::iequals(*it, name)) + std::string sql = FileUtils::readFileAsString(post_sql); + if (!sql.size()) { - log()->get(logDEBUG) << "it exists!" << std::endl; - return true; + // if there was no file to read because the data in post_sql was + // actually the sql code the user wanted to run instead of the + // filename to open, we'll use that instead. + sql = post_sql; } + m_session->once << sql; } - log()->get(logDEBUG) << debug.str(); - log()->get(logDEBUG) << " -- '" << name << "' not found." << std::endl; - - return false; + m_session->commit(); + return; } -void Writer::CreateBlockTable(std::string const& name, boost::uint32_t srid) + +boost::uint32_t Writer::SetupSchema(Schema const& buffer_schema, boost::uint32_t srid) { - std::ostringstream oss; - - if (m_type == DATABASE_ORACLE) - { - // We just create a new block table as a copy of - // the SDO_PC_BLK_TYPE - oss << "CREATE TABLE " << name << " AS SELECT * FROM MDSYS.SDO_PC_BLK_TABLE"; - m_session->once << oss.str(); - oss.str(""); - } - else if (m_type == DATABASE_POSTGRESQL) - { - std::string cloud_column = getOptions().getValueOrDefault("cloud_column", "id"); - std::string cloud_table = getOptions().getValueOrThrow("cloud_table"); + // We strip any ignored dimensions from the schema before creating the table + pdal::Schema output_schema(PackSchema(buffer_schema)); - oss << "CREATE TABLE " << boost::to_lower_copy(name) - << "(" << boost::to_lower_copy(cloud_column) << " INTEGER REFERENCES " << boost::to_lower_copy(cloud_table) <<"," - // << " obj_id INTEGER," - << " block_id INTEGER," - << " num_points INTEGER," - << " points bytea," - << " bbox box3d, " - << " extent geometry" - << ")"; - - m_session->once << oss.str(); + // If the user has specified a PCID they want to use, + // does it exist in the database? + std::ostringstream oss; + long schema_count; + if ( m_pcid ) + { + oss << "SELECT Count(pcid) FROM pointcloud_formats WHERE pcid = " << m_pcid; + m_session->once << oss.str(), ::soci::into(schema_count); oss.str(""); - + if ( schema_count == 0 ) { - bool is3d = getOptions().getValueOrDefault("is3d", false); - boost::uint32_t nDim = 2; - - oss << "SELECT AddGeometryColumn('', '" << boost::to_lower_copy(name) - << "'," << "'extent'" << "," - << srid << ", 'POLYGON', " << nDim << ")"; - m_session->once << oss.str(); - oss.str(""); + oss << "requested PCID '" << m_pcid << "' does not exist in POINTCLOUD_FORMATS"; + throw pdal_error(oss.str()); } + return m_pcid; } -} - -void Writer::DeleteBlockTable(std::string const& cloud_table_name, - std::string const& cloud_column_name, - std::string const& block_table_name) -{ - std::ostringstream oss; - // Delete all the items from the table first - oss << "DELETE FROM " << block_table_name; - m_session->once << oss.str(); + // Do we have any existing schemas in the POINTCLOUD_FORMATS table? + boost::uint32_t pcid; + bool bCreatePCPointSchema = true; + oss << "SELECT Count(pcid) FROM pointcloud_formats"; + m_session->once << oss.str(), ::soci::into(schema_count); oss.str(""); - - // Drop the table's dependencies - if (m_type == DATABASE_ORACLE) - { - // These need to be uppercase to satisfy the PLSQL function - oss << "declare\n" - "begin \n" - " mdsys.sdo_pc_pkg.drop_dependencies('" - << boost::to_upper_copy(cloud_table_name) << - "', '" - << boost::to_upper_copy(cloud_column_name) << - "'); end;"; - m_session->once << oss.str(); - oss.str(""); - } - - // Go drop the table - if (m_type == DATABASE_ORACLE) - { - // We need to clean up the geometry column before dropping the table - // Oracle upper cases the table name when inserting it in the - // USER_SDO_GEOM_METADATA. - oss << "DELETE FROM USER_SDO_GEOM_METADATA WHERE TABLE_NAME='" << boost::to_upper_copy(block_table_name) << "'" ; - m_session->once << oss.str(); - oss.str(""); - - oss << "DROP TABLE " << block_table_name; - m_session->once << oss.str(); - oss.str(""); - - } else if (m_type == DATABASE_POSTGRESQL) - { - // We need to clean up the geometry column before dropping the table - oss << "SELECT DropGeometryColumn('" << boost::to_lower_copy(block_table_name) << "', 'extent')"; - m_session->once << oss.str(); - oss.str(""); - - oss << "DROP TABLE " << boost::to_lower_copy(block_table_name); - m_session->once << oss.str(); - oss.str(""); - } - -} - - -void Writer::CreateCloudTable(std::string const& name, boost::uint32_t srid) -{ - std::ostringstream oss; - - if (m_type == DATABASE_POSTGRESQL) + // Do any of the existing schemas match the one we want to use? + if (schema_count > 0) { - oss << "CREATE SEQUENCE " << boost::to_lower_copy(name)<<"_id_seq"; - m_session->once << oss.str(); - oss.str(""); + std::vector pg_schemas(schema_count); + std::vector pg_schema_ids(schema_count); + m_session->once << "SELECT pcid, schema FROM pointcloud_formats", ::soci::into(pg_schema_ids), ::soci::into(pg_schemas); - std::string cloud_column = getOptions().getValueOrDefault("cloud_column", "id"); - oss << "CREATE TABLE " << boost::to_lower_copy(name) - << " (" << boost::to_lower_copy(cloud_column) << " INTEGER DEFAULT NEXTVAL('" << boost::to_lower_copy(name)<<"_id_seq" <<"')," - << " schema XML," - << " block_table varchar(64)," - << " PRIMARY KEY ("<< boost::to_lower_copy(cloud_column) <<")" - << ")"; - - m_session->once << oss.str(); - oss.str(""); + for(int i=0; i("is3d", false); - boost::uint32_t nDim = 2 ;// is3d ? 3 : 2; - - oss << "SELECT AddGeometryColumn('', '" << boost::to_lower_copy(name) - << "'," << "'extent'" << "," - << srid << ", 'POLYGON', " << nDim << ")"; - m_session->once << oss.str(); - oss.str(""); + if (pdal::Schema::from_xml(pg_schemas[i]) == output_schema) + { + bCreatePCPointSchema = false; + pcid = pg_schema_ids[i]; + break; + } } } -} - -void Writer::DeleteCloudTable(std::string const& cloud_table_name, - std::string const& cloud_column_name) -{ - std::ostringstream oss; - - // Delete all the items from the table first - oss << "DELETE FROM " << cloud_table_name; - m_session->once << oss.str(); - oss.str(""); - // Go drop the table - if (m_type == DATABASE_ORACLE) + if (bCreatePCPointSchema) { + std::string xml; + std::string compression; - try - { - // We need to clean up the geometry column before dropping the table - // Oracle upper cases the table name when inserting it in the - // USER_SDO_GEOM_METADATA. - oss << "DELETE FROM USER_SDO_GEOM_METADATA WHERE TABLE_NAME='" << boost::to_upper_copy(cloud_table_name) << "'" ; - m_session->once << oss.str(); - oss.str(""); - } catch (::soci::soci_error const& e) - { - - } - - oss.str(""); - - try + if (schema_count == 0) { - oss << "DROP TABLE " << cloud_table_name; - m_session->once << oss.str(); - oss.str(""); - } catch (::soci::soci_error const& e) + pcid = 1; + } + else { - - } - oss.str(""); - - } else if (m_type == DATABASE_POSTGRESQL) - { - // We need to clean up the geometry column before dropping the table + m_session->once << "SELECT Max(pcid)+1 AS pcid FROM pointcloud_formats", + ::soci::into(pcid); + } - try + /* If the writer specifies a compression, we should set that */ + if ( m_patch_compression_type == COMPRESSION_DIMENSIONAL ) { - oss << "SELECT DropGeometryColumn('" << boost::to_lower_copy(cloud_table_name) << "', 'extent')"; - m_session->once << oss.str(); - } catch (::soci::soci_error const& e) - { - + compression = "dimensional"; } - oss.str(""); - - - try - { - oss << "DROP TABLE " << boost::to_lower_copy(cloud_table_name); - m_session->once << oss.str(); - } catch (::soci::soci_error const& e) + else if ( m_patch_compression_type == COMPRESSION_GHT ) { - + compression = "ght"; } - oss.str(""); + Metadata metadata("compression", compression, ""); + xml = pdal::Schema::to_xml(output_schema, &(metadata.toPTree())); + // xml = pdal::Schema::to_xml(output_schema); + oss << "INSERT INTO pointcloud_formats (pcid, srid, schema) "; + oss << "VALUES (:pcid, :srid, :xml)"; - try - { - oss << "DROP SEQUENCE " << boost::to_lower_copy(cloud_table_name)<<"_id_seq"; - m_session->once << oss.str(); - } catch (::soci::soci_error const& e) - { - - } + m_session->once << oss.str(), ::soci::use(pcid, "pcid"), ::soci::use(srid, "srid"), ::soci::use(xml, "xml"); oss.str(""); - } - + m_pcid = pcid; + return m_pcid; } -void Writer::CreateIndexes( std::string const& table_name, - std::string const& spatial_column_name, - bool is3d, - bool isBlockTable ) +void Writer::DeleteTable(std::string const& schema_name, + std::string const& table_name) { std::ostringstream oss; - std::ostringstream index_name_ss; - index_name_ss << table_name << "_cloud_idx"; - std::string index_name = index_name_ss.str().substr(0,29); - - // Spatial indexes - if (m_type == DATABASE_ORACLE) - { - oss << "CREATE INDEX "<< index_name << " on " - << table_name << "(" << spatial_column_name - << ") INDEXTYPE IS MDSYS.SPATIAL_INDEX"; - if (is3d) - { - oss <<" PARAMETERS('sdo_indx_dims=3')"; - } - m_session->once << oss.str(); - oss.str(""); - } else if (m_type == DATABASE_POSTGRESQL) - { - oss << "CREATE INDEX "<< index_name << " on " - << boost::to_lower_copy(table_name) << " USING GIST ("<< boost::to_lower_copy(spatial_column_name) << ")"; - m_session->once << oss.str(); - oss.str(""); - } + oss << "DROP TABLE IF EXISTS "; - // Primary key - - if (isBlockTable) + if ( schema_name.size() ) { - index_name_ss.str(""); - index_name_ss << table_name <<"_objectid_idx"; - index_name = index_name_ss.str().substr(0,29); - - std::string cloud_column = getOptions().getValueOrDefault("cloud_column", "id"); - - oss << "ALTER TABLE "<< table_name << " ADD CONSTRAINT "<< index_name << - " PRIMARY KEY ("<once << oss.str(); - + oss << schema_name << "."; } - else - { + oss << table_name; - - } + m_session->once << oss.str(); + oss.str(""); } -Schema Writer::getPackedSchema( Schema const& schema) const +Schema Writer::PackSchema( Schema const& schema) const { schema::index_by_index const& idx = schema.getDimensions().get(); log()->get(logDEBUG3) << "Packing ignored dimension from PointBuffer " << std::endl; @@ -517,256 +389,143 @@ Schema Writer::getPackedSchema( Schema const& schema) const position++; } } - return clean_schema; + std::string xml = pdal::Schema::to_xml(clean_schema); + return clean_schema; } -std::string Writer::loadGeometryWKT(std::string const& filename_or_wkt) const +bool Writer::CheckPointCloudExists() { - std::ostringstream wkt_s; + std::ostringstream oss; + oss << "SELECT PC_Version()"; + + log()->get(logDEBUG) << "checking for pointcloud existence ... " << std::endl; - if (!filename_or_wkt.empty()) + try + { + m_session->once << oss.str(); + } + catch (::soci::soci_error const &e) { - if (!FileUtils::fileExists(filename_or_wkt)) - { - if (!IsValidGeometryWKT(filename_or_wkt)) - { - std::ostringstream oss; - oss << "WKT for not valid and '" << filename_or_wkt - << "' doesn't exist as a file"; - throw pdal::pdal_error(oss.str()); - } - wkt_s << filename_or_wkt; - } - else - { - std::string wkt = FileUtils::readFileAsString(filename_or_wkt); - if (!IsValidGeometryWKT(wkt)) - { - std::ostringstream oss; - oss << "WKT for was from file '" << filename_or_wkt - << "' is not valid"; - throw pdal::pdal_error(oss.str()); - } - wkt_s << wkt; - } - } - return wkt_s.str(); + oss.str(""); + return false; + } + + oss.str(""); + return true; } -bool Writer::IsValidGeometryWKT(std::string const& input) const +bool Writer::CheckPostGISExists() { -#ifdef PDAL_HAVE_GDAL + std::ostringstream oss; + oss << "SELECT PostGIS_Version()"; - OGRGeometryH g; + log()->get(logDEBUG) << "checking for PostGIS existence ... " << std::endl; - char* wkt = const_cast(input.c_str()); - OGRErr e = OGR_G_CreateFromWkt(&wkt, NULL, &g); - OGR_G_DestroyGeometry(g); - if (e != 0) return false; + try + { + m_session->once << oss.str(); + } + catch (::soci::soci_error const &e) + { + oss.str(""); + return false; + } + oss.str(""); return true; -#else - - throw pdal_error("GDAL support not available for WKT validation"); -#endif } -void Writer::CreateSDOEntry(std::string const& block_table, - boost::uint32_t srid, - pdal::Bounds bounds, - bool is3d) -{ - std::ostringstream oss; - oss.setf(std::ios_base::fixed, std::ios_base::floatfield); - boost::uint32_t precision = getOptions().getValueOrDefault("stream_output_precision", 8); - oss.precision(precision); - std::ostringstream s_srid; +bool Writer::CheckTableExists(std::string const& name) +{ - if (srid == 0) - { - s_srid << "NULL"; - } - else - { - s_srid << srid; - } + std::ostringstream oss; + oss << "SELECT tablename FROM pg_tables"; - double tolerance = 0.05; + log()->get(logDEBUG) << "checking for " << name << " existence ... " << std::endl; + ::soci::rowset rs = (m_session->prepare << oss.str()); - pdal::Bounds e = bounds; - - if (srid) + std::ostringstream debug; + for (::soci::rowset::const_iterator it = rs.begin(); it != rs.end(); ++it) { - SpatialReference ref; - ref.setFromUserInput("EPSG:"+s_srid.str()); - if (ref.isGeographic()) + debug << ", " << *it; + if (boost::iequals(*it, name)) { - // FIXME: This should be overrideable - e.setMinimum(0,-180.0); - e.setMaximum(0,180.0); - e.setMinimum(1,-90.0); - e.setMaximum(1,90.0); - e.setMinimum(2,0.0); - e.setMaximum(2,20000.0); - - tolerance = 0.0005; + log()->get(logDEBUG) << "it exists!" << std::endl; + return true; } } + log()->get(logDEBUG) << debug.str(); + log()->get(logDEBUG) << " -- '" << name << "' not found." << std::endl; + return false; +} - oss << "INSERT INTO user_sdo_geom_metadata VALUES ('" << block_table << - "','extent', MDSYS.SDO_DIM_ARRAY("; - - oss << "MDSYS.SDO_DIM_ELEMENT('X', " << e.getMinimum(0) << "," << e.getMaximum(0) <<"," << tolerance << ")," - "MDSYS.SDO_DIM_ELEMENT('Y', " << e.getMinimum(1) << "," << e.getMaximum(1) <<"," << tolerance << ")"; - - if (is3d) +void Writer::CreateTable(std::string const& schema_name, + std::string const& table_name, + std::string const& column_name, + boost::uint32_t pcid) +{ + std::ostringstream oss; + oss << "CREATE TABLE "; + if ( schema_name.size() ) { - oss << ","; - oss <<"MDSYS.SDO_DIM_ELEMENT('Z', "<< e.getMinimum(2) << "," << e.getMaximum(2) << "," << tolerance << ")"; + oss << schema_name << "."; } - oss << ")," << s_srid.str() << ")"; + oss << table_name; + oss << " (id SERIAL PRIMARY KEY, " << column_name << " PcPatch"; + if ( pcid ) + { + oss << "(" << pcid << ")"; + } + oss << ")"; m_session->once << oss.str(); - oss.str(""); - } -void Writer::writeEnd(boost::uint64_t /*actualNumPointsWritten*/) +// Make sure you test for the presence of PostGIS before calling this +void Writer::CreateIndex(std::string const& schema_name, + std::string const& table_name, + std::string const& column_name) { - - if (m_doCreateIndex) - { - std::string block_table_name = getOptions().getValueOrThrow("block_table"); - std::string cloud_table_name = getOptions().getValueOrThrow("cloud_table"); - boost::uint32_t srid = getOptions().getValueOrThrow("srid"); - bool is3d = getOptions().getValueOrDefault("is3d", false); - - CreateIndexes(block_table_name, "extent", is3d); - if (m_type == DATABASE_POSTGRESQL) - CreateIndexes(cloud_table_name, "extent", is3d, false); - } - - if (m_type == DATABASE_ORACLE) + std::ostringstream oss; + + oss << "CREATE INDEX "; + if ( schema_name.size() ) { - std::string block_table_name = getOptions().getValueOrThrow("block_table"); - boost::uint32_t srid = getOptions().getValueOrThrow("srid"); - bool is3d = getOptions().getValueOrDefault("is3d", false); - CreateSDOEntry( block_table_name, - srid, - m_bounds, - is3d); + oss << schema_name << "_"; } - - m_session->commit(); - return; + oss << table_name << "_pc_gix"; + oss << " USING GIST (Geometry(" << column_name << "))"; + + m_session->once << oss.str(); + oss.str(""); } + +// +// Called by PDAL core before *each buffer* is written. +// So it gets called a lot. The hack below does something +// the first time it is called only. Hopefully we do +// not need that hack anymore. +// void Writer::writeBufferBegin(PointBuffer const& data) { - if (m_sdo_pc_is_initialized) return; - - CreateCloud(data.getSchema()); - m_sdo_pc_is_initialized = true; + if ( ! m_sdo_pc_is_initialized) + { + // Currently Unused + // Do somethine only once, after PointBuffer is sent in + // like setting up tables, for example, in case the + // schema we get from the parent is not valid? + m_sdo_pc_is_initialized = true; + } return; - } -void Writer::CreateCloud(Schema const& buffer_schema) -{ - std::string cloud_table = getOptions().getValueOrThrow("cloud_table"); - std::string block_table = getOptions().getValueOrThrow("block_table"); - - std::ostringstream oss; - - pdal::Schema output_schema(buffer_schema); - bool pack = getOptions().getValueOrDefault("pack_ignored_fields", true); - if (pack) - { - output_schema = getPackedSchema(buffer_schema); - } - - std::string bounds = getOptions().getValueOrDefault("cloud_boundary_wkt", ""); - if (bounds.size()) - { - log()->get(logDEBUG2) << "have cloud_boundary_wkt of size " << bounds.size() << std::endl; - bounds = loadGeometryWKT(bounds); - - } - - if (m_type == DATABASE_POSTGRESQL) - { - // strk: create table tabref( ref regclass ); - // [11:32am] strk: insert into tabref values ('geometry_columns'); - // [11:33am] strk: select ref from tableref; - // [11:33am] strk: select ref::oid from tableref; - // [11:33am] strk: create table dropme (a int); - // [11:33am] strk: insert into tabref values ('dropme'); - // [11:34am] strk: select ref, ref::oid from tableref; - // [11:34am] strk: drop table dropme; - // [11:34am] strk: -- oops, is this what you want ? - // [11:34am] strk: select ref, ref::oid from tableref; - // [11:34am] strk: create table dropme (a int); - // [11:34am] strk: select ref, ref::oid from tableref; -- no, it's not back - // [11:35am] hobu: thanks - // [11:35am] hobu: not quite a pointer, but close enough - // [11:36am] strk: it's really a pointer, just maybe a too "hard" one - // [11:36am] strk: not a soft pointer - // [11:36am] strk: it's not a string, is an oid - // [11:36am] strk: looks like a string with the default output - // [11:36am] strk: try changing your search_path - // [11:36am] strk: the canonical text output should also change to include the pointed-to path - - long id; - std::string cloud_column = getOptions().getValueOrDefault("cloud_column", "id"); - oss << "INSERT INTO " << boost::to_lower_copy(cloud_table) - << "(" << boost::to_lower_copy(cloud_column) - << ", block_table, schema) VALUES (DEFAULT,'" - << boost::to_lower_copy(block_table) - << "',:xml) RETURNING " << boost::to_lower_copy(cloud_column); - std::string xml = pdal::Schema::to_xml(output_schema); - m_session->once << oss.str(), ::soci::use(xml), ::soci::into(id); - oss.str(""); - // - // int id; - // oss << "SELECT CURRVAL('"<< boost::to_lower_copy(cloud_table) <<"_id_seq')"; - // (m_session->once << oss.str(), ::soci::into(id)); - // oss.str(""); - - log()->get(logDEBUG) << "Point cloud id was " << id << std::endl; - try - { - Option& pc_id = getOptions().getOptionByRef("pc_id"); - pc_id.setValue(id); - } - catch (pdal::option_not_found&) - { - Option pc_id("pc_id", id, "Point Cloud Id"); - getOptions().add(pc_id); - } - - if (bounds.size()) - { - boost::uint32_t srid = getOptions().getValueOrDefault("srid", 4326); - bool is3d = getOptions().getValueOrDefault("is3d", false); - std::string force = "ST_Force_2D"; - - oss << "UPDATE " - << boost::to_lower_copy(cloud_table) - << " SET extent="<< force - << "(ST_GeometryFromText(:wkt,:srid)) where " - << boost::to_lower_copy(cloud_column) <<"=:id"; - - m_session->once << oss.str(), ::soci::use(bounds, "wkt"), ::soci::use(srid,"srid"), ::soci::use(id, "id"); - - } - } -} boost::uint32_t Writer::writeBuffer(const PointBuffer& buffer) { @@ -779,102 +538,58 @@ boost::uint32_t Writer::writeBuffer(const PointBuffer& buffer) bool Writer::WriteBlock(PointBuffer const& buffer) { - boost::uint8_t* point_data; boost::uint32_t point_data_length; boost::uint32_t schema_byte_size; + PackPointData(buffer, &point_data, point_data_length, schema_byte_size); - bool pack = getOptions().getValueOrDefault("pack_ignored_fields", true); - if (pack) - PackPointData(buffer, &point_data, point_data_length, schema_byte_size); - else - { - point_data = buffer.getData(0); - point_data_length = buffer.getSchema().getByteSize() * buffer.getNumPoints(); - } - - std::string block_table = getOptions().getValueOrThrow("block_table"); - - // // Pluck the block id out of the first point in the buffer + // Pluck the block id out of the first point in the buffer pdal::Schema const& schema = buffer.getSchema(); Dimension const& blockDim = schema.getDimension("BlockID"); - m_block_id = buffer.getField(blockDim, 0); - m_obj_id = getOptions().getValueOrThrow("pc_id"); - m_num_points = static_cast(buffer.getNumPoints()); - if (m_type == DATABASE_POSTGRESQL) +// boost::int32_t blk_id = buffer.getField(blockDim, 0); + boost::uint32_t num_points = static_cast(buffer.getNumPoints()); + + if ( num_points > m_patch_capacity ) { - std::string cloud_column = getOptions().getValueOrDefault("cloud_column", "id"); - bool is3d = getOptions().getValueOrDefault("is3d", false); - - - std::vector block_data; - for (boost::uint32_t i = 0; i < point_data_length; ++i ) - { - block_data.push_back(point_data[i]); - } - m_block_bytes.str(""); - Utils::binary_to_hex_stream(point_data, m_block_bytes, 0, point_data_length); - m_block_data = m_block_bytes.str(); - //std::cout << "hex: " << hex.substr(0, 30) << std::endl; - m_srid = getOptions().getValueOrDefault("srid", 4326); - - boost::uint32_t precision(9); - pdal::Bounds bounds = buffer.calculateBounds(3); - // m_extent.str(""); - m_extent = bounds.toWKT(precision); // polygons are only 2d, not cubes - // m_bbox.str(""); - m_bbox = bounds.toBox(precision, 3); - log()->get(logDEBUG) << "extent: " << m_extent << std::endl; - log()->get(logDEBUG) << "bbox: " << m_bbox << std::endl; - - if (!m_block_statement) - { - // m_block_statement = (m_session->prepare << m_block_insert_query.str(), \ - // ::soci::use(m_obj_id, "obj_id"), \ - // ::soci::use(m_block_id, "block_id"), \ - // ::soci::use(m_num_points, "num_points"), \ - // ::soci::use(m_block_bytes.str(),"hex"), \ - // ::soci::use(m_extent.str(), "extent"), \ - // ::soci::use(m_srid, "srid"), \ - // ::soci::use(m_bbox.str(), "bbox")); - m_block_statement = new ::soci::statement(*m_session); - - m_block_statement->exchange( ::soci::use(m_obj_id, "obj_id")); - m_block_statement->exchange(::soci::use(m_block_id, "block_id")); - m_block_statement->exchange(::soci::use(m_num_points, "num_points")); - m_block_statement->exchange(::soci::use(m_block_data,"hex")); - m_block_statement->exchange(::soci::use(m_extent, "extent")); - m_block_statement->exchange(::soci::use(m_srid, "srid")); - m_block_statement->exchange(::soci::use(m_bbox, "bbox")); - m_block_statement->alloc(); - m_block_statement->prepare(m_block_insert_query.str()); - m_block_statement->define_and_bind(); - - } - // ::soci::statement st = (m_session->prepare << m_block_insert_query.str(), \ - // ::soci::use(m_obj_id, "obj_id"), \ - // ::soci::use(m_block_id, "block_id"), \ - // ::soci::use(m_num_points, "num_points"), \ - // ::soci::use(m_block_bytes.str(),"hex"), \ - // ::soci::use(m_extent.str(), "extent"), \ - // ::soci::use(m_srid, "srid"), \ - // ::soci::use(m_bbox.str(), "bbox")); - try - { - m_block_statement->execute(true); - } - catch (std::exception const& e) - { - std::ostringstream oss; - oss << "Insert query failed with error '" << e.what() << "'"; - m_session->rollback(); - throw pdal_error(oss.str()); - } - + // error here } + std::vector block_data; + for (boost::uint32_t i = 0; i < point_data_length; ++i) + { + block_data.push_back(point_data[i]); + } + + /* We are always getting uncompressed bytes off the block_data */ + /* so we always used compression type 0 (uncompressed) in writing our WKB */ + boost::int32_t pcid = m_pcid; + boost::uint32_t compression = COMPRESSION_NONE; + + std::stringstream oss; + oss << "INSERT INTO " << m_table_name << " (pa) VALUES (:hex)"; + + std::stringstream options; + #ifdef BOOST_LITTLE_ENDIAN + options << boost::format("%02x") % 1; + SWAP_ENDIANNESS(pcid); + SWAP_ENDIANNESS(compression); + SWAP_ENDIANNESS(num_points); + #elif BOOST_BIG_ENDIAN + options << boost::format("%02x") % 0; + #endif + + options << boost::format("%08x") % pcid; + options << boost::format("%08x") % compression; + options << boost::format("%08x") % num_points; + + std::stringstream hex; + hex << options.str() << Utils::binary_to_hex_string(block_data); + ::soci::statement st = (m_session->prepare << oss.str(), ::soci::use(hex.str(),"hex")); + st.execute(true); + oss.str(""); + return true; }