SQLite plugin with Spatialite support - playing around & initial implementation #2422

SConstruct

@@ -99,7 +99,8 @@ pretty_dep_names = {
     'osm':'more info: https://github.com/mapnik/mapnik/wiki/OsmPlugin',
     'boost_regex_icu':'libboost_regex built with optional ICU unicode support is needed for unicode regex support in mapnik.',
     'sqlite_rtree':'The SQLite plugin requires libsqlite3 built with RTREE support (-DSQLITE_ENABLE_RTREE=1)',
-    'pgsql2sqlite_rtree':'The pgsql2sqlite program requires libsqlite3 built with RTREE support (-DSQLITE_ENABLE_RTREE=1)'
+    'pgsql2sqlite_rtree':'The pgsql2sqlite program requires libsqlite3 built with RTREE support (-DSQLITE_ENABLE_RTREE=1)',
+    'spatialite':'Spatialite extension for SQLite (-DHAS_SPATIALITE=1)'
     }
 
 # Core plugin build configuration
@@ -379,6 +380,9 @@ opts.AddVariables(
     PathVariable('OCCI_LIBS', 'Search path for OCCI library files', '/usr/lib/oracle/10.2.0.3/client/'+ LIBDIR_SCHEMA_DEFAULT, PathVariable.PathAccept),
     PathVariable('SQLITE_INCLUDES', 'Search path for SQLITE include files', '/usr/include/', PathVariable.PathAccept),
     PathVariable('SQLITE_LIBS', 'Search path for SQLITE library files', '/usr/' + LIBDIR_SCHEMA_DEFAULT, PathVariable.PathAccept),
+    BoolVariable('SPATIALITE', 'Attempt to build with Spatialite support', 'True'),
+    PathVariable('SPATIALITE_INCLUDES', 'Search path for Spatialite include files', '/usr/include/', PathVariable.PathAccept),
+    PathVariable('SPATIALITE_LIBS', 'Search path for Spatialite library files', '/usr/' + LIBDIR_SCHEMA_DEFAULT, PathVariable.PathAccept),
     PathVariable('RASTERLITE_INCLUDES', 'Search path for RASTERLITE include files', '/usr/include/', PathVariable.PathAccept),
     PathVariable('RASTERLITE_LIBS', 'Search path for RASTERLITE library files', '/usr/' + LIBDIR_SCHEMA_DEFAULT, PathVariable.PathAccept),
 
@@ -1214,7 +1218,7 @@ if not preconfigured:
     # set any custom cxxflags and ldflags to come first
     if sys.platform == 'darwin' and not env['HOST']:
         DEFAULT_CXX11_CXXFLAGS += ' -stdlib=libc++'
-        DEFAULT_CXX11_LINKFLAGS = ' -stdlib=libc++'
+        DEFAULT_CXX11_LINKFLAGS += ' -stdlib=libc++'
     env.Append(CPPDEFINES = env['CUSTOM_DEFINES'])
     env.Append(CXXFLAGS = DEFAULT_CXX11_CXXFLAGS)
     env.Append(CXXFLAGS = env['CUSTOM_CXXFLAGS'])
@@ -1349,6 +1353,15 @@ if not preconfigured:
     else:
         env['SKIPPED_DEPS'].extend(['tiff'])
 
+    if env['SPATIALITE']:
+        OPTIONAL_LIBSHEADERS.append(['spatialite', ['sqlite3.h', 'spatialite.h'], False, 'C', '-DHAS_SPATIALITE'])
+        inc_path = env['%s_INCLUDES' % 'SPATIALITE']
+        lib_path = env['%s_LIBS' % 'SPATIALITE']
+        env.AppendUnique(CPPPATH = fix_path(inc_path))
+        env.AppendUnique(LIBPATH = fix_path(lib_path))
+    else:
+        env['SKIPPED_DEPS'].extend(['spatalite'])
+
     # if requested, sort LIBPATH and CPPPATH before running CheckLibWithHeader tests
     if env['PRIORITIZE_LINKING']:
         conf.prioritize_paths(silent=True)

bootstrap.sh

@@ -55,6 +55,9 @@ function install_mason_deps() {
     install libtiff 4.0.6 libtiff &
     install libpq 9.5.2 &
     install sqlite 3.14.1 libsqlite3 &
+    # these could somehow be checked via env['SPATIALITE'] check
+    install geos 3.5.0 libgeos &
+    install spatialite 4.3.0a libspatialite &
     install expat 2.2.0 libexpat &
     install icu ${ICU_VERSION} &
     install proj 4.9.2 libproj &
@@ -126,6 +129,8 @@ CAIRO_INCLUDES = '${MASON_LINKED_REL}/include'
 CAIRO_LIBS = '${MASON_LINKED_REL}/lib'
 SQLITE_INCLUDES = '${MASON_LINKED_REL}/include'
 SQLITE_LIBS = '${MASON_LINKED_REL}/lib'
+SPATIALITE_INCLUDES = '${MASON_LINKED_REL}/include'
+SPATIALITE_LIBS = '${MASON_LINKED_REL}/lib'
 BENCHMARK = True
 CPP_TESTS = True
 PGSQL2SQLITE = True

plugins/input/sqlite/build.py

@@ -34,7 +34,10 @@
 )
 
 # Link Library to Dependencies
-libraries = [ 'sqlite3' ]
+if env['SPATIALITE']:
+    libraries = [ 'spatialite', 'sqlite3', 'xml2', 'proj', 'geos_c', 'geos' ]
+else:
+    libraries = [ 'sqlite3' ]
 
 linkflags = []
 if env['SQLITE_LINKFLAGS']:

plugins/input/sqlite/sqlite_connection.hpp

@@ -40,6 +40,10 @@
 // sqlite
 extern "C" {
 #include <sqlite3.h>
+
+#ifdef HAS_SPATIALITE
+#include <spatialite.h>
+#endif
 }
 
 #include "sqlite_resultset.hpp"
@@ -78,6 +82,10 @@ class sqlite_connection
             throw mapnik::datasource_exception (s.str());
         }
 
+#ifdef HAS_SPATIALITE
+        conn_ = spatialite_alloc_connection();
+        spatialite_init_ex(db_, conn_, 0);
+#endif
         sqlite3_busy_timeout(db_,5000);
     }
 
@@ -97,13 +105,21 @@ class sqlite_connection
 
             throw mapnik::datasource_exception (s.str());
         }
+
+#ifdef HAS_SPATIALITE
+        conn_ = spatialite_alloc_connection();
+        spatialite_init_ex(db_, conn_, 0);
+#endif
     }
 
     virtual ~sqlite_connection ()
     {
         if (db_)
         {
             sqlite3_close (db_);
+#ifdef HAS_SPATIALITE
+            spatialite_cleanup_ex(conn_);
+#endif
         }
     }
 
@@ -177,6 +193,9 @@ class sqlite_connection
 
     sqlite3* db_;
     std::string file_;
+#ifdef HAS_SPATIALITE
+    void* conn_;
+#endif
 };
 
 #endif // MAPNIK_SQLITE_CONNECTION_HPP

test/unit/datasource/sqlite.cpp

@@ -0,0 +1,124 @@
+/*****************************************************************************
+*
+* This file is part of Mapnik (c++ mapping toolkit)
+*
+* Copyright (C) 2016 jsimomaa
+*
+* This library is free software; you can redistribute it and/or
+* modify it under the terms of the GNU Lesser General Public
+* License as published by the Free Software Foundation; either
+* version 2.1 of the License, or (at your option) any later version.
+*
+* This library is distributed in the hope that it will be useful,
+* but WITHOUT ANY WARRANTY; without even the implied warranty of
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+* Lesser General Public License for more details.
+*
+* You should have received a copy of the GNU Lesser General Public
+* License along with this library; if not, write to the Free Software
+* Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+*
+*****************************************************************************/
+
+#include "catch.hpp"
+#include "ds_test_util.hpp"
+
+#include <mapnik/map.hpp>
+#include <mapnik/unicode.hpp>
+#include <mapnik/datasource.hpp>
+#include <mapnik/datasource_cache.hpp>
+#include <mapnik/geometry.hpp>
+#include <mapnik/geometry_types.hpp>
+#include <mapnik/geometry_type.hpp>
+#include <mapnik/expression.hpp>
+#include <mapnik/expression_evaluator.hpp>
+#include <mapnik/debug.hpp>
+#include <mapnik/util/fs.hpp>
+#include <boost/format.hpp>
+#include <boost/optional/optional_io.hpp>
+
+#pragma GCC diagnostic push
+#include <mapnik/warning_ignore.hpp>
+#include <boost/algorithm/string.hpp>
+#pragma GCC diagnostic pop
+
+#include <iostream>
+
+namespace {
+    mapnik::datasource_ptr get_sqlite_ds(std::string const& file_name, std::string const& table = "", bool strict = true, std::string const& base = "")
+    {
+        mapnik::parameters params;
+
+        params["type"]  = std::string("sqlite");
+        params["file"]  = file_name;
+        params["geometry_field"] = "geometry";
+        params["table"] = table;
+        if (!base.empty()) {
+            params["base"] = base;
+        }
+        params["strict"] = mapnik::value_bool(strict);
+        auto ds = mapnik::datasource_cache::instance().create(params);
+        // require a non-null pointer returned
+        REQUIRE(ds != nullptr);
+        return ds;
+    }
+} // anonymous namespace
+
+TEST_CASE("sqlite"){
+    std::string sqlite_plugin("./plugins/input/sqlite.input");
+
+    if (mapnik::util::exists(sqlite_plugin)) {
+        // make the tests silent since we intentionally test error conditions that are noisy
+        auto const severity = mapnik::logger::instance().get_severity();
+        mapnik::logger::instance().set_severity(mapnik::logger::none);
+
+        // check the sqlite datasource is loaded
+        const std::vector<std::string> plugin_names = mapnik::datasource_cache::instance().plugin_names();
+        const bool have_sqlite_plugin =
+          std::find(plugin_names.begin(), plugin_names.end(), "sqlite") != plugin_names.end();
+
+        SECTION("SQLite I/O errors")
+        {
+            std::string filename = "does_not_exist.sqlite";
+
+            for (auto create_index : { true, false }) {
+                if (create_index) {
+                    int ret       = create_disk_index(filename);
+                    int ret_posix = (ret >> 8) & 0x000000ff;
+                    INFO(ret);
+                    INFO(ret_posix);
+                    // index wont be created
+                    CHECK(!mapnik::util::exists(filename + ".index"));
+                }
+                mapnik::parameters params;
+                params["type"] = "sqlite";
+                params["file"] = filename;
+                REQUIRE_THROWS(mapnik::datasource_cache::instance().create(params));
+                params["base"] = "";
+                REQUIRE_THROWS(mapnik::datasource_cache::instance().create(params));
+                params["base"] = "/";
+                REQUIRE_THROWS(mapnik::datasource_cache::instance().create(params));
+            }
+        }
+
+        SECTION("sqlite emptydb support")
+        {
+            for (auto create_index : { false, true }) {
+                if (have_sqlite_plugin) {
+                    mapnik::datasource_ptr sqlite_ds = get_sqlite_ds("test/data/sqlite/empty.db", "(select * from empty where \"a\"!=\"b\" and !intersects!)");
+                    auto fs = all_features(sqlite_ds);
+                    REQUIRE(mapnik::is_valid(fs));
+                }
+            }
+        } // END SECTION
+
+#ifdef HAS_SPATIALITE
+        SECTION("sqlite spatialite support")
+        {
+            mapnik::datasource_ptr sqlite_ds = get_sqlite_ds("test/data/sqlite/qgis_spatiallite.sqlite", "(select ST_IsValid(geometry) from lines)");
+        }
+#endif
+
+        mapnik::logger::instance().set_severity(severity);
+    }
+} // END TEST CASE