Merge pull request #809 from CartoDB/679-mvt-pass-variables

Pass variables to replace tokens in query

.gitignore

@@ -23,3 +23,7 @@ documentation
 .mason/
 .toolchain/
 mason_packages/
+mason-postgis-config.env
+local-postgres/
+postgres.log
+local-unix-socket/

.travis.yml

@@ -31,6 +31,7 @@ install:
 # we use before_script rather than script to ensure fast failure (the script section continues even after an error)
 # https://docs.travis-ci.com/user/customizing-the-build#Breaking-the-Build
 before_script:
+  - source scripts/postgis/setup_and_run.sh
   - npm test
   # after successful tests, publish binaries if specified in commit message
   - ./scripts/publish.sh --toolset=${TOOLSET:-} --debug=$([ "${BUILDTYPE}" == 'debug' ] && echo "true" || echo "false")
@@ -133,6 +134,7 @@ matrix:
         - export LD_PRELOAD=${MASON_LLVM_RT_PRELOAD}
         # TODO: re-enable detect_leaks=0 once we can build and test against an asan sanitized libc++
         - export ASAN_OPTIONS=fast_unwind_on_malloc=0:detect_leaks=0:${ASAN_OPTIONS}
+        - source scripts/postgis/setup_and_run.sh
         - npm test
         - unset LD_PRELOAD
         # after successful tests, publish binaries if specified in commit message
@@ -153,6 +155,7 @@ matrix:
         - make ${BUILDTYPE}
       # Overrides `script` to disable publishing
       before_script:
+        - source scripts/postgis/setup_and_run.sh
         - npm test
     # test building with out SSE_MATH enabled
     - os: linux
@@ -167,6 +170,7 @@ matrix:
       node_js: 4
       # Overrides `script` to publish coverage data to codecov
       before_script:
+        - source scripts/postgis/setup_and_run.sh
         - npm test
         - mason install llvm-cov ${MASON_LLVM_RELEASE}
         - mason link llvm-cov ${MASON_LLVM_RELEASE}

scripts/postgis/initialize.sh

@@ -0,0 +1,47 @@
+#!/bin/bash
+
+set -eu
+set -o pipefail
+
+if [[ ! -f mason-postgis-config.env ]]; then
+    echo "Please run setup.sh first"
+    exit 1
+fi
+
+# do each time you use the local postgis:
+source mason-postgis-config.env
+
+# do once: create directories to hold postgres data
+echo "Creating pghost: ${PGHOST} and temp dir: ${PGTEMP_DIR}"
+mkdir ${PGHOST}
+mkdir ${PGTEMP_DIR}
+
+# do once: initialize local db cluster
+echo "Initializing database cluser at ${PGDATA}"
+./mason_packages/.link/bin/initdb
+sleep 2
+
+# start server and background (NOTE: if running interactively hit return to fully background and get your prompt back)
+./mason_packages/.link/bin/postgres -k $PGHOST > postgres.log &
+sleep 2
+
+# set up postgres to know about local temp directory
+./mason_packages/.link/bin/psql postgres -c "CREATE TABLESPACE temp_disk LOCATION '${PGTEMP_DIR}';"
+./mason_packages/.link/bin/psql postgres -c "SET temp_tablespaces TO 'temp_disk';"
+
+# add plpython support if you need
+./mason_packages/.link/bin/psql postgres -c "CREATE PROCEDURAL LANGUAGE 'plpythonu' HANDLER plpython_call_handler;"
+
+# create postgis enabled db
+./mason_packages/.link/bin/createdb template_postgis -T postgres
+./mason_packages/.link/bin/psql template_postgis -c "CREATE EXTENSION postgis;"
+./mason_packages/.link/bin/psql template_postgis -c "SELECT PostGIS_Full_Version();"
+# load hstore, fuzzystrmatch, and unaccent extensions
+./mason_packages/.link/bin/psql template_postgis -c "CREATE EXTENSION hstore;"
+./mason_packages/.link/bin/psql template_postgis -c "CREATE EXTENSION fuzzystrmatch;"
+./mason_packages/.link/bin/psql template_postgis -c "CREATE EXTENSION unaccent;"
+
+echo "Fully bootstrapped template_postgis database is now ready"
+
+# stop the database
+./mason_packages/.link/bin/pg_ctl -w stop

scripts/postgis/run.sh

@@ -0,0 +1,21 @@
+#!/bin/bash
+
+set -eu
+set -o pipefail
+
+if [[ ! -f mason-postgis-config.env ]]; then
+    echo "Please run setup.sh first"
+    exit 1
+fi
+
+# do each time you use the local postgis:
+source mason-postgis-config.env
+
+# do each time you use this local postgis:
+# start server and background (NOTE: if running interactively hit return to fully background and get your prompt back)
+./mason_packages/.link/bin/postgres -k $PGHOST > postgres.log &
+sleep 2
+
+echo "Server is now running"
+echo "To stop server do:"
+echo "    source mason-postgis-config.env && ./mason_packages/.link/bin/pg_ctl -w stop"

scripts/postgis/setup.sh

@@ -0,0 +1,30 @@
+#!/bin/bash
+
+set -eu
+set -o pipefail
+
+GDAL_VERSION="2.1.3"
+POSTGIS_VERSION="2.3.2-1"
+
+# do once: install stuff
+./mason/mason install libgdal ${GDAL_VERSION}
+GDAL_DATA_VALUE=$(./mason/mason prefix libgdal ${GDAL_VERSION})/share/gdal/
+./mason/mason install postgis ${POSTGIS_VERSION}
+./mason/mason link postgis ${POSTGIS_VERSION}
+
+
+if [[ ! -d ${GDAL_DATA_VALUE} ]]; then
+    echo "${GDAL_DATA_VALUE} not found (needed for postgis to access GDAL_DATA)"
+    exit 1
+fi
+
+
+# setup config
+echo 'export CURRENT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"' > mason-postgis-config.env
+echo 'export PGDATA=${CURRENT_DIR}/local-postgres' >> mason-postgis-config.env
+echo 'export PGHOST=${CURRENT_DIR}/local-unix-socket' >> mason-postgis-config.env
+echo 'export PGTEMP_DIR=${CURRENT_DIR}/local-tmp' >> mason-postgis-config.env
+echo 'export PGPORT=1111' >> mason-postgis-config.env
+echo 'export PATH=${CURRENT_DIR}/mason_packages/.link/bin:${PATH}' >> mason-postgis-config.env
+echo "export GDAL_DATA=${GDAL_DATA_VALUE}" >> mason-postgis-config.env
+echo "generated mason-postgis-config.env"

scripts/postgis/setup_and_run.sh

@@ -0,0 +1,4 @@
+./scripts/postgis/setup.sh
+./scripts/postgis/initialize.sh
+./scripts/postgis/run.sh
+source mason-postgis-config.env

scripts/postgis/stop.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+
+set -eu
+set -o pipefail
+
+source mason-postgis-config.env && ./mason_packages/.link/bin/pg_ctl -w stop

src/mapnik_map.cpp

@@ -2171,7 +2171,7 @@ void Map::EIO_RenderVectorTile(uv_work_t* req)
     {
         mapnik::Map const& map = *closure->m->get();
 
-        mapnik::vector_tile_impl::processor ren(map);
+        mapnik::vector_tile_impl::processor ren(map, closure->variables);
         ren.set_simplify_distance(closure->simplify_distance);
         ren.set_multi_polygon_union(closure->multi_polygon_union);
         ren.set_fill_type(closure->fill_type);

test/data/postgis-create-db-and-tables.sql

@@ -0,0 +1,9 @@
+CREATE TABLE test(gid serial PRIMARY KEY, geom geometry, colbigint bigint, col_text text, "col-char" char, "col+bool" boolean, "colnumeric" numeric, "colsmallint" smallint, "colfloat4" real, "colfloat8" double precision, "colcharacter" character);
+INSERT INTO test VALUES (DEFAULT, GeomFromEWKT('SRID=4326;POINT(0 0)'), -9223372036854775808, 'I am a point', 'A', TRUE, 1234567809990001, 0, 0.0, 0.0, 'A');
+INSERT INTO test VALUES (DEFAULT, GeomFromEWKT('SRID=4326;POINT(-2 2)'), 9223372036854775807, 'I, too, am a point!', 'B', FALSE, -123456780999001, 0, 0.0, 0.0, 'A');
+INSERT INTO test VALUES (DEFAULT, GeomFromEWKT('SRID=4326;MULTIPOINT(2 1,1 2)'), -1, 'I`m even a MULTI Point', 'Z', FALSE, 12345678099901, 0, 0.0, 0.0, 'A');
+INSERT INTO test VALUES (DEFAULT, GeomFromEWKT('SRID=4326;LINESTRING(0 0,1 1,1 2)'), 0, 'This is a line string', 'ß', FALSE, -9, 0, 0.0, 0.0, 'A');
+INSERT INTO test VALUES (DEFAULT, GeomFromEWKT('SRID=4326;MULTILINESTRING((1 0,0 1,3 2),(3 2,5 4))'), 1, 'multi line string', 'Ü', TRUE, 0.00001, 0, 0.0, 0.0, 'A');
+INSERT INTO test VALUES (DEFAULT, GeomFromEWKT('SRID=4326;POLYGON((0 0,4 0,4 4,0 4,0 0),(1 1, 2 1, 2 2, 1 2,1 1))'), 1, 'polygon', 'Ü', TRUE, 0.00001, 0, 0.0, 0.0, 'A');
+INSERT INTO test VALUES (DEFAULT, GeomFromEWKT('SRID=4326;MULTIPOLYGON(((1 1,3 1,3 3,1 3,1 1),(1 1,2 1,2 2,1 2,1 1)), ((-1 -1,-1 -2,-2 -2,-2 -1,-1 -1)))'), 5432, 'multi ploygon', 'X', TRUE, 999, 0, 0.0, 0.0, 'A');
+INSERT INTO test VALUES (DEFAULT, GeomFromEWKT('SRID=4326;GEOMETRYCOLLECTION(POLYGON((1 1, 2 1, 2 2, 1 2,1 1)),POINT(2 3),LINESTRING(2 3,3 4))'), 8080, 'GEOMETRYCOLLECTION', 'm', TRUE, 9999, 0, 0.0, 0.0, 'A');

test/data/postgis_datasource_tokens_query.xml

@@ -0,0 +1,17 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Map srs="+init=epsg:3857">
+  <Layer name="field_shapes" status="on" srs="+init=epsg:4326">
+    <Datasource>
+      <Parameter name="type">postgis</Parameter>
+      <Parameter name="host">localhost</Parameter>
+      <Parameter name="dbname">node-mapnik-tmp-postgis-test-db</Parameter>
+      <Parameter name="srid">4326</Parameter>
+      <Parameter name="geometry_field">geom</Parameter>
+      <Parameter name="table">
+  (SELECT gid, geom FROM test WHERE gid = @fieldid) as field_shapes
+      </Parameter>
+      <Parameter name="estimate_extent">false</Parameter>
+      <Parameter name="extent">-150,-85,150,85</Parameter>
+    </Datasource>
+  </Layer>
+</Map>

test/vector-tile.postgis.test.js

@@ -0,0 +1,87 @@
+"use strict";
+
+var mapnik = require('../');
+var assert = require('assert');
+var fs = require('fs');
+var path = require('path');
+var spawn = require('child_process').spawn;
+
+var dbname = 'node-mapnik-tmp-postgis-test-db';
+
+var postgis_registered = false;
+var hasPostgisAvailable = false;
+
+describe('mapnik.VectorTile postgis.input', function() {
+
+    before(function(done) {
+        // Check that the postgis plugin is available
+        mapnik.register_datasource(path.join(mapnik.settings.paths.input_plugins,'postgis.input'));
+        postgis_registered = mapnik.datasources().indexOf('postgis') > -1;
+
+        if (postgis_registered) {
+            // Check if postgres is available
+            spawn('psql', ['--version'])
+                .on('error', function(code, signal) {
+                    console.warn('psql --version could not be executed.');
+                    return done();
+                })
+                .on('exit', function(code, signal) {
+                    if (code !== 0) {
+                        console.warn('psql --version returned ' + code);
+                        return done();
+                    }
+                    // Drop the test database if it exists
+                    spawn('dropdb', ['--if-exists', dbname])
+                        .on('exit', function(code, signal) {
+                            if (code !== 0) {
+                                console.warn('dropdb --if-exists ' + dbname + ' returned ' + code);
+                                return done();
+                            }
+                            // Create the test database
+                            spawn('createdb', ['-T', 'template_postgis', dbname])
+                                .on('exit', function(code, signal) {
+                                    if (code !== 0) {
+                                        console.warn('createdb -T template_postgis ' + dbname + 'returned ' + code);
+                                        return done();
+                                    }
+                                    hasPostgisAvailable = true;
+                                    return done();
+                                });
+                        })
+                });
+        } else {
+            console.warn('postgis input datasource not registered.');
+            return done();
+        }
+    });
+
+    it('passes variables to replace tokens in query', function(done) {
+        if (!hasPostgisAvailable) {
+            this.skip('postgis not available');
+        }
+
+        spawn('psql', ['-q', '-f', './test/data/postgis-create-db-and-tables.sql', dbname])
+            .on('exit', function(code, signal) {
+                assert.equal(code, 0, 'could not load data in postgis');
+                        var map = new mapnik.Map(256, 256);
+                map.loadSync('./test/data/postgis_datasource_tokens_query.xml');
+
+                var opts = {};
+                opts.variables = { "fieldid": 2 };
+                map.render(new mapnik.VectorTile(0, 0, 0), opts, function(err, vtile) {
+                    if (err) throw err;
+                    assert(!vtile.empty());
+                    var out = JSON.parse(vtile.toGeoJSON(0));
+                    assert.equal(out.type,'FeatureCollection');
+                    assert.equal(out.features.length,1);
+                    assert.equal(out.features[0].properties.gid, 2);
+                    var coords = out.features[0].geometry.coordinates;
+                    var expected_coords = [-2.0, 2.0]; // From DB, GeomFromEWKT('SRID=4326;POINT(-2 2)')
+                    assert.ok(Math.abs(coords[0] - expected_coords[0]) < 0.3);
+                    assert.ok(Math.abs(coords[1] - expected_coords[1]) < 0.3);
+                    done();
+                });
+            });
+
+    });
+});