Add support for raster layers (#191)

Requires unofficial grainstore version.
Includes small testcase.

NEWS

@@ -1,3 +1,7 @@
+ New features:
+
+ - Add support for raster layers (#190)
+
  Bug fixes:
 
  - Support specifying column name in MapConfig (#191)

doc/MapConfig-1.2.0.md

@@ -0,0 +1,168 @@
+# 1. Purpose
+
+This specification describes 
+[MapConfig](MapConfig-specification) format version 1.2.0.
+
+
+# 2. File format
+
+Layergroup files use the JSON format as described in [RFC 4627](http://www.ietf.org/rfc/rfc4627.txt).
+
+```javascript
+{
+        // OPTIONAL
+        // default map extent, in map projection
+        // (only webmercator supported at this version)
+        extent: [-20037508.5, -20037508.5, 20037508.5, 20037508.5],
+
+        // OPTIONAL
+        // Spatial reference identifier for the map
+        // Defaults to 3857
+        srid: 3857,
+
+        // OPTIONAL
+        // maxzoom to be renderer. From this zoom tiles will respond 404
+        // default: undefined (infinite)
+        maxzoom: 18,
+
+        // OPTIONAL
+        // minzoom to be renderer. From this zoom tiles will respond 404. Must be less than maxzoom
+        // default: 0
+        minzoom:3,
+
+        // OPTIONAL
+        // global CartoCSS, is prepend in cartocss generated when the full configuration is rendered
+        // takes precedence over per-layer cartocss setting
+        global_cartocss:'#layer0{} #layer1{}...', 
+
+        // OPTIONAL
+        // global CartoCSS version, takes precedence over per-layer setting
+        global_cartocss_version: '2.0.1', // optional,
+
+        // REQUIRED
+        // Array of layers defined in render order. Different kind of layers supported
+        // are described below
+        layers: [{
+
+             // REQUIRED
+             // string, sets layer type, can take 3 values:
+             //  - 'mapnik'  - rasterize tiles
+             //  - 'cartodb' - an alias for mapnik, for backward compatibility
+             //  - 'torque'  - render vector tiles in torque format (to be linked)
+             type: 'mapnik',
+
+             // REQUIRED
+             // object, set different options for each layer type, there are 3 common mandatory attributes
+             options: {
+               // REQUIRED
+               // string, SQL to be performed on user database to fetch the data to be rendered.
+               //
+               // It should select at least the columns specified in ``geom_column``, 
+               // ``interactivity`` and  ``attributes`` configurations below.
+               //
+               // For ``mapnik`` layers it can contain substitution tokens !bbox!,
+               // !pixel_width! and !pixel_height!, see implication of that in the 
+               // ``attributes`` configuration below.
+               // 
+               sql: 'select * from table',
+
+               // OPTIONAL
+               // name of the column containing the geometry
+               // Defaults to 'the_geom_webmercator'
+               geom_column: 'the_geom_webmercator',
+
+               // OPTIONAL
+               // type of column, can be 'geometry' or 'raster'
+               // Defaults to 'geometry'
+               geom_type: 'geometry',
+
+               // OPTIONAL
+               // raster band, only valid when geom_type = 'raster'.
+               // If 0 or not specified makes rasters being interpreted
+               // as either grayscale (for single bands) or RGB (for 3 bands)
+               // or RGBA (for 4 bands).
+               // Defaults to 0
+               raster_band: '1',
+
+               // OPTIONAL
+               // spatial reference identifier of the geometry column
+               // Defaults to 3857
+               srid: 3857,
+
+               // REQUIRED
+               // string, CartoCSS style to render the tiles
+               //
+               // CartoCSS specification depend on layer type:
+               //  Torque: http://github.com/CartoDB/torque/blob/2.2.00/lib/torque/cartocss_reference.js
+               //  Mapnik: http://github.com/mapnik/mapnik-reference/blob/v5.0.7/2.2.0/reference.json
+               cartocss: '#layer { ... }',
+
+               // REQUIRED
+               // string, CartoCSS style version of cartocss attribute
+               // global_cartocss_version takes precedence over this, if present
+               //
+               // Version semantic is specific to the layer type.
+               //
+               cartocss_version: '2.0.1',
+
+               // OPTIONAL
+               // string array, contains tables that SQL uses. It used when affected tables can't be
+               // guessed from SQL (for example, plsql functions are used)
+               affected_tables: [ 'table1', 'schema.table2', '"MixedCase"."Table"' ],
+
+               // OPTIONAL
+               // string array, contains fields renderer inside grid.json
+               // all the params should be exposed by the results of executing the query in sql attribute
+               interactivity: [ 'field1', 'field2', .. ]
+
+               // OPTIONAL
+               // values returned by attributes service (disabled if no config is given)
+               // NOTE: enabling the attribute service is forbidden if the "sql" option contains
+               //       substitution token that make it dependent on zoom level or viewport extent.
+               attributes: {
+                 // REQUIRED
+                 // used as key value to fetch columns
+                 id: 'identifying_column', 
+
+                 // REQUIRED
+                 // string list of columns returned by attributes service
+                 columns: ['column1', 'column2']
+               }
+             }
+        }]
+    }
+```
+# Extensions
+
+The document may be extended for specific uses.
+For example, Windshaft-CartoDB defines the addition of a "stats_tag" element
+in the config. See https://github.com/CartoDB/Windshaft-cartodb/wiki/MultiLayer-API
+
+Specification for how to name extensions is yet to be defined as of this version
+of MapConfig.
+
+# TODO
+
+ - Allow for each layer to specify the name of the geometry column to use for tiles
+ - Allow to specify layer projection/srid and map projection/srid
+ - Allow to specify quadtree configuration (max extent, mostly)
+ - Link to a document describing "CartoCSS" version (ie: what's required for torque etc.)
+
+# History
+
+## 1.2.0
+
+ - Add support for 'geom_type' and 'raster_band' in 'mapnik' type layers
+
+## 1.1.0
+
+ - Add support for 'torque' type layers
+ - Add support for 'attributes' specification
+
+## 1.0.1
+
+ - Layer.options.interactivity became an array (from a string)
+
+## 1.0.0
+
+ - Initial version

lib/windshaft/render_cache.js

@@ -183,6 +183,8 @@ module.exports = function(timeout, mml_store, map_store, mapnik_opts) {
         var sql = [];
         var style = [];
         var geom_column = [];
+        var column_type = [];
+        var extra_ds_opts = [];
         var interactivity = [];
         var style_version = cfg.hasOwnProperty('global_cartocss_version') ? cfg.global_cartocss_version : [];
         for ( var i=0; i<cfg.layers.length; ++i ) {
@@ -210,11 +212,18 @@ module.exports = function(timeout, mml_store, map_store, mapnik_opts) {
           } 
           interactivity.push(lyropt.interactivity);
           geom_column.push( lyropt['geom_column'] ); // possibly undefined
+          column_type.push( lyropt['geom_type'] ); // possibly undefined
+          extra_opt = {};
+          if ( lyropt.hasOwnProperty('raster_band') ) {
+            extra_opt['band'] = lyropt['raster_band'];
+          }
+          extra_ds_opts.push( extra_opt );
         }
         if ( ! sql.length ) throw new Error("No 'mapnik' layers in MapConfig");
         return { sql:sql, style:style, style_version:style_version,
                  interactivity:interactivity, ttl:0,
-                 gcols:geom_column };
+                 gcols:geom_column, gcoltypes:column_type,
+                 extra_ds_opts:extra_ds_opts };
     };
 
     function TileliveAdaptor(renderer, format) {

package.json

@@ -24,7 +24,7 @@
     "dependencies": {
         "underscore": "~1.3",
         "step": "~0.0.5",
-        "grainstore": "http://github.com/CartoDB/grainstore/tarball/4d560ab1",
+        "grainstore": "git://github.com/strk/grainstore.git#ee2d90b2",
         "generic-pool": "~2.0.3",
         "express": "~2.5.11",
         "tilelive": "~4.4.2",

test/acceptance/raster.js

@@ -0,0 +1,153 @@
+// FLUSHALL Redis before starting
+
+var   assert        = require('../support/assert')
+    , tests         = module.exports = {}
+    , _             = require('underscore')
+    , querystring   = require('querystring')
+    , fs            = require('fs')
+    , redis         = require('redis')
+    , th            = require('../support/test_helper')
+    , Step          = require('step')
+    , mapnik        = require('mapnik')
+    , Windshaft     = require('../../lib/windshaft')
+    , ServerOptions = require('../support/server_options')
+    , http          = require('http');
+
+suite('raster', function() {
+
+    ////////////////////////////////////////////////////////////////////
+    //
+    // SETUP
+    //
+    ////////////////////////////////////////////////////////////////////
+
+    var server = new Windshaft.Server(ServerOptions);
+    server.setMaxListeners(0);
+    var redis_client = redis.createClient(ServerOptions.redis.port);
+
+    checkCORSHeaders = function(res) {
+      var h = res.headers['access-control-allow-headers'];
+      assert.ok(h);
+      assert.equal(h, 'X-Requested-With, X-Prototype-Version, X-CSRF-Token');
+      var h = res.headers['access-control-allow-origin'];
+      assert.ok(h);
+      assert.equal(h, '*');
+    };
+
+    var IMAGE_EQUALS_TOLERANCE_PER_MIL = 25;
+
+    suiteSetup(function(done) {
+
+      // Check that we start with an empty redis db 
+      redis_client.keys("*", function(err, matches) {
+          if ( err ) { done(err); return; }
+          assert.equal(matches.length, 0, "redis keys present at setup time:\n" + matches.join("\n"));
+          done();
+      });
+
+    });
+
+    test("can render raster for valid mapconfig", function(done) {
+
+      var mapconfig =  {
+        version: '1.1.0',
+        layers: [
+           { type: 'mapnik', options: {
+               sql: "select ST_AsRaster(" +
+                    " ST_MakeEnvelope(-100,-40, 100, 40, 4326), " +
+                    " 1.0, -1.0, '8BUI', 127) as rst",
+               geom_column: 'rst',
+               geom_type: 'raster',
+               cartocss: '#layer { raster-opacity:1.0 }',
+               cartocss_version: '2.0.1'
+             } }
+        ]
+      };
+      var expected_token; 
+      Step(
+        function do_post()
+        {
+          var next = this;
+          assert.response(server, {
+              url: '/database/windshaft_test/layergroup',
+              method: 'POST',
+              headers: {'Content-Type': 'application/json' },
+              data: JSON.stringify(mapconfig)
+          }, {}, function(res, err) { next(err, res); });
+        },
+        function checkPost(err, res) {
+          if ( err ) throw err;
+          assert.equal(res.statusCode, 200, res.statusCode + ': ' + res.body);
+          // CORS headers should be sent with response
+          // from layergroup creation via POST
+          checkCORSHeaders(res);
+          var parsedBody = JSON.parse(res.body);
+          if ( expected_token ) assert.deepEqual(parsedBody, {layergroupid: expected_token, layercount: 2});
+          else expected_token = parsedBody.layergroupid;
+          return null;
+        },
+        function do_get_tile(err)
+        {
+          if ( err ) throw err;
+          var next = this;
+          assert.response(server, {
+              url: '/database/windshaft_test/layergroup/' + expected_token + '/0/0/0.png',
+              method: 'GET',
+              encoding: 'binary'
+          }, {}, function(res, err) { next(err, res); });
+        },
+        function check_response(err, res) {
+          if ( err ) throw err;
+          assert.equal(res.statusCode, 200, res.body);
+          assert.deepEqual(res.headers['content-type'], "image/png");
+          var next = this;
+          assert.imageEqualsFile(res.body,
+            './test/fixtures/raster_gray_rect.png',
+            IMAGE_EQUALS_TOLERANCE_PER_MIL, function(err) {
+              try {
+                if (err) throw err;
+                next();
+              } catch (err) { next(err); }
+            });
+        },
+        function finish(err) {
+          var errors = [];
+          if ( err ) errors.push(''+err);
+          redis_client.exists("map_cfg|" +  expected_token, function(err, exists) {
+              if ( err ) errors.push(err.message);
+              //assert.ok(exists, "Missing expected token " + expected_token + " from redis");
+              redis_client.del("map_cfg|" +  expected_token, function(err) {
+                if ( err ) errors.push(err.message);
+                if ( errors.length ) done(new Error(errors));
+                else done(null);
+              });
+          });
+        }
+      );
+    });
+
+    ////////////////////////////////////////////////////////////////////
+    //
+    // TEARDOWN
+    //
+    ////////////////////////////////////////////////////////////////////
+
+    suiteTeardown(function(done) {
+
+      // Check that we left the redis db empty
+      redis_client.keys("*", function(err, matches) {
+          try {
+            assert.equal(matches.length, 0, "Left over redis keys:\n" + matches.join("\n"));
+          } catch (err2) {
+            if ( err ) err.message += '\n' + err2.message;
+            else err = err2;
+          }
+          redis_client.flushall(function() {
+            done(err);
+          });
+      });
+
+    });
+
+});
+