From c55c393558363647e917e32ebcbe58643a2bb413 Mon Sep 17 00:00:00 2001 From: Terence Tuhinanshu Date: Wed, 26 Jun 2019 16:07:28 -0400 Subject: [PATCH] Add Windshaft Server components Previously Windshaft included a built-in server. That must now be specified explicitly, as done in https://github.com/OpenTreeMap/otm-tiler/pull/116, and as shown in their examples https://github.com/CartoDB/Windshaft/tree/be55cb931a157be3e7da9e9c810929f48c342789/examples The beforeTileRender and afterTileRender hooks are gone, so that functionality is moved in to the server code. Some of this code has been updated / adapted from the examples to make more sense, and be less redundant. Other parts remain from the examples. It could use a proper overhaul. --- src/tiler/http/mapController.js | 145 ++++++++++++++++ src/tiler/http/windshaftServer.js | 266 ++++++++++++++++++++++++++++++ src/tiler/server.js | 51 +----- 3 files changed, 416 insertions(+), 46 deletions(-) create mode 100644 src/tiler/http/mapController.js create mode 100644 src/tiler/http/windshaftServer.js diff --git a/src/tiler/http/mapController.js b/src/tiler/http/mapController.js new file mode 100644 index 000000000..ea4fa2886 --- /dev/null +++ b/src/tiler/http/mapController.js @@ -0,0 +1,145 @@ +'use strict'; + +var step = require('step'); +var windshaft = require('windshaft'); +var _ = require('underscore'); + +var MapConfig = windshaft.model.MapConfig; +var DummyMapConfigProvider = require('windshaft/lib/windshaft/models/providers/dummy_mapconfig_provider'); +var MapStoreMapConfigProvider = windshaft.model.provider.MapStoreMapConfig; + +/** + * @param app + * @param {MapStore} mapStore + * @param {MapBackend} mapBackend + * @param {TileBackend} tileBackend + * @constructor + */ +function MapController(app, mapStore, mapBackend, tileBackend) { + this._app = app; + this.mapStore = mapStore; + this.mapBackend = mapBackend; + this.tileBackend = tileBackend; +} + +MapController.prototype.register = function(app) { + var tile = this.tile.bind(this); + var cors = this.cors.bind(this); + + app.get(app.base_url + '/:z/:x/:y@:scale_factor?x.:format(png|grid\.json)', tile); + app.get(app.base_url + '/:z/:x/:y.:format(png|grid\.json)', tile); + app.options(app.base_url, cors); +}; + +// send CORS headers when client send options. +MapController.prototype.cors = function(req, res, next) { + this._app.doCORS(res, 'Content-Type'); + return next(); +}; + +MapController.prototype.create = function initLayergroup(req, mapConfig, callback) { + this.mapBackend.createLayergroup( + mapConfig, req.params, new DummyMapConfigProvider(mapConfig, req.params), callback + ); +}; + +// Gets a tile for a given token and set of tile ZXY coords. (OSM style) +MapController.prototype.tile = function(req, res) { + var self = this; + var mapConfig; + + this._app.doCORS(res); + step( + function mapController$prepareParams() { + self._app.req2params(req, this); + }, + function mapController$getMapConfig(err) { + mapConfig = MapConfig.create({ + layers: [{ + type: 'mapnik', + options: { + sql: req.params.sql, + cartocss: req.params.style, + cartocss_version: '2.0.1', + interactivity: req.params.interactivity, + geom_column: 'geom', + } + }] + }); + self.mapStore.load(mapConfig.id(), this); + }, + function mapController$saveMapConfig(err, layer) { + if (layer) { + this(null, layer); + } else { + self.create(req, mapConfig, this); + } + }, + function mapController$getTile(err, layer) { + if ( err ) { + throw err; + } + req.params.token = layer.layergroupid; + self.tileBackend.getTile(new MapStoreMapConfigProvider(self.mapStore, req.params), req.params, this); + }, + function mapController$finalize(err, tile, headers) { + self.finalizeGetTileOrGrid(err, req, res, tile, headers); + self._app.cacheTile(req, tile); + return null; + }, + function finish(err) { + if ( err ) { + // TODO Replace with Rollbar + console.error("windshaft.tiles: " + err); + } + } + ); +}; + +// This function is meant for being called as the very last +// step by all endpoints serving tiles or grids +MapController.prototype.finalizeGetTileOrGrid = function(err, req, res, tile, headers) { + if (err) { + // See https://github.com/Vizzuality/Windshaft-cartodb/issues/68 + var errMsg = err.message ? ( '' + err.message ) : ( '' + err ); + + // Rewrite mapnik parsing errors to start with layer number + var matches = errMsg.match("(.*) in style 'layer([0-9]+)'"); + if (matches) { + errMsg = 'style'+matches[2]+': ' + matches[1]; + } + + var status = err.http_status || statusFromErrorMessage(errMsg); + + this._app.sendError(res, { errors: [errMsg] }, status, 'TILE', err); + } else { + res.status(200) + // Add cache header for 30 days + .set(_.extend(headers, { 'Cache-Control': 'max-age=2592000' })) + .send(tile); + } +}; + +function statusFromErrorMessage(errMsg) { + // Find an appropriate statusCode based on message + var statusCode = 400; + if ( -1 !== errMsg.indexOf('permission denied') ) { + statusCode = 403; + } + else if ( -1 !== errMsg.indexOf('authentication failed') ) { + statusCode = 403; + } + else if (errMsg.match(/Postgis Plugin.*[\s|\n].*column.*does not exist/)) { + statusCode = 400; + } + else if ( -1 !== errMsg.indexOf('does not exist') ) { + if ( -1 !== errMsg.indexOf(' role ') ) { + statusCode = 403; // role 'xxx' does not exist + } else { + statusCode = 404; + } + } + return statusCode; +} + +module.exports = MapController; diff --git a/src/tiler/http/windshaftServer.js b/src/tiler/http/windshaftServer.js new file mode 100644 index 000000000..c6c02158c --- /dev/null +++ b/src/tiler/http/windshaftServer.js @@ -0,0 +1,266 @@ +'use strict'; + +// Adapted from: +// https://github.com/CartoDB/Windshaft/blob/1a9146e33/examples/http/server.js + +var debug = require('debug')('windshaft:server'); +var express = require('express'); +var RedisPool = require('redis-mpool'); +var _ = require('underscore'); +var mapnik = require('@carto/mapnik'); +var aws = require('aws-sdk'); +var rollbar = require('rollbar'); +var stream = require('stream'); + +// Express Middleware +var morgan = require('morgan'); + +var windshaft = require('windshaft'); + +var MapController = require('./mapController.js'); + +// +// @param opts server options object. Example value: +// { +// base_url: '/database/:dbname/table/:table', +// base_url_notable: '/database/:dbname', // @deprecated +// base_url_mapconfig: base_url_notable + '/layergroup', +// req2params: function(req, callback){ +// callback(null,req) +// }, +// grainstore: { +// datasource: { +// user:'postgres', host: '127.0.0.1', +// port: 5432, geometry_field: 'the_geom_webmercator', +// srid: 3857 +// } +// }, //see grainstore npm for other options +// mapnik: { +// metatile: 4, +// bufferSize:64 +// }, +// renderer: { +// // function to use when getTile fails in a renderer, it enables modifying the default behaviour +// onTileErrorStrategy: function(err, tile, headers, stats, format, callback) { +// // allows to change behaviour based on `err` or `format` for instance +// callback(err, file, headers, stats); +// }, +// mapnik: { +// +// }, +// http: { +// +// }, +// }, +// renderCache: { +// ttl: 60000, // seconds +// }, +// redis: { +// host: '127.0.0.1', port: 6379 +// // or 'pool', for a pre-configured pooler +// // with interface of node-redis-mpool +// }, +// https: { +// key: fs.readFileSync('test/fixtures/keys/agent2-key.pem'), +// cert: fs.readFileSync('test/fixtures/keys/agent2-cert.pem') +// } +// } +// +module.exports = function(opts) { + opts = opts || {}; + + opts.grainstore = opts.grainstore || {}; + opts.grainstore.mapnik_version = mapnikVersion(opts); + + validateOptions(opts); + + bootstrapFonts(opts); + + // initialize express server + var app = bootstrap(opts); + addFilters(app, opts); + + var redisPool = makeRedisPool(opts.redis); + + var map_store = new windshaft.storage.MapStore({ + pool: redisPool, + expire_time: opts.grainstore.default_layergroup_ttl + }); + + opts.renderer = opts.renderer || {}; + + var rendererFactory = new windshaft.renderer.Factory({ + onTileErrorStrategy: opts.renderer.onTileErrorStrategy, + mapnik: { + grainstore: opts.grainstore, + mapnik: opts.renderer.mapnik || opts.mapnik + }, + torque: opts.renderer.torque, + http: opts.renderer.http + }); + + // initialize render cache + var rendererCacheOpts = _.defaults(opts.renderCache || {}, { + ttl: 60000, // 60 seconds TTL by default + statsInterval: 60000 // reports stats every milliseconds defined here + }); + var rendererCache = new windshaft.cache.RendererCache(rendererFactory, rendererCacheOpts); + + var attributesBackend = new windshaft.backend.Attributes(); + var tileBackend = new windshaft.backend.Tile(rendererCache); + var mapValidatorBackend = new windshaft.backend.MapValidator(tileBackend, attributesBackend); + var mapBackend = new windshaft.backend.Map(rendererCache, map_store, mapValidatorBackend); + + app.sendError = function(res, err, statusCode, label, tolog) { + var olabel = '['; + if ( label ) { + olabel += label + ' '; + } + olabel += 'ERROR]'; + if ( ! tolog ) { + tolog = err; + } + var log_msg = olabel + ' -- ' + statusCode + ': ' + tolog; + //if ( tolog.stack ) log_msg += '\n' + tolog.stack; + debug(log_msg); // use console.log for statusCode != 500 ? + // If a callback was requested, force status to 200 + if ( res.req ) { + // NOTE: res.req can be undefined when we fake a call to + // ourself from POST to /layergroup + if ( res.req.query.callback ) { + statusCode = 200; + } + } + + res.status(statusCode).send(err); + }; + + app.cacheTile = function(req, tile) { + try { + // Skip caching if environment not setup for it + if (req.headers.host === 'localhost' || !opts.s3Cache.bucket) { + return; + } + + var cleanUrl = req.url[0] === '/' ? req.url.substr(1) : req.url, + s3Obj = new aws.S3({params: {Bucket: opts.s3Cache.bucket, Key: cleanUrl}}), + body; + + if (Buffer.isBuffer(tile)) { + body = new stream.PassThrough(); + body.end(tile); + } else { + body = JSON.stringify(tile); + } + + if (body) { + s3Obj.upload({Body: body}, function(err) { + if (err) { + throw (err); + } + }); + } + } catch (ex) { + rollbar.handleError(ex, req); + } + }; + + /******************************************************************************************************************* + * Routing + ******************************************************************************************************************/ + + var mapController = new MapController(app, map_store, mapBackend, tileBackend, attributesBackend); + mapController.register(app); + + /******************************************************************************************************************* + * END Routing + ******************************************************************************************************************/ + + // temporary measure until we upgrade to newer version expressjs so we can check err.status + app.use(function(err, req, res, next) { + if (err) { + if (err.name === 'SyntaxError') { + app.sendError(res, { errors: [err.name + ': ' + err.message] }, 400, 'JSON', err); + } else { + next(err); + } + } else { + next(); + } + }); + + return app; +}; + +function validateOptions(opts) { + if (!_.isString(opts.base_url) || !_.isFunction(opts.req2params)) { + throw new Error('Must initialise Windshaft with: "base_url" URL and req2params function'); + } + + // Be nice and warn if configured mapnik version is != instaled mapnik version + if (mapnik.versions.mapnik !== opts.grainstore.mapnik_version) { + console.warn('WARNING: detected mapnik version (' + mapnik.versions.mapnik + ')' + + ' != configured mapnik version (' + opts.grainstore.mapnik_version + ')'); + } +} + +function makeRedisPool(redisOpts) { + redisOpts = redisOpts || {}; + return redisOpts.pool || new RedisPool(_.extend(redisOpts, {name: 'windshaft:server'})); +} + +function bootstrapFonts(opts) { + // Set carto renderer configuration for MMLStore + opts.grainstore.carto_env = opts.grainstore.carto_env || {}; + var cenv = opts.grainstore.carto_env; + cenv.validation_data = cenv.validation_data || {}; + if ( ! cenv.validation_data.fonts ) { + mapnik.register_system_fonts(); + mapnik.register_default_fonts(); + cenv.validation_data.fonts = _.keys(mapnik.fontFiles()); + } +} + +function bootstrap(opts) { + var app; + if (_.isObject(opts.https)) { + // use https if possible + app = express(opts.https); + } else { + // fall back to http by default + app = express(); + } + app.enable('jsonp callback'); + + if (opts.log_format) { + app.use(morgan(opts.log_format)); + } + + return app; +} + +// set default before/after filters if not set in opts object +function addFilters(app, opts) { + + // Extend windshaft with all the elements of the options object + _.extend(app, opts); + + // filters can be used for custom authentication, caching, logging etc + _.defaults(app, { + // Enable CORS access by web browsers if set + doCORS: function(res, extraHeaders) { + if (opts.enable_cors) { + var baseHeaders = 'X-Requested-With, X-Prototype-Version, X-CSRF-Token'; + if(extraHeaders) { + baseHeaders += ', ' + extraHeaders; + } + res.header('Access-Control-Allow-Origin', '*'); + res.header('Access-Control-Allow-Headers', baseHeaders); + } + } + }); +} + +function mapnikVersion(opts) { + return opts.grainstore.mapnik_version || mapnik.versions.mapnik; +} diff --git a/src/tiler/server.js b/src/tiler/server.js index 78a23a88a..0184f7087 100644 --- a/src/tiler/server.js +++ b/src/tiler/server.js @@ -1,5 +1,4 @@ -var aws = require('aws-sdk'), - Windshaft = require('windshaft'), +var WindshaftServer = require('./http/windshaftServer'), healthCheck = require('./healthCheck'), rollbar = require('rollbar'), fs = require('fs'), @@ -201,49 +200,9 @@ var config = { enable_cors: true, - beforeTileRender: function(req, res, callback) { - try { - callback(null); - } catch (ex) { - rollbar.handleError(ex, req); - callback(ex); - } - }, - - afterTileRender: function(req, res, tile, headers, callback) { - try { - // Complete render pipline first, add cache header for - // 30 days - headers['Cache-Control'] = 'max-age=2592000'; - callback(null, tile, headers); - - // Check if the environment is set up to cache tiles - if (!shouldCacheRequest(req)) { return; } - - var cleanUrl = req.url[0] === '/' ? req.url.substr(1) : req.url, - s3Obj = new aws.S3({params: {Bucket: tileCacheBucket, Key: cleanUrl}}), - body; - - if (Buffer.isBuffer(tile)) { - body = new stream.PassThrough(); - body.end(tile); - } else { - body = JSON.stringify(tile); - } - - if (body) { - s3Obj.upload({Body: body}, function(err, data) { - if (err) { - throw (err); - } - }); - } - - callback(null); - } catch (ex) { - rollbar.handleError(ex, req); - callback(ex, null); - } + // Custom config used for caching tiles to S3 + s3Cache: { + bucket: tileCacheBucket, }, req2params: function(req, callback) { @@ -279,7 +238,7 @@ var config = { }; // Initialize tile server on port 4000 -var ws = new Windshaft.Server(config); +var ws = new WindshaftServer(config); ws.get('/health-check', healthCheck(config)); ws.listen(4000); ws.use(rollbar.errorHandler(rollbarAccessToken, {environment: stackType}));