diff --git a/.travis.yml b/.travis.yml index 772433e13..0582a5063 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,18 +1,18 @@ -before_script: +before_install: - lsb_release -a - - sudo mv /etc/apt/sources.list.d/pgdg-source.list* /tmp + - sudo mv /etc/apt/sources.list.d/pgdg.list* /tmp - sudo apt-get -qq purge postgis* postgresql* - sudo rm -Rf /var/lib/postgresql /etc/postgresql - - sudo apt-add-repository --yes ppa:cartodb/postgresql-9.3 + - sudo apt-add-repository --yes ppa:cartodb/postgresql-9.5 - sudo apt-add-repository --yes ppa:cartodb/gis - sudo apt-get update - - sudo apt-get install -q postgresql-9.3-postgis-2.1 - - sudo apt-get install -q postgresql-contrib-9.3 - - sudo apt-get install -q postgresql-plpython-9.3 + - sudo apt-get install -q postgresql-9.5-postgis-2.2 + - sudo apt-get install -q postgresql-contrib-9.5 + - sudo apt-get install -q postgresql-plpython-9.5 - sudo apt-get install -q postgis - sudo apt-get install -q gdal-bin - sudo apt-get install -q ogr2ogr2-static-bin - - echo -e "local\tall\tall\ttrust\nhost\tall\tall\t127.0.0.1/32\ttrust\nhost\tall\tall\t::1/128\ttrust" |sudo tee /etc/postgresql/9.3/main/pg_hba.conf + - echo -e "local\tall\tall\ttrust\nhost\tall\tall\t127.0.0.1/32\ttrust\nhost\tall\tall\t::1/128\ttrust" |sudo tee /etc/postgresql/9.5/main/pg_hba.conf - sudo service postgresql restart - psql -c 'create database template_postgis;' -U postgres - psql -c 'CREATE EXTENSION postgis;' -U postgres -d template_postgis diff --git a/Makefile b/Makefile index acc116a26..ba88ea3d8 100644 --- a/Makefile +++ b/Makefile @@ -13,9 +13,11 @@ jshint: @echo "***jshint***" @./node_modules/.bin/jshint app/ batch/ test/ app.js -TEST_SUITE := $(shell find test/{acceptance,unit} -name "*.js") +TEST_SUITE := $(shell find test/{unit,integration,acceptance} -name "*.js") TEST_SUITE_UNIT := $(shell find test/unit -name "*.js") +TEST_SUITE_INTEGRATION := $(shell find test/integration -name "*.js") TEST_SUITE_ACCEPTANCE := $(shell find test/acceptance -name "*.js") +TEST_SUITE_BATCH := $(shell find test/*/batch -name "*.js") test: @echo "***tests***" @@ -25,10 +27,18 @@ test-unit: @echo "***unit tests***" @$(SHELL) test/run_tests.sh ${RUNTESTFLAGS} $(TEST_SUITE_UNIT) +test-integration: + @echo "***integration tests***" + @$(SHELL) test/run_tests.sh ${RUNTESTFLAGS} $(TEST_SUITE_INTEGRATION) + test-acceptance: @echo "***acceptance tests***" @$(SHELL) test/run_tests.sh ${RUNTESTFLAGS} $(TEST_SUITE_ACCEPTANCE) +test-batch: + @echo "***batch queries tests***" + @$(SHELL) test/run_tests.sh ${RUNTESTFLAGS} $(TEST_SUITE_BATCH) + test-all: jshint test coverage: diff --git a/NEWS.md b/NEWS.md index 357be777e..50534806c 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,3 +1,217 @@ +1.42.3 - 2016-11-07 +------------------- + +Announcements: + * Raise payload limit for batch-queries to 16kb. + + +1.42.2 - 2016-11-07 +------------------- + +Bug fixes: + * Improve error handling while registering jobs to be tracked. + + +1.42.1 - 2016-11-03 +------------------- + +Bug fixes: + * Avoid to use SCAN command to find work-in-progress queues. + + +1.42.0 - 2016-11-02 +------------------- + +Announcements: + * Adds endpoint to check running batch queries + + +1.41.0 - 2016-10-21 +------------------- + +Announcements: + * Stop migrating old queues by default. + +Bug fixes: + * Fix some scenarios where batch queries got stuck waiting for available slots. + + +1.40.0 - 2016-10-20 +------------------- + +New features: + * Batch queries are handled per db host. + - There is an scheduler controlling how many queries and in what order they are run. + - Priority is based on: number of queries already ran, and oldest user in queue. + * Batch queries capacity: allow to configure how many jobs to run per db host. + + +1.39.1 - 2016-10-17 +------------------- + +Enhancements: + * Log creation and waiting time for fallback jobs' queries. + + +1.39.0 - 2016-10-17 +------------------- + +Enhancements: + * Use just one Redis pool across the whole application. + +New features: + * Batch queries use per user-queues. + * Batch queries queues can limit the number of queued jobs per user. + - Default is 64 jobs. + - Configuration key `batch_max_queued_jobs` allows to modify the limit. + + +1.38.2 - 2016-10-13 +------------------- + +Bug fixes: + * Batch queries: release redis clients to pool from locker and seeker. + + +1.38.1 - 2016-10-13 +------------------- + +Enhancements: + * Batch queries: improvements over leader locking. + + +1.38.0 - 2016-10-11 +------------------- + +Announcements: + * Allow to set statement timeout per query in multi query batch queries. + * Batch queries default statement timeout set to 12 hours. + * Multiple queries jobs pushed as first job between queries. + + +1.37.1 - 2016-10-05 +------------------- + +Bug fixes: + * Body parser accepting multipart requests. + + +1.37.0 - 2016-10-04 +------------------- + +Enhancements: + * Migrate to Express.js 4.x series. + + +1.36.2 - 2016-10-03 +------------------- + +Bug fixes: + - Batch Queries logs: use path instead of stream to be able to reopen FD. + + +1.36.1 - 2016-09-30 +------------------- + +Enhancements: + * Tag fallback jobs logs. + + +1.36.0 - 2016-09-30 +------------------- + +New features: + * Log queries from batch fallback jobs. + +Enhancements: + * assert.response following callback(err, obj) pattern. + + +1.35.0 - 2016-09-15 +------------------- + +New features: + * Allow to use `--config /path/to/config.js` to specify configuration file. + - Environment will be loaded from config file if `environment` key is present, otherwise it keeps current behaviour. + +Bug fixes: + * Allow to use absolute paths for log files. + +Announcements: + * Removes support for optional rollbar logging. + + +1.34.2 - 2016-08-30 +------------------- + +Announcements: + * Upgrades cartodb-redis to 0.13.1. + * Set TTL of finished job to 2h + + +1.34.1 - 2016-07-11 +------------------- + +Bug fixes: + * Fixed issue with redis connections in Batch API #326 + + +1.34.0 - 2016-07-11 +------------------- + +New features: + * Skip tables with no updated_at registered in cdb_tablemetadata. + * Allow to setup more than one domain to validate oauth against. + + +1.33.0 - 2016-07-01 +------------------- + +New features: + * Add `<%= job_id %>` template support for onerror and onsuccess fallback queries. + + +1.32.0 - 2016-06-30 +------------------- + +New features: + * Broadcast after enqueueing jobs to improve query distribution load. + * Batch pub-sub channel handles its connections using `redis-mpool`. + + +1.31.0 - 2016-06-29 +------------------- + +New features: + * Adds start and end time for batch queries with fallback. + * Add `<%= error_message %>` template support for onerror fallback queries. + + +1.30.1 - 2016-06-23 +------------------- + +Bug fixes: + * Fixed issue with profiling in Batch API #318 + + +1.30.0 - 2016-06-14 +------------------- + +Announcements: + * Now Batch API sends stats metrics to statsd server #312 + * Now Batch API sets "skipped" instead of "pending" to queries that won't be performed #311 + + Bug fixes: + * Fixed issue with error handling in Batch API #316 + + +1.29.2 - 2016-05-25 +------------------- + +Bug fixes: + * Fixed issue with status transition in fallback jobs #308 + + 1.29.1 - 2016-05-24 ------------------- diff --git a/app.js b/app.js index e62e040ff..b65758d03 100755 --- a/app.js +++ b/app.js @@ -9,65 +9,67 @@ * environments: [development, test, production] * */ -var _ = require('underscore'); var fs = require('fs'); var path = require('path'); -var ENV = process.env.NODE_ENV || 'development'; - -if (process.argv[2]) { - ENV = process.argv[2]; +var argv = require('yargs') + .usage('Usage: $0 [options]') + .help('h') + .example( + '$0 production -c /etc/sql-api/config.js', + 'start server in production environment with /etc/sql-api/config.js as config file' + ) + .alias('h', 'help') + .alias('c', 'config') + .nargs('c', 1) + .describe('c', 'Load configuration from path') + .argv; + +var environmentArg = argv._[0] || process.env.NODE_ENV || 'development'; +var configurationFile = path.resolve(argv.config || './config/environments/' + environmentArg + '.js'); +if (!fs.existsSync(configurationFile)) { + console.error('Configuration file "%s" does not exist', configurationFile); + process.exit(1); } -process.env.NODE_ENV = ENV; +global.settings = require(configurationFile); +var ENVIRONMENT = argv._[0] || process.env.NODE_ENV || global.settings.environment; +process.env.NODE_ENV = ENVIRONMENT; var availableEnvironments = ['development', 'production', 'test', 'staging']; // sanity check arguments -if (availableEnvironments.indexOf(ENV) === -1) { - console.error("\nnode app.js [environment]"); - console.error("environments: " + availableEnvironments.join(', ')); +if (availableEnvironments.indexOf(ENVIRONMENT) === -1) { + console.error("node app.js [environment]"); + console.error("Available environments: " + availableEnvironments.join(', ')); process.exit(1); } -// set Node.js app settings and boot -global.settings = require(__dirname + '/config/settings'); -var env = require(__dirname + '/config/environments/' + ENV); -env.api_hostname = require('os').hostname().split('.')[0]; -_.extend(global.settings, env); +global.settings.api_hostname = require('os').hostname().split('.')[0]; global.log4js = require('log4js'); -var log4js_config = { - appenders: [], - replaceConsole:true +var log4jsConfig = { + appenders: [], + replaceConsole: true }; -if ( env.log_filename ) { - var logdir = path.dirname(env.log_filename); - // See cwd inlog4js.configure call below - logdir = path.resolve(__dirname, logdir); - if ( ! fs.existsSync(logdir) ) { - console.error("Log filename directory does not exist: " + logdir); +if ( global.settings.log_filename ) { + var logFilename = path.resolve(global.settings.log_filename); + var logDirectory = path.dirname(logFilename); + if (!fs.existsSync(logDirectory)) { + console.error("Log filename directory does not exist: " + logDirectory); process.exit(1); } - console.log("Logs will be written to " + env.log_filename); - log4js_config.appenders.push( - { type: "file", filename: env.log_filename } + console.log("Logs will be written to " + logFilename); + log4jsConfig.appenders.push( + { type: "file", absolute: true, filename: logFilename } ); } else { - log4js_config.appenders.push( + log4jsConfig.appenders.push( { type: "console", layout: { type:'basic' } } ); } - -if ( global.settings.rollbar ) { - log4js_config.appenders.push({ - type: __dirname + "/app/models/log4js_rollbar.js", - options: global.settings.rollbar - }); -} - -global.log4js.configure(log4js_config, { cwd: __dirname }); +global.log4js.configure(log4jsConfig); global.logger = global.log4js.getLogger(); @@ -78,12 +80,14 @@ if ( ! global.settings.base_url ) { var version = require("./package").version; -var app = require(global.settings.app_root + '/app/app')(); -app.listen(global.settings.node_port, global.settings.node_host, function() { - console.log( - "CartoDB SQL API %s listening on %s:%s with base_url %s (%s)", - version, global.settings.node_host, global.settings.node_port, global.settings.base_url, ENV - ); +var server = require('./app/server')(); +var listener = server.listen(global.settings.node_port, global.settings.node_host); +listener.on('listening', function() { + console.info('Using configuration file "%s"', configurationFile); + console.log( + "CartoDB SQL API %s listening on %s:%s PID=%d (%s)", + version, global.settings.node_host, global.settings.node_port, process.pid, ENVIRONMENT + ); }); process.on('uncaughtException', function(err) { @@ -92,15 +96,19 @@ process.on('uncaughtException', function(err) { process.on('SIGHUP', function() { global.log4js.clearAndShutdownAppenders(function() { - global.log4js.configure(log4js_config); + global.log4js.configure(log4jsConfig); global.logger = global.log4js.getLogger(); console.log('Log files reloaded'); }); + + if (server.batch && server.batch.logger) { + server.batch.logger.reopenFileStreams(); + } }); process.on('SIGTERM', function () { - app.batch.stop(); - app.batch.drain(function (err) { + server.batch.stop(); + server.batch.drain(function (err) { if (err) { console.log('Exit with error'); return process.exit(1); diff --git a/app/auth/oauth.js b/app/auth/oauth.js index 6c8809c27..9370fe58e 100644 --- a/app/auth/oauth.js +++ b/app/auth/oauth.js @@ -3,6 +3,8 @@ var _ = require('underscore'); var OAuthUtil = require('oauth-client'); var step = require('step'); var assert = require('assert'); +var CdbRequest = require('../models/cartodb_request'); +var cdbReq = new CdbRequest(); var oAuth = (function(){ var me = { @@ -60,79 +62,87 @@ var oAuth = (function(){ return removed; }; + me.getAllowedHosts= function() { + var oauthConfig = global.settings.oauth || {}; + return oauthConfig.allowedHosts || ['carto.com', 'cartodb.com']; + }; // do new fancy get User ID me.verifyRequest = function(req, metadataBackend, callback) { var that = this; //TODO: review this var httpProto = req.protocol; - var passed_tokens; - var ohash; + if(!httpProto || (httpProto !== 'http' && httpProto !== 'https')) { + var msg = "Unknown HTTP protocol " + httpProto + "."; + var unknownProtocolErr = new Error(msg); + unknownProtocolErr.http_status = 500; + return callback(unknownProtocolErr); + } + + var username = cdbReq.userByReq(req); + var requestTokens; var signature; step( function getTokensFromURL(){ return oAuth.parseTokens(req); }, - function getOAuthHash(err, data){ + function getOAuthHash(err, _requestTokens) { assert.ifError(err); // this is oauth request only if oauth headers are present - this.is_oauth_request = !_.isEmpty(data); + this.is_oauth_request = !_.isEmpty(_requestTokens); if (this.is_oauth_request) { - passed_tokens = data; - that.getOAuthHash(metadataBackend, passed_tokens.oauth_token, this); + requestTokens = _requestTokens; + that.getOAuthHash(metadataBackend, requestTokens.oauth_token, this); } else { return null; } }, - function regenerateSignature(err, data){ + function regenerateSignature(err, oAuthHash){ assert.ifError(err); if (!this.is_oauth_request) { return null; } - ohash = data; - var consumer = OAuthUtil.createConsumer(ohash.consumer_key, ohash.consumer_secret); - var access_token = OAuthUtil.createToken(ohash.access_token_token, ohash.access_token_secret); + var consumer = OAuthUtil.createConsumer(oAuthHash.consumer_key, oAuthHash.consumer_secret); + var access_token = OAuthUtil.createToken(oAuthHash.access_token_token, oAuthHash.access_token_secret); var signer = OAuthUtil.createHmac(consumer, access_token); var method = req.method; - var host = req.headers.host; - - if(!httpProto || (httpProto !== 'http' && httpProto !== 'https')) { - var msg = "Unknown HTTP protocol " + httpProto + "."; - err = new Error(msg); - err.http_status = 500; - callback(err); - return; - } - - var path = httpProto + '://' + host + req.path; - that.splitParams(req.query); - - // remove signature from passed_tokens - signature = passed_tokens.oauth_signature; - delete passed_tokens.oauth_signature; - - var joined = {}; + var hostsToValidate = {}; + var requestHost = req.headers.host; + hostsToValidate[requestHost] = true; + that.getAllowedHosts().forEach(function(allowedHost) { + hostsToValidate[username + '.' + allowedHost] = true; + }); + that.splitParams(req.query); // remove oauth_signature from body if(req.body) { delete req.body.oauth_signature; } - _.extend(joined, req.body ? req.body : null); - _.extend(joined, passed_tokens); - _.extend(joined, req.query); - - return signer.sign(method, path, joined); + signature = requestTokens.oauth_signature; + // remove signature from requestTokens + delete requestTokens.oauth_signature; + var requestParams = _.extend({}, req.body, requestTokens, req.query); + + var hosts = Object.keys(hostsToValidate); + var requestSignatures = hosts.map(function(host) { + var url = httpProto + '://' + host + req.path; + return signer.sign(method, url, requestParams); + }); + + return requestSignatures.reduce(function(validSignature, requestSignature) { + if (signature === requestSignature && !_.isUndefined(requestSignature)) { + validSignature = true; + } + return validSignature; + }, false); }, - function checkSignature(err, data){ - assert.ifError(err); - - //console.log(data + " should equal the provided signature: " + signature); - callback(err, (signature === data && !_.isUndefined(data)) ? true : null); + function finishValidation(err, hasValidSignature) { + return callback(err, hasValidSignature || null); } ); }; diff --git a/app/controllers/health_check_controller.js b/app/controllers/health_check_controller.js index 29458cf98..a5f84f122 100644 --- a/app/controllers/health_check_controller.js +++ b/app/controllers/health_check_controller.js @@ -24,11 +24,11 @@ HealthCheckController.prototype.handleHealthCheck = function (req, res) { if (err) { response.err = err.message; } - res.send(response, ok ? 200 : 503); + res.status(ok ? 200 : 503).send(response); }); } else { - res.send({enabled: false, ok: true}, 200); + res.status(200).send({enabled: false, ok: true}); } }; diff --git a/app/controllers/job_controller.js b/app/controllers/job_controller.js index 3aa00d738..f600bd197 100644 --- a/app/controllers/job_controller.js +++ b/app/controllers/job_controller.js @@ -1,17 +1,14 @@ 'use strict'; var _ = require('underscore'); -var step = require('step'); -var assert = require('assert'); var util = require('util'); -var AuthApi = require('../auth/auth_api'); -var CdbRequest = require('../models/cartodb_request'); +var userMiddleware = require('../middlewares/user'); +var authenticatedMiddleware = require('../middlewares/authenticated-request'); var handleException = require('../utils/error_handler'); -var cdbReq = new CdbRequest(); var ONE_KILOBYTE_IN_BYTES = 1024; -var MAX_LIMIT_QUERY_SIZE_IN_KB = 8; +var MAX_LIMIT_QUERY_SIZE_IN_KB = 16; var MAX_LIMIT_QUERY_SIZE_IN_BYTES = MAX_LIMIT_QUERY_SIZE_IN_KB * ONE_KILOBYTE_IN_BYTES; function getMaxSizeErrorMessage(sql) { @@ -26,9 +23,10 @@ function getMaxSizeErrorMessage(sql) { ); } -function JobController(userDatabaseService, jobService) { +function JobController(userDatabaseService, jobService, statsdClient) { this.userDatabaseService = userDatabaseService; this.jobService = jobService; + this.statsdClient = statsdClient || { increment: function () {} }; } function bodyPayloadSizeMiddleware(req, res, next) { @@ -45,353 +43,107 @@ module.exports.MAX_LIMIT_QUERY_SIZE_IN_BYTES = MAX_LIMIT_QUERY_SIZE_IN_BYTES; module.exports.getMaxSizeErrorMessage = getMaxSizeErrorMessage; JobController.prototype.route = function (app) { - app.post(global.settings.base_url + '/sql/job', bodyPayloadSizeMiddleware, this.createJob.bind(this)); - app.get(global.settings.base_url + '/sql/job', this.listJob.bind(this)); - app.get(global.settings.base_url + '/sql/job/:job_id', this.getJob.bind(this)); - app.delete(global.settings.base_url + '/sql/job/:job_id', this.cancelJob.bind(this)); - app.put(global.settings.base_url + '/sql/job/:job_id', bodyPayloadSizeMiddleware, this.updateJob.bind(this)); - app.patch(global.settings.base_url + '/sql/job/:job_id', bodyPayloadSizeMiddleware, this.updateJob.bind(this)); -}; - -JobController.prototype.cancelJob = function (req, res) { - var self = this; - var job_id = req.params.job_id; - var body = (req.body) ? req.body : {}; - var params = _.extend({}, req.query, body); // clone so don't modify req.params or req.body so oauth is not broken - var cdbUsername = cdbReq.userByReq(req); - - if ( req.profiler ) { - req.profiler.start('sqlapi.job'); - req.profiler.done('init'); - } - - step( - function getUserDBInfo() { - var next = this; - var authApi = new AuthApi(req, params); - - self.userDatabaseService.getConnectionParams(authApi, cdbUsername, next); - }, - function cancelJob(err, userDatabase) { - assert.ifError(err); - - if (!userDatabase.authenticated) { - throw new Error('permission denied'); - } - - var next = this; - - if ( req.profiler ) { - req.profiler.done('setDBAuth'); - } - - self.jobService.cancel(job_id, function (err, job) { - if (err) { - return next(err); - } - - next(null, { - job: job.serialize(), - host: userDatabase.host - }); - }); - }, - function handleResponse(err, result) { - if ( err ) { - return handleException(err, res); - } - - if ( req.profiler ) { - req.profiler.done('cancelJob'); - res.header('X-SQLAPI-Profiler', req.profiler.toJSONString()); - } - - if (global.settings.api_hostname) { - res.header('X-Served-By-Host', global.settings.api_hostname); - } - - if (result.host) { - res.header('X-Served-By-DB-Host', result.host); - } - - res.send(result.job); - } + app.post( + global.settings.base_url + '/sql/job', + bodyPayloadSizeMiddleware, userMiddleware, authenticatedMiddleware(this.userDatabaseService), + this.createJob.bind(this) + ); + app.get( + global.settings.base_url + '/sql/job/wip', + userMiddleware, authenticatedMiddleware(this.userDatabaseService), + this.listWorkInProgressJobs.bind(this) + ); + app.get( + global.settings.base_url + '/sql/job/:job_id', + userMiddleware, authenticatedMiddleware(this.userDatabaseService), + this.getJob.bind(this) + ); + app.delete( + global.settings.base_url + '/sql/job/:job_id', + userMiddleware, authenticatedMiddleware(this.userDatabaseService), + this.cancelJob.bind(this) ); }; -JobController.prototype.listJob = function (req, res) { - var self = this; - var body = (req.body) ? req.body : {}; - var params = _.extend({}, req.query, body); // clone so don't modify req.params or req.body so oauth is not broken - var cdbUsername = cdbReq.userByReq(req); - - if ( req.profiler ) { - req.profiler.start('sqlapi.job'); - req.profiler.done('init'); - } - - step( - function getUserDBInfo() { - var next = this; - var authApi = new AuthApi(req, params); - - self.userDatabaseService.getConnectionParams(authApi, cdbUsername, next); - }, - function listJob(err, userDatabase) { - assert.ifError(err); - - if (!userDatabase.authenticated) { - throw new Error('permission denied'); - } - - var next = this; - - if ( req.profiler ) { - req.profiler.done('setDBAuth'); - } - - self.jobService.list(cdbUsername, function (err, jobs) { - if (err) { - return next(err); - } - - next(null, { - jobs: jobs.map(function (job) { - return job.serialize(); - }), - host: userDatabase.host - }); - }); - }, - function handleResponse(err, result) { - if ( err ) { - return handleException(err, res); - } - - if ( req.profiler ) { - req.profiler.done('listJob'); - res.header('X-SQLAPI-Profiler', req.profiler.toJSONString()); - } - - if (global.settings.api_hostname) { - res.header('X-Served-By-Host', global.settings.api_hostname); - } - - if (result.host) { - res.header('X-Served-By-DB-Host', result.host); - } - - res.send(result.jobs); - } - ); +JobController.prototype.cancelJob = function (req, res) { + this.jobService.cancel(req.params.job_id, jobResponse(req, res, this.statsdClient, 'cancel')); }; JobController.prototype.getJob = function (req, res) { - var self = this; - var job_id = req.params.job_id; - var body = (req.body) ? req.body : {}; - var params = _.extend({}, req.query, body); // clone so don't modify req.params or req.body so oauth is not broken - var cdbUsername = cdbReq.userByReq(req); - - if ( req.profiler ) { - req.profiler.start('sqlapi.job'); - req.profiler.done('init'); - } - - step( - function getUserDBInfo() { - var next = this; - var authApi = new AuthApi(req, params); - - self.userDatabaseService.getConnectionParams(authApi, cdbUsername, next); - }, - function getJob(err, userDatabase) { - assert.ifError(err); - - if (!userDatabase.authenticated) { - throw new Error('permission denied'); - } - - var next = this; - - if ( req.profiler ) { - req.profiler.done('setDBAuth'); - } - - self.jobService.get(job_id, function (err, job) { - if (err) { - return next(err); - } - - next(null, { - job: job.serialize(), - host: userDatabase.host - }); - }); - }, - function handleResponse(err, result) { - if ( err ) { - return handleException(err, res); - } - - if ( req.profiler ) { - req.profiler.done('getJob'); - res.header('X-SQLAPI-Profiler', req.profiler.toJSONString()); - } - - if (global.settings.api_hostname) { - res.header('X-Served-By-Host', global.settings.api_hostname); - } - - if (result.host) { - res.header('X-Served-By-DB-Host', result.host); - } - - res.send(result.job); - } - ); + this.jobService.get(req.params.job_id, jobResponse(req, res, this.statsdClient, 'retrieve')); }; JobController.prototype.createJob = function (req, res) { - var self = this; var body = (req.body) ? req.body : {}; var params = _.extend({}, req.query, body); // clone so don't modify req.params or req.body so oauth is not broken var sql = (params.query === "" || _.isUndefined(params.query)) ? null : params.query; - var cdbUsername = cdbReq.userByReq(req); - - if ( req.profiler ) { - req.profiler.start('sqlapi.job'); - req.profiler.done('init'); - } - - step( - function getUserDBInfo() { - var next = this; - var authApi = new AuthApi(req, params); - - self.userDatabaseService.getConnectionParams(authApi, cdbUsername, next); - }, - function persistJob(err, userDatabase) { - assert.ifError(err); - - if (!userDatabase.authenticated) { - throw new Error('permission denied'); - } - var next = this; + var data = { + user: req.context.user, + query: sql, + host: req.context.userDatabase.host + }; - if ( req.profiler ) { - req.profiler.done('setDBAuth'); - } - - var data = { - user: cdbUsername, - query: sql, - host: userDatabase.host - }; - - self.jobService.create(data, function (err, job) { - if (err) { - return next(err); - } - - next(null, { - job: job.serialize(), - host: userDatabase.host - }); - }); - }, - function handleResponse(err, result) { - if ( err ) { - return handleException(err, res); - } - - if ( req.profiler ) { - req.profiler.done('persistJob'); - res.header('X-SQLAPI-Profiler', req.profiler.toJSONString()); - } - - if (global.settings.api_hostname) { - res.header('X-Served-By-Host', global.settings.api_hostname); - } - - if (result.host) { - res.header('X-Served-By-DB-Host', result.host); - } - - res.status(201).send(result.job); - } - ); + this.jobService.create(data, jobResponse(req, res, this.statsdClient, 'create', 201)); }; -JobController.prototype.updateJob = function (req, res) { +JobController.prototype.listWorkInProgressJobs = function (req, res) { var self = this; - var job_id = req.params.job_id; - var body = (req.body) ? req.body : {}; - var params = _.extend({}, req.query, body); // clone so don't modify req.params or req.body so oauth is not broken - var sql = (params.query === "" || _.isUndefined(params.query)) ? null : params.query; - var cdbUsername = cdbReq.userByReq(req); - - if ( req.profiler ) { - req.profiler.start('sqlapi.job'); - req.profiler.done('init'); - } - step( - function getUserDBInfo() { - var next = this; - var authApi = new AuthApi(req, params); + this.jobService.listWorkInProgressJobs(function (err, list) { + if (err) { + self.statsdClient.increment('sqlapi.job.error'); + return handleException(err, res); + } - self.userDatabaseService.getConnectionParams(authApi, cdbUsername, next); - }, - function updateJob(err, userDatabase) { - assert.ifError(err); + res.header('X-Served-By-DB-Host', req.context.userDatabase.host); - if (!userDatabase.authenticated) { - throw new Error('permission denied'); - } + req.profiler.done('list'); + req.profiler.end(); + req.profiler.sendStats(); - var next = this; + res.header('X-SQLAPI-Profiler', req.profiler.toJSONString()); + self.statsdClient.increment('sqlapi.job.success'); - if ( req.profiler ) { - req.profiler.done('setDBAuth'); - } + if (process.env.NODE_ENV !== 'test') { + console.info(JSON.stringify({ + type: 'sql_api_batch_job', + username: req.context.user, + action: 'list' + })); + } - var data = { - job_id: job_id, - query: sql - }; + res.status(200).send(list); + }); +}; - self.jobService.update(data, function (err, job) { - if (err) { - return next(err); - } +function jobResponse(req, res, statsdClient, action, status) { + return function handler(err, job) { + status = status || 200; - next(null, { - job: job.serialize(), - host: userDatabase.host - }); - }); - }, - function handleResponse(err, result) { - if ( err ) { - return handleException(err, res); - } + if (err) { + statsdClient.increment('sqlapi.job.error'); + return handleException(err, res); + } - if ( req.profiler ) { - req.profiler.done('updateJob'); - res.header('X-SQLAPI-Profiler', req.profiler.toJSONString()); - } + res.header('X-Served-By-DB-Host', req.context.userDatabase.host); - if (global.settings.api_hostname) { - res.header('X-Served-By-Host', global.settings.api_hostname); - } + req.profiler.done(action); + req.profiler.end(); + req.profiler.sendStats(); - if (result.host) { - res.header('X-Served-By-DB-Host', result.host); - } + res.header('X-SQLAPI-Profiler', req.profiler.toJSONString()); + statsdClient.increment('sqlapi.job.success'); - res.send(result.job); + if (process.env.NODE_ENV !== 'test') { + console.info(JSON.stringify({ + type: 'sql_api_batch_job', + username: req.context.user, + action: action, + job_id: job.job_id + })); } - ); -}; + + res.status(status).send(job.serialize()); + }; +} diff --git a/app/controllers/query_controller.js b/app/controllers/query_controller.js index e8673d7fe..c287d2ed7 100644 --- a/app/controllers/query_controller.js +++ b/app/controllers/query_controller.js @@ -186,9 +186,10 @@ QueryController.prototype.handleQuery = function (req, res) { } // Only set an X-Cache-Channel for responses we want Varnish to cache. - if (!!affectedTables && affectedTables.tables.length > 0 && !mayWrite) { - res.header('X-Cache-Channel', affectedTables.getCacheChannel()); - res.header('Surrogate-Key', affectedTables.key().join(' ')); + var skipNotUpdatedAtTables = true; + if (!!affectedTables && affectedTables.getTables(skipNotUpdatedAtTables).length > 0 && !mayWrite) { + res.header('X-Cache-Channel', affectedTables.getCacheChannel(skipNotUpdatedAtTables)); + res.header('Surrogate-Key', affectedTables.key(skipNotUpdatedAtTables).join(' ')); } if(!!affectedTables) { @@ -227,9 +228,6 @@ QueryController.prototype.handleQuery = function (req, res) { }; } - if (global.settings.api_hostname) { - res.header('X-Served-By-Host', global.settings.api_hostname); - } if (dbopts.host) { res.header('X-Served-By-DB-Host', dbopts.host); } @@ -243,7 +241,7 @@ QueryController.prototype.handleQuery = function (req, res) { } if ( req.profiler ) { - req.profiler.sendStats(); // TODO: do on nextTick ? + req.profiler.sendStats(); } if (self.statsd_client) { if ( err ) { diff --git a/app/middlewares/authenticated-request.js b/app/middlewares/authenticated-request.js new file mode 100644 index 000000000..05f20f593 --- /dev/null +++ b/app/middlewares/authenticated-request.js @@ -0,0 +1,35 @@ +'use strict'; + +var _ = require('underscore'); +var AuthApi = require('../auth/auth_api'); +var handleException = require('../utils/error_handler'); + +function authenticatedMiddleware(userDatabaseService) { + return function middleware(req, res, next) { + req.profiler.start('sqlapi.job'); + req.profiler.done('init'); + + var body = (req.body) ? req.body : {}; + // clone so don't modify req.params or req.body so oauth is not broken + var params = _.extend({}, req.query, body); + + var authApi = new AuthApi(req, params); + userDatabaseService.getConnectionParams(authApi, req.context.user, function cancelJob(err, userDatabase) { + req.profiler.done('setDBAuth'); + + if (err) { + return handleException(err, res); + } + + if (!userDatabase.authenticated) { + return handleException(new Error('permission denied'), res); + } + + req.context.userDatabase = userDatabase; + + return next(null); + }); + }; +} + +module.exports = authenticatedMiddleware; diff --git a/app/middlewares/body-parser.js b/app/middlewares/body-parser.js new file mode 100644 index 000000000..6c44e295e --- /dev/null +++ b/app/middlewares/body-parser.js @@ -0,0 +1,145 @@ + +/*! + * Connect - bodyParser + * Copyright(c) 2010 Sencha Inc. + * Copyright(c) 2011 TJ Holowaychuk + * MIT Licensed + */ + +/** + * Module dependencies. + */ + +var qs = require('qs'); +var multer = require('multer'); + +/** + * Extract the mime type from the given request's + * _Content-Type_ header. + * + * @param {IncomingMessage} req + * @return {String} + * @api private + */ + +function mime(req) { + var str = req.headers['content-type'] || ''; + return str.split(';')[0]; +} + +/** + * Parse request bodies. + * + * By default _application/json_, _application/x-www-form-urlencoded_, + * and _multipart/form-data_ are supported, however you may map `connect.bodyParser.parse[contentType]` + * to a function receiving `(req, options, callback)`. + * + * Examples: + * + * connect.createServer( + * connect.bodyParser() + * , function(req, res) { + * res.end('viewing user ' + req.body.user.name); + * } + * ); + * + * $ curl -d 'user[name]=tj' http://localhost/ + * $ curl -d '{"user":{"name":"tj"}}' -H "Content-Type: application/json" http://localhost/ + * + * Multipart req.files: + * + * As a security measure files are stored in a separate object, stored + * as `req.files`. This prevents attacks that may potentially alter + * filenames, and depending on the application gain access to restricted files. + * + * Multipart configuration: + * + * The `options` passed are provided to each parser function. + * The _multipart/form-data_ parser merges these with formidable's + * IncomingForm object, allowing you to tweak the upload directory, + * size limits, etc. For example you may wish to retain the file extension + * and change the upload directory: + * + * server.use(bodyParser({ uploadDir: '/www/mysite.com/uploads' })); + * + * View [node-formidable](https://github.com/felixge/node-formidable) for more information. + * + * If you wish to use formidable directly within your app, and do not + * desire this behaviour for multipart requests simply remove the + * parser: + * + * delete connect.bodyParser.parse['multipart/form-data']; + * + * Or + * + * delete express.bodyParser.parse['multipart/form-data']; + * + * @param {Object} options + * @return {Function} + * @api public + */ + +exports = module.exports = function bodyParser(options){ + options = options || {}; + return function bodyParser(req, res, next) { + if (req.body) { + return next(); + } + req.body = {}; + + if ('GET' === req.method || 'HEAD' === req.method) { + return next(); + } + var parser = exports.parse[mime(req)]; + if (parser) { + parser(req, options, next); + } else { + next(); + } + }; +}; + +/** + * Parsers. + */ + +exports.parse = {}; + +/** + * Parse application/x-www-form-urlencoded. + */ + +exports.parse['application/x-www-form-urlencoded'] = function(req, options, fn){ + var buf = ''; + req.setEncoding('utf8'); + req.on('data', function(chunk){ buf += chunk; }); + req.on('end', function(){ + try { + req.body = buf.length ? qs.parse(buf) : {}; + fn(); + } catch (err){ + fn(err); + } + }); +}; + +/** + * Parse application/json. + */ + +exports.parse['application/json'] = function(req, options, fn){ + var buf = ''; + req.setEncoding('utf8'); + req.on('data', function(chunk){ buf += chunk; }); + req.on('end', function(){ + try { + req.body = buf.length ? JSON.parse(buf) : {}; + fn(); + } catch (err){ + fn(err); + } + }); +}; + +var multipartMiddleware = multer({ limits: { fieldSize: Infinity } }); +exports.parse['multipart/form-data'] = multipartMiddleware.none(); diff --git a/app/middlewares/user.js b/app/middlewares/user.js new file mode 100644 index 000000000..57e9d8b5f --- /dev/null +++ b/app/middlewares/user.js @@ -0,0 +1,7 @@ +var CdbRequest = require('../models/cartodb_request'); +var cdbRequest = new CdbRequest(); + +module.exports = function userMiddleware(req, res, next) { + req.context.user = cdbRequest.userByReq(req); + next(); +}; diff --git a/app/models/log4js_rollbar.js b/app/models/log4js_rollbar.js deleted file mode 100644 index c26e1eb6b..000000000 --- a/app/models/log4js_rollbar.js +++ /dev/null @@ -1,51 +0,0 @@ -var rollbar = require("rollbar"); - -/** - * Rollbar Appender. Sends logging events to Rollbar using node-rollbar - * - * @param config object with rollbar configuration data - * { - * token: 'your-secret-token', - * options: node-rollbar options - * } - */ -function rollbarAppender(config) { - - var opt = config.options; - rollbar.init(opt.token, opt.options); - - return function(loggingEvent) { -/* -For logger.trace('one','two','three'): -{ startTime: Wed Mar 12 2014 16:27:40 GMT+0100 (CET), - categoryName: '[default]', - data: [ 'one', 'two', 'three' ], - level: { level: 5000, levelStr: 'TRACE' }, - logger: { category: '[default]', _events: { log: [Object] } } } -*/ - - // Levels: - // TRACE 5000 - // DEBUG 10000 - // INFO 20000 - // WARN 30000 - // ERROR 40000 - // FATAL 50000 - // - // We only log error and higher errors - // - if ( loggingEvent.level.level < 40000 ) { - return; - } - - rollbar.reportMessage(loggingEvent.data); - }; -} - -function configure(config) { - return rollbarAppender(config); -} - -exports.name = "rollbar"; -exports.appender = rollbarAppender; -exports.configure = configure; diff --git a/app/app.js b/app/server.js similarity index 84% rename from app/app.js rename to app/server.js index 173c2e863..04b7bc92a 100644 --- a/app/app.js +++ b/app/server.js @@ -15,17 +15,18 @@ // var express = require('express'); +var bodyParser = require('./middlewares/body-parser'); var os = require('os'); -var Profiler = require('step-profiler'); +var Profiler = require('./stats/profiler-proxy'); var StatsD = require('node-statsd').StatsD; var _ = require('underscore'); var LRU = require('lru-cache'); -var redis = require('redis'); +var RedisPool = require('redis-mpool'); +var cartodbRedis = require('cartodb-redis'); var UserDatabaseService = require('./services/user_database_service'); -var JobPublisher = require('../batch/job_publisher'); +var JobPublisher = require('../batch/pubsub/job-publisher'); var JobQueue = require('../batch/job_queue'); -var UserIndexer = require('../batch/user_indexer'); var JobBackend = require('../batch/job_backend'); var JobCanceller = require('../batch/job_canceller'); var JobService = require('../batch/job_service'); @@ -51,15 +52,17 @@ require('./utils/date_to_json'); // jshint maxcomplexity:12 function App() { - var app = express.createServer(); + var app = express(); - var metadataBackend = require('cartodb-redis')({ + var redisPool = new RedisPool({ + name: 'sql-api', host: global.settings.redis_host, port: global.settings.redis_port, max: global.settings.redisPool, idleTimeoutMillis: global.settings.redisIdleTimeoutMillis, reapIntervalMillis: global.settings.redisReapIntervalMillis }); + var metadataBackend = cartodbRedis({ pool: redisPool }); // Set default configuration @@ -102,16 +105,6 @@ function App() { } }; app.use(global.log4js.connectLogger(global.log4js.getLogger(), _.defaults(loggerOpts, {level:'info'}))); - } else { - // Express logger uses tokens as described here: http://www.senchalabs.org/connect/logger.html - express.logger.token('sql', function(req) { - return app.getSqlQueryFromRequestBody(req); - }); - app.use(express.logger({ - buffer: true, - format: global.settings.log_format || - ':remote-addr :method :req[Host]:url :status :response-time ms -> :res[Content-Type]' - })); } // Initialize statsD client if requested @@ -156,12 +149,20 @@ function App() { app.use(cors()); // Use step-profiler - if ( global.settings.useProfiler ) { - app.use(function(req, res, next) { - req.profiler = new Profiler({statsd_client:statsd_client}); + app.use(function bootstrap$prepareRequestResponse(req, res, next) { + req.context = req.context || {}; + + if (global.settings.api_hostname) { + res.header('X-Served-By-Host', global.settings.api_hostname); + } + + var profile = global.settings.useProfiler; + req.profiler = new Profiler({ + profile: profile, + statsd_client: statsd_client + }); next(); - }); - } + }); // Set connection timeout if ( global.settings.hasOwnProperty('node_socket_timeout') ) { @@ -172,30 +173,30 @@ function App() { }); } - app.use(express.bodyParser()); + app.use(bodyParser()); app.enable('jsonp callback'); app.set("trust proxy", true); + app.disable('x-powered-by'); + app.disable('etag'); // basic routing var userDatabaseService = new UserDatabaseService(metadataBackend); - var jobQueue = new JobQueue(metadataBackend); - var jobPublisher = new JobPublisher(redis); - var userIndexer = new UserIndexer(metadataBackend); - var jobBackend = new JobBackend(metadataBackend, jobQueue, jobPublisher, userIndexer); + var jobPublisher = new JobPublisher(redisPool); + var jobQueue = new JobQueue(metadataBackend, jobPublisher); + var jobBackend = new JobBackend(metadataBackend, jobQueue); var userDatabaseMetadataService = new UserDatabaseMetadataService(metadataBackend); var jobCanceller = new JobCanceller(userDatabaseMetadataService); var jobService = new JobService(jobBackend, jobCanceller); - var genericController = new GenericController(); genericController.route(app); var queryController = new QueryController(userDatabaseService, tableCache, statsd_client); queryController.route(app); - var jobController = new JobController(userDatabaseService, jobService, jobCanceller); + var jobController = new JobController(userDatabaseService, jobService, statsd_client); jobController.route(app); var cacheStatusController = new CacheStatusController(tableCache); @@ -210,7 +211,10 @@ function App() { var isBatchProcess = process.argv.indexOf('--no-batch') === -1; if (global.settings.environment !== 'test' && isBatchProcess) { - app.batch = batchFactory(metadataBackend); + var batchName = global.settings.api_hostname || 'batch'; + app.batch = batchFactory( + metadataBackend, redisPool, batchName, statsd_client, global.settings.batch_log_filename + ); app.batch.start(); } diff --git a/app/stats/profiler-proxy.js b/app/stats/profiler-proxy.js new file mode 100644 index 000000000..1e1f91e77 --- /dev/null +++ b/app/stats/profiler-proxy.js @@ -0,0 +1,53 @@ +var Profiler = require('step-profiler'); + +/** + * Proxy to encapsulate node-step-profiler module so there is no need to check if there is an instance + */ +function ProfilerProxy(opts) { + this.profile = !!opts.profile; + + this.profiler = null; + if (!!opts.profile) { + this.profiler = new Profiler({statsd_client: opts.statsd_client}); + } +} + +ProfilerProxy.prototype.done = function(what) { + if (this.profile) { + this.profiler.done(what); + } +}; + +ProfilerProxy.prototype.end = function() { + if (this.profile) { + this.profiler.end(); + } +}; + +ProfilerProxy.prototype.start = function(what) { + if (this.profile) { + this.profiler.start(what); + } +}; + +ProfilerProxy.prototype.add = function(what) { + if (this.profile) { + this.profiler.add(what || {}); + } +}; + +ProfilerProxy.prototype.sendStats = function() { + if (this.profile) { + this.profiler.sendStats(); + } +}; + +ProfilerProxy.prototype.toString = function() { + return this.profile ? this.profiler.toString() : ""; +}; + +ProfilerProxy.prototype.toJSONString = function() { + return this.profile ? this.profiler.toJSONString() : "{}"; +}; + +module.exports = ProfilerProxy; diff --git a/app/utils/error_handler.js b/app/utils/error_handler.js index e642de31a..9d46def5e 100644 --- a/app/utils/error_handler.js +++ b/app/utils/error_handler.js @@ -25,14 +25,22 @@ module.exports = function handleException(err, res) { // Force inline content disposition res.header("Content-Disposition", 'inline'); - if ( res.req && res.req.profiler ) { - res.req.profiler.done('finish'); - res.header('X-SQLAPI-Profiler', res.req.profiler.toJSONString()); + var req = res.req; + + if (req && req.profiler ) { + req.profiler.done('finish'); + res.header('X-SQLAPI-Profiler', req.profiler.toJSONString()); } - res.send(msg, getStatusError(pgErrorHandler, res.req)); + res.header('Content-Type', 'application/json; charset=utf-8'); + res.status(getStatusError(pgErrorHandler, req)); + if (req.query && req.query.callback) { + res.jsonp(msg); + } else { + res.json(msg); + } - if ( res.req && res.req.profiler ) { + if (req && req.profiler) { res.req.profiler.sendStats(); } }; diff --git a/batch/README.md b/batch/README.md new file mode 100644 index 000000000..60cc64e1a --- /dev/null +++ b/batch/README.md @@ -0,0 +1,122 @@ +# Batch Queries + +This document describes features from Batch Queries, it also details some internals that might be useful for maintainers +and developers. + + +## Redis data structures + +### Jobs definition + +Redis Hash: `batch:jobs:{UUID}`. + +Redis DB: 5. + +It stores the job definition, the user, and some metadata like the final status, the failure reason, and so. + +### Job queues + +Redis List: `batch:queue:{username}`. + +Redis DB: 5. + +It stores a pending list of jobs per user. It points to a job definition with the `{UUID}`. + +### Job notifications + +Redis Pub/Sub channel: `batch:users`. + +Redis DB: 0. + +In order to notify new jobs, it uses a Pub/Sub channel were the username for the queued job is published. + + +## Job types + +Format for the currently supported query types, and what they are missing in terms of features. + +### Simple + +```json +{ + "query": "update ..." +} +``` + +Does not support main fallback queries. Ideally it should support something like: + +```json +{ + "query": "update ...", + "onsuccess": "select 'general success fallback'", + "onerror": "select 'general error fallback'" +} +``` + +### Multiple + +```json +{ + "query": [ + "update ...", + "select ... into ..." + ] +} +``` + +Does not support main fallback queries. Ideally it should support something like: + +```json +{ + "query": [ + "update ...", + "select ... into ..." + ], + "onsuccess": "select 'general success fallback'", + "onerror": "select 'general error fallback'" +} +``` + +### Fallback + +```json +{ + "query": { + "query": [ + { + "query": "select 1", + "onsuccess": "select 'success fallback query 1'", + "onerror": "select 'error fallback query 1'" + }, + { + "query": "select 2", + "onerror": "select 'error fallback query 2'" + } + ], + "onsuccess": "select 'general success fallback'", + "onerror": "select 'general error fallback'" + } +} +``` + +It's weird to have two nested `query` attributes. Also, it's not possible to mix _plain_ with _fallback_ ones. +Ideally it should support something like: + +```json +{ + "query": [ + { + "query": "select 1", + "onsuccess": "select 'success fallback query 1'", + "onerror": "select 'error fallback query 1'" + }, + "select 2" + ], + "onsuccess": "select 'general success fallback'", + "onerror": "select 'general error fallback'" + } +} +``` + +Where you don't need a nested `query` attribute, it's just an array as in Multiple job type, and you can mix objects and +plain queries. diff --git a/batch/batch-logger.js b/batch/batch-logger.js new file mode 100644 index 000000000..05dc54f8f --- /dev/null +++ b/batch/batch-logger.js @@ -0,0 +1,29 @@ +'use strict'; + +var bunyan = require('bunyan'); + +function BatchLogger (path) { + var stream = { + level: process.env.NODE_ENV === 'test' ? 'fatal' : 'info' + }; + if (path) { + stream.path = path; + } else { + stream.stream = process.stdout; + } + this.path = path; + this.logger = bunyan.createLogger({ + name: 'batch-queries', + streams: [stream] + }); +} + +module.exports = BatchLogger; + +BatchLogger.prototype.log = function (job) { + return job.log(this.logger); +}; + +BatchLogger.prototype.reopenFileStreams = function () { + this.logger.reopenFileStreams(); +}; diff --git a/batch/batch.js b/batch/batch.js index 66b158c77..ad4ec064c 100644 --- a/batch/batch.js +++ b/batch/batch.js @@ -3,101 +3,114 @@ var util = require('util'); var EventEmitter = require('events').EventEmitter; var debug = require('./util/debug')('batch'); -var forever = require('./util/forever'); var queue = require('queue-async'); -var jobStatus = require('./job_status'); +var HostScheduler = require('./scheduler/host-scheduler'); -function Batch(jobSubscriber, jobQueuePool, jobRunner, jobService) { +var EMPTY_QUEUE = true; + +function Batch(name, jobSubscriber, jobQueue, jobRunner, jobService, jobPublisher, redisPool, logger) { EventEmitter.call(this); + this.name = name || 'batch'; this.jobSubscriber = jobSubscriber; - this.jobQueuePool = jobQueuePool; + this.jobQueue = jobQueue; this.jobRunner = jobRunner; this.jobService = jobService; + this.jobPublisher = jobPublisher; + this.logger = logger; + this.hostScheduler = new HostScheduler(this.name, { run: this.processJob.bind(this) }, redisPool); + + // map: user => jobId. Will be used for draining jobs. + this.workInProgressJobs = {}; } util.inherits(Batch, EventEmitter); module.exports = Batch; Batch.prototype.start = function () { - this._subscribe(); -}; - -Batch.prototype._subscribe = function () { var self = this; - this.jobSubscriber.subscribe(function (channel, host) { - var queue = self.jobQueuePool.getQueue(host); - - // there is nothing to do. It is already running jobs - if (queue) { - return; - } - queue = self.jobQueuePool.createQueue(host); - - // do forever, it does not throw a stack overflow - forever(function (next) { - self._consumeJobs(host, queue, next); - }, function (err) { - self.jobQueuePool.removeQueue(host); - - if (err.name === 'EmptyQueue') { - return debug(err.message); + this.jobSubscriber.subscribe( + function onJobHandler(user, host) { + debug('[%s] onJobHandler(%s, %s)', self.name, user, host); + self.hostScheduler.add(host, user, function(err) { + if (err) { + return debug( + 'Could not schedule host=%s user=%s from %s. Reason: %s', + host, self.name, user, err.message + ); + } + }); + }, + function onJobSubscriberReady(err) { + if (err) { + return self.emit('error', err); } - debug(err); - }); - }); + self.emit('ready'); + } + ); }; - -Batch.prototype._consumeJobs = function (host, queue, callback) { +Batch.prototype.processJob = function (user, callback) { var self = this; - - queue.dequeue(host, function (err, job_id) { + self.jobQueue.dequeue(user, function (err, jobId) { if (err) { - return callback(err); + return callback(new Error('Could not get job from "' + user + '". Reason: ' + err.message), !EMPTY_QUEUE); } - if (!job_id) { - var emptyQueueError = new Error('Queue ' + host + ' is empty'); - emptyQueueError.name = 'EmptyQueue'; - return callback(emptyQueueError); + if (!jobId) { + debug('Queue empty user=%s', user); + return callback(null, EMPTY_QUEUE); } - self.jobQueuePool.setCurrentJobId(host, job_id); + self._processWorkInProgressJob(user, jobId, function (err, job) { + if (err) { + debug(err); + if (err.name === 'JobNotRunnable') { + return callback(null, !EMPTY_QUEUE); + } + return callback(err, !EMPTY_QUEUE); + } - self.jobRunner.run(job_id, function (err, job) { - self.jobQueuePool.removeCurrentJobId(host); + debug( + '[%s] Job=%s status=%s user=%s (failed_reason=%s)', + self.name, jobId, job.data.status, user, job.failed_reason + ); - if (err && err.name === 'JobNotRunnable') { - debug(err.message); - return callback(); - } + self.logger.log(job); - if (err) { - return callback(err); - } + return callback(null, !EMPTY_QUEUE); + }); + }); +}; - if (job.data.status === jobStatus.FAILED) { - debug('Job %s %s in %s due to: %s', job_id, job.data.status, host, job.failed_reason); - } else { - debug('Job %s %s in %s', job_id, job.data.status, host); - } +Batch.prototype._processWorkInProgressJob = function (user, jobId, callback) { + var self = this; + + self.setWorkInProgressJob(user, jobId, function (errSet) { + if (errSet) { + debug(new Error('Could not add job to work-in-progress list. Reason: ' + errSet.message)); + } - self.emit('job:' + job.data.status, job_id); + self.jobRunner.run(jobId, function (err, job) { + self.clearWorkInProgressJob(user, jobId, function (errClear) { + if (errClear) { + debug(new Error('Could not clear job from work-in-progress list. Reason: ' + errClear.message)); + } - callback(); + return callback(err, job); + }); }); }); }; Batch.prototype.drain = function (callback) { var self = this; - var queues = this.jobQueuePool.list(); - var batchQueues = queue(queues.length); + var workingUsers = this.getWorkInProgressUsers(); + var batchQueues = queue(workingUsers.length); - queues.forEach(function (host) { - batchQueues.defer(self._drainJob.bind(self), host); + workingUsers.forEach(function (user) { + batchQueues.defer(self._drainJob.bind(self), user); }); batchQueues.awaitAll(function (err) { @@ -111,9 +124,9 @@ Batch.prototype.drain = function (callback) { }); }; -Batch.prototype._drainJob = function (host, callback) { +Batch.prototype._drainJob = function (user, callback) { var self = this; - var job_id = self.jobQueuePool.getCurrentJobId(host); + var job_id = this.getWorkInProgressJob(user); if (!job_id) { return process.nextTick(function () { @@ -121,8 +134,6 @@ Batch.prototype._drainJob = function (host, callback) { }); } - var queue = self.jobQueuePool.getQueue(host); - this.jobService.drain(job_id, function (err) { if (err && err.name === 'CancelNotAllowedError') { return callback(); @@ -132,10 +143,32 @@ Batch.prototype._drainJob = function (host, callback) { return callback(err); } - queue.enqueueFirst(job_id, host, callback); + self.jobQueue.enqueueFirst(user, job_id, callback); }); }; -Batch.prototype.stop = function () { - this.jobSubscriber.unsubscribe(); +Batch.prototype.stop = function (callback) { + this.removeAllListeners(); + this.jobSubscriber.unsubscribe(callback); +}; + + +/* Work in progress jobs */ + +Batch.prototype.setWorkInProgressJob = function(user, jobId, callback) { + this.workInProgressJobs[user] = jobId; + this.jobService.addWorkInProgressJob(user, jobId, callback); +}; + +Batch.prototype.getWorkInProgressJob = function(user) { + return this.workInProgressJobs[user]; +}; + +Batch.prototype.clearWorkInProgressJob = function(user, jobId, callback) { + delete this.workInProgressJobs[user]; + this.jobService.clearWorkInProgressJob(user, jobId, callback); +}; + +Batch.prototype.getWorkInProgressUsers = function() { + return Object.keys(this.workInProgressJobs); }; diff --git a/batch/index.js b/batch/index.js index 985d72042..8e662917b 100644 --- a/batch/index.js +++ b/batch/index.js @@ -1,34 +1,39 @@ 'use strict'; -var redis = require('redis'); var JobRunner = require('./job_runner'); var QueryRunner = require('./query_runner'); var JobCanceller = require('./job_canceller'); -var JobQueuePool = require('./job_queue_pool'); -var JobSubscriber = require('./job_subscriber'); -var QueueSeeker = require('./queue_seeker'); +var JobSubscriber = require('./pubsub/job-subscriber'); var UserDatabaseMetadataService = require('./user_database_metadata_service'); -var JobPublisher = require('./job_publisher'); +var JobPublisher = require('./pubsub/job-publisher'); var JobQueue = require('./job_queue'); -var UserIndexer = require('./user_indexer'); var JobBackend = require('./job_backend'); var JobService = require('./job_service'); +var BatchLogger = require('./batch-logger'); var Batch = require('./batch'); -module.exports = function batchFactory (metadataBackend) { - var queueSeeker = new QueueSeeker(metadataBackend); - var jobSubscriber = new JobSubscriber(redis, queueSeeker); - var jobQueuePool = new JobQueuePool(metadataBackend); - var jobPublisher = new JobPublisher(redis); - var jobQueue = new JobQueue(metadataBackend); - var userIndexer = new UserIndexer(metadataBackend); - var jobBackend = new JobBackend(metadataBackend, jobQueue, jobPublisher, userIndexer); +module.exports = function batchFactory (metadataBackend, redisPool, name, statsdClient, loggerPath) { var userDatabaseMetadataService = new UserDatabaseMetadataService(metadataBackend); - // TODO: down userDatabaseMetadataService - var queryRunner = new QueryRunner(); + + var jobSubscriber = new JobSubscriber(redisPool, userDatabaseMetadataService); + var jobPublisher = new JobPublisher(redisPool); + + var jobQueue = new JobQueue(metadataBackend, jobPublisher); + var jobBackend = new JobBackend(metadataBackend, jobQueue); + var queryRunner = new QueryRunner(userDatabaseMetadataService); var jobCanceller = new JobCanceller(userDatabaseMetadataService); var jobService = new JobService(jobBackend, jobCanceller); - var jobRunner = new JobRunner(jobService, jobQueue, queryRunner, userDatabaseMetadataService); + var jobRunner = new JobRunner(jobService, jobQueue, queryRunner, metadataBackend, statsdClient); + var logger = new BatchLogger(loggerPath); - return new Batch(jobSubscriber, jobQueuePool, jobRunner, jobService); + return new Batch( + name, + jobSubscriber, + jobQueue, + jobRunner, + jobService, + jobPublisher, + redisPool, + logger + ); }; diff --git a/batch/job_backend.js b/batch/job_backend.js index 6b27870dd..4a1876959 100644 --- a/batch/job_backend.js +++ b/batch/job_backend.js @@ -1,24 +1,17 @@ 'use strict'; - -var queue = require('queue-async'); -var debug = require('./util/debug')('job-backend'); var REDIS_PREFIX = 'batch:jobs:'; var REDIS_DB = 5; -var JOBS_TTL_IN_SECONDS = global.settings.jobs_ttl_in_seconds || 48 * 3600; // 48 hours -var jobStatus = require('./job_status'); -var finalStatus = [ - jobStatus.CANCELLED, - jobStatus.DONE, - jobStatus.FAILED, - jobStatus.UNKNOWN -]; - -function JobBackend(metadataBackend, jobQueueProducer, jobPublisher, userIndexer) { +var JobStatus = require('./job_status'); +var queue = require('queue-async'); +var debug = require('./util/debug')('job-backend'); + +function JobBackend(metadataBackend, jobQueue) { this.metadataBackend = metadataBackend; - this.jobQueueProducer = jobQueueProducer; - this.jobPublisher = jobPublisher; - this.userIndexer = userIndexer; + this.jobQueue = jobQueue; + this.maxNumberOfQueuedJobs = global.settings.batch_max_queued_jobs || 64; + this.inSecondsJobTTLAfterFinished = global.settings.finished_jobs_ttl_in_seconds || 2 * 3600; // 2 hours + this.hostname = global.settings.api_hostname || 'batch'; } function toRedisParams(job) { @@ -29,8 +22,7 @@ function toRedisParams(job) { for (var property in obj) { if (obj.hasOwnProperty(property)) { redisParams.push(property); - // TODO: this should be moved to job model ?? - if ((property === 'query' || property === 'status') && typeof obj[property] !== 'string') { + if (property === 'query' && typeof obj[property] !== 'string') { redisParams.push(JSON.stringify(obj[property])); } else { redisParams.push(obj[property]); @@ -65,9 +57,8 @@ function toObject(job_id, redisParams, redisValues) { return obj; } -// TODO: is it really necessary?? function isJobFound(redisValues) { - return redisValues[0] && redisValues[1] && redisValues[2] && redisValues[3] && redisValues[4]; + return !!(redisValues[0] && redisValues[1] && redisValues[2] && redisValues[3] && redisValues[4]); } JobBackend.prototype.get = function (job_id, callback) { @@ -104,30 +95,34 @@ JobBackend.prototype.get = function (job_id, callback) { JobBackend.prototype.create = function (job, callback) { var self = this; - self.get(job.job_id, function (err) { - if (err && err.name !== 'NotFoundError') { - return callback(err); + this.jobQueue.size(job.user, function(err, size) { + if (err) { + return callback(new Error('Failed to create job, could not determine user queue size')); } - self.save(job, function (err, jobSaved) { - if (err) { + if (size >= self.maxNumberOfQueuedJobs) { + return callback(new Error( + 'Failed to create job. ' + + 'Max number of jobs (' + self.maxNumberOfQueuedJobs + ') queued reached' + )); + } + + self.get(job.job_id, function (err) { + if (err && err.name !== 'NotFoundError') { return callback(err); } - self.jobQueueProducer.enqueue(job.job_id, job.host, function (err) { + self.save(job, function (err, jobSaved) { if (err) { return callback(err); } - // broadcast to consumers - self.jobPublisher.publish(job.host); + self.jobQueue.enqueue(job.user, job.job_id, function (err) { + if (err) { + return callback(err); + } - self.userIndexer.add(job.user, job.job_id, function (err) { - if (err) { - return callback(err); - } - - callback(null, jobSaved); + return callback(null, jobSaved); }); }); }); @@ -138,6 +133,7 @@ JobBackend.prototype.update = function (job, callback) { var self = this; self.get(job.job_id, function (err) { + if (err) { return callback(err); } @@ -171,84 +167,111 @@ JobBackend.prototype.save = function (job, callback) { }); }; -function isFinalStatus(status) { - return finalStatus.indexOf(status) !== -1; -} - -JobBackend.prototype.setTTL = function (job, callback) { - var self = this; - var redisKey = REDIS_PREFIX + job.job_id; - - if (!isFinalStatus(job.status)) { - return callback(); - } +var WORK_IN_PROGRESS_JOB = { + DB: 5, + PREFIX_USER: 'batch:wip:user:', + USER_INDEX_KEY: 'batch:wip:users' +}; - self.metadataBackend.redisCmd(REDIS_DB, 'EXPIRE', [ redisKey, JOBS_TTL_IN_SECONDS ], callback); +JobBackend.prototype.addWorkInProgressJob = function (user, jobId, callback) { + var userWIPKey = WORK_IN_PROGRESS_JOB.PREFIX_USER + user; + debug('add job %s to user %s (%s)', jobId, user, userWIPKey); + this.metadataBackend.redisMultiCmd(WORK_IN_PROGRESS_JOB.DB, [ + ['SADD', WORK_IN_PROGRESS_JOB.USER_INDEX_KEY, user], + ['RPUSH', userWIPKey, jobId] + ], callback); }; -JobBackend.prototype.list = function (user, callback) { +JobBackend.prototype.clearWorkInProgressJob = function (user, jobId, callback) { var self = this; + var DB = WORK_IN_PROGRESS_JOB.DB; + var userWIPKey = WORK_IN_PROGRESS_JOB.PREFIX_USER + user; - this.userIndexer.list(user, function (err, job_ids) { + var params = [userWIPKey, 0, jobId]; + self.metadataBackend.redisCmd(DB, 'LREM', params, function (err) { if (err) { return callback(err); } - var initialLength = job_ids.length; - - self._getCleanedList(user, job_ids, function (err, jobs) { + params = [userWIPKey, 0, -1]; + self.metadataBackend.redisCmd(DB, 'LRANGE', params, function (err, workInProgressJobs) { if (err) { return callback(err); } - if (jobs.length < initialLength) { - return self.list(user, callback); + debug('user %s has work in progress jobs %j', user, workInProgressJobs); + + if (workInProgressJobs.length < 0) { + return callback(); } - callback(null, jobs); + debug('delete user %s from index', user); + + params = [WORK_IN_PROGRESS_JOB.USER_INDEX_KEY, user]; + self.metadataBackend.redisCmd(DB, 'SREM', params, function (err) { + if (err) { + return callback(err); + } + + return callback(); + }); }); }); }; -JobBackend.prototype._getCleanedList = function (user, job_ids, callback) { - var self = this; - - var jobsQueue = queue(job_ids.length); +JobBackend.prototype.listWorkInProgressJobByUser = function (user, callback) { + var userWIPKey = WORK_IN_PROGRESS_JOB.PREFIX_USER + user; + var params = [userWIPKey, 0, -1]; + this.metadataBackend.redisCmd(WORK_IN_PROGRESS_JOB.DB, 'LRANGE', params, callback); +}; - job_ids.forEach(function(job_id) { - jobsQueue.defer(self._getIndexedJob.bind(self), job_id, user); - }); +JobBackend.prototype.listWorkInProgressJobs = function (callback) { + var self = this; + var DB = WORK_IN_PROGRESS_JOB.DB; - jobsQueue.awaitAll(function (err, jobs) { + var params = [WORK_IN_PROGRESS_JOB.USER_INDEX_KEY]; + this.metadataBackend.redisCmd(DB, 'SMEMBERS', params, function (err, workInProgressUsers) { if (err) { return callback(err); } - callback(null, jobs.filter(function (job) { - return job ? true : false; - })); + if (workInProgressUsers < 1) { + return callback(null, {}); + } + + debug('found %j work in progress users', workInProgressUsers); + + var usersQueue = queue(4); + + workInProgressUsers.forEach(function (user) { + usersQueue.defer(self.listWorkInProgressJobByUser.bind(self), user); + }); + + usersQueue.awaitAll(function (err, userWorkInProgressJobs) { + if (err) { + return callback(err); + } + + var workInProgressJobs = workInProgressUsers.reduce(function (users, user, index) { + users[user] = userWorkInProgressJobs[index]; + debug('found %j work in progress jobs for user %s', userWorkInProgressJobs[index], user); + return users; + }, {}); + + callback(null, workInProgressJobs); + }); }); }; -JobBackend.prototype._getIndexedJob = function (job_id, user, callback) { +JobBackend.prototype.setTTL = function (job, callback) { var self = this; + var redisKey = REDIS_PREFIX + job.job_id; - this.get(job_id, function (err, job) { - if (err && err.name === 'NotFoundError') { - return self.userIndexer.remove(user, job_id, function (err) { - if (err) { - debug('Error removing key %s in user set', job_id, err); - } - callback(); - }); - } - - if (err) { - return callback(err); - } + if (!JobStatus.isFinal(job.status)) { + return callback(); + } - callback(null, job); - }); + self.metadataBackend.redisCmd(REDIS_DB, 'EXPIRE', [ redisKey, this.inSecondsJobTTLAfterFinished ], callback); }; module.exports = JobBackend; diff --git a/batch/job_publisher.js b/batch/job_publisher.js deleted file mode 100644 index 86cc38d87..000000000 --- a/batch/job_publisher.js +++ /dev/null @@ -1,12 +0,0 @@ -'use strict'; - -function JobPublisher(redis) { - this.channel = 'batch:hosts'; - this.client = redis.createClient(global.settings.redis_port, global.settings.redis_host); -} - -JobPublisher.prototype.publish = function (host) { - this.client.publish(this.channel, host); -}; - -module.exports = JobPublisher; diff --git a/batch/job_queue.js b/batch/job_queue.js index e9e9f46dc..7ed1cc59d 100644 --- a/batch/job_queue.js +++ b/batch/job_queue.js @@ -1,21 +1,44 @@ 'use strict'; -function JobQueue(metadataBackend) { +var debug = require('./util/debug')('queue'); + +function JobQueue(metadataBackend, jobPublisher) { this.metadataBackend = metadataBackend; - this.db = 5; - this.redisPrefix = 'batch:queues:'; + this.jobPublisher = jobPublisher; } -JobQueue.prototype.enqueue = function (job_id, host, callback) { - this.metadataBackend.redisCmd(this.db, 'LPUSH', [ this.redisPrefix + host, job_id ], callback); +module.exports = JobQueue; + +var QUEUE = { + DB: 5, + PREFIX: 'batch:queue:' +}; +module.exports.QUEUE = QUEUE; + +JobQueue.prototype.enqueue = function (user, jobId, callback) { + debug('JobQueue.enqueue user=%s, jobId=%s', user, jobId); + this.metadataBackend.redisCmd(QUEUE.DB, 'LPUSH', [ QUEUE.PREFIX + user, jobId ], function (err) { + if (err) { + return callback(err); + } + + this.jobPublisher.publish(user); + callback(); + }.bind(this)); }; -JobQueue.prototype.dequeue = function (host, callback) { - this.metadataBackend.redisCmd(this.db, 'RPOP', [ this.redisPrefix + host ], callback); +JobQueue.prototype.size = function (user, callback) { + this.metadataBackend.redisCmd(QUEUE.DB, 'LLEN', [ QUEUE.PREFIX + user ], callback); }; -JobQueue.prototype.enqueueFirst = function (job_id, host, callback) { - this.metadataBackend.redisCmd(this.db, 'RPUSH', [ this.redisPrefix + host, job_id ], callback); +JobQueue.prototype.dequeue = function (user, callback) { + this.metadataBackend.redisCmd(QUEUE.DB, 'RPOP', [ QUEUE.PREFIX + user ], function(err, jobId) { + debug('JobQueue.dequeued user=%s, jobId=%s', user, jobId); + return callback(err, jobId); + }); }; -module.exports = JobQueue; +JobQueue.prototype.enqueueFirst = function (user, jobId, callback) { + debug('JobQueue.enqueueFirst user=%s, jobId=%s', user, jobId); + this.metadataBackend.redisCmd(QUEUE.DB, 'RPUSH', [ QUEUE.PREFIX + user, jobId ], callback); +}; diff --git a/batch/job_queue_pool.js b/batch/job_queue_pool.js deleted file mode 100644 index 8369d657b..000000000 --- a/batch/job_queue_pool.js +++ /dev/null @@ -1,55 +0,0 @@ -'use strict'; - -var JobQueue = require('./job_queue'); - -function JobQueuePool(metadataBackend) { - this.metadataBackend = metadataBackend; - this.queues = {}; -} - -JobQueuePool.prototype.get = function (host) { - return this.queues[host]; -}; - -JobQueuePool.prototype.getQueue = function (host) { - if (this.get(host)) { - return this.get(host).queue; - } -}; - -JobQueuePool.prototype.removeQueue = function (host) { - if (this.queues[host].queue) { - delete this.queues[host].queue; - } -}; - -JobQueuePool.prototype.list = function () { - return Object.keys(this.queues); -}; - -JobQueuePool.prototype.createQueue = function (host) { - this.queues[host] = { - queue: new JobQueue(this.metadataBackend), - currentJobId: null - }; - - return this.getQueue(host); -}; - -JobQueuePool.prototype.setCurrentJobId = function (host, job_id) { - this.get(host).currentJobId = job_id; -}; - -JobQueuePool.prototype.getCurrentJobId = function (host) { - if (this.get(host).currentJobId) { - return this.get(host).currentJobId; - } -}; - -JobQueuePool.prototype.removeCurrentJobId = function (host) { - if (this.get(host).currentJobId) { - delete this.get(host).currentJobId; - } -}; - -module.exports = JobQueuePool; diff --git a/batch/job_runner.js b/batch/job_runner.js index 6dada4e43..80c905c8f 100644 --- a/batch/job_runner.js +++ b/batch/job_runner.js @@ -2,87 +2,135 @@ var errorCodes = require('../app/postgresql/error_codes').codeToCondition; var jobStatus = require('./job_status'); +var Profiler = require('step-profiler'); +var _ = require('underscore'); -function JobRunner(jobService, jobQueue, queryRunner, userDatabaseMetadataService) { +var REDIS_LIMITS = { + DB: 5, + PREFIX: 'limits:batch:' // + username +}; + +function JobRunner(jobService, jobQueue, queryRunner, metadataBackend, statsdClient) { this.jobService = jobService; this.jobQueue = jobQueue; this.queryRunner = queryRunner; - this.userDatabaseMetadataService = userDatabaseMetadataService; // TODO: move to queryRunner + this.metadataBackend = metadataBackend; + this.statsdClient = statsdClient; } JobRunner.prototype.run = function (job_id, callback) { var self = this; + var profiler = new Profiler({ statsd_client: self.statsdClient }); + profiler.start('sqlapi.batch.job'); + self.jobService.get(job_id, function (err, job) { if (err) { return callback(err); } - var query = job.getNextQuery(); + self.getQueryStatementTimeout(job.data.user, function(err, timeout) { + if (err) { + return callback(err); + } - try { - job.setStatus(jobStatus.RUNNING); - } catch (err) { - return callback(err); - } + var query = job.getNextQuery(); - self.jobService.save(job, function (err, job) { - if (err) { + if (_.isObject(query)) { + if (Number.isFinite(query.timeout) && query.timeout > 0) { + timeout = Math.min(timeout, query.timeout); + } + query = query.query; + } + + try { + job.setStatus(jobStatus.RUNNING); + } catch (err) { return callback(err); } - self._run(job, query, callback); + self.jobService.save(job, function (err, job) { + if (err) { + return callback(err); + } + + profiler.done('running'); + + self._run(job, query, timeout, profiler, callback); + }); }); }); }; -JobRunner.prototype._run = function (job, query, callback) { +JobRunner.prototype.getQueryStatementTimeout = function(username, callback) { + var timeout = 12 * 3600 * 1000; + if (Number.isFinite(global.settings.batch_query_timeout)) { + timeout = global.settings.batch_query_timeout; + } + + var batchLimitsKey = REDIS_LIMITS.PREFIX + username; + this.metadataBackend.redisCmd(REDIS_LIMITS.DB, 'HGET', [batchLimitsKey, 'timeout'], function(err, timeoutLimit) { + if (timeoutLimit !== null && Number.isFinite(+timeoutLimit)) { + timeout = +timeoutLimit; + } + + return callback(null, timeout); + }); +}; + +JobRunner.prototype._run = function (job, query, timeout, profiler, callback) { var self = this; - // TODO: move to query - self.userDatabaseMetadataService.getUserMetadata(job.data.user, function (err, userDatabaseMetadata) { + self.queryRunner.run(job.data.job_id, query, job.data.user, timeout, function (err /*, result */) { if (err) { - return callback(err); + if (!err.code) { + return callback(err); + } + // if query has been cancelled then it's going to get the current + // job status saved by query_canceller + if (cancelledByUser(err)) { + return self.jobService.get(job.data.job_id, callback); + } } - self.queryRunner.run(job.data.job_id, query, userDatabaseMetadata, function (err /*, result */) { + try { if (err) { - // if query has been cancelled then it's going to get the current - // job status saved by query_canceller - if (errorCodes[err.code.toString()] === 'query_canceled') { - return self.jobService.get(job.data.job_id, callback); - } + profiler.done('failed'); + job.setStatus(jobStatus.FAILED, err.message); + } else { + profiler.done('success'); + job.setStatus(jobStatus.DONE); } + } catch (err) { + return callback(err); + } - try { - if (err) { - job.setStatus(jobStatus.FAILED, err.message); - } else { - job.setStatus(jobStatus.DONE); - } - } catch (err) { + self.jobService.save(job, function (err, job) { + if (err) { return callback(err); } - self.jobService.save(job, function (err, job) { + profiler.done('done'); + profiler.end(); + profiler.sendStats(); + + if (!job.hasNextQuery()) { + return callback(null, job); + } + + self.jobQueue.enqueueFirst(job.data.user, job.data.job_id, function (err) { if (err) { return callback(err); } - if (!job.hasNextQuery()) { - return callback(null, job); - } - - self.jobQueue.enqueue(job.data.job_id, userDatabaseMetadata.host, function (err) { - if (err) { - return callback(err); - } - - callback(null, job); - }); + callback(null, job); }); }); }); }; +function cancelledByUser(err) { + return errorCodes[err.code.toString()] === 'query_canceled' && err.message.match(/user.*request/); +} + module.exports = JobRunner; diff --git a/batch/job_service.js b/batch/job_service.js index cf5197bdf..d81b3cd8c 100644 --- a/batch/job_service.js +++ b/batch/job_service.js @@ -29,31 +29,6 @@ JobService.prototype.get = function (job_id, callback) { }); }; -JobService.prototype.list = function (user, callback) { - this.jobBackend.list(user, function (err, dataList) { - if (err) { - return callback(err); - } - - var jobList = dataList.map(function (data) { - var job; - - try { - job = JobFactory.create(data); - } catch (err) { - return debug(err); - } - - return job; - }) - .filter(function (job) { - return job !== undefined; - }); - - callback(null, jobList); - }); -}; - JobService.prototype.create = function (data, callback) { try { var job = JobFactory.create(data); @@ -69,23 +44,6 @@ JobService.prototype.create = function (data, callback) { } }; -JobService.prototype.update = function (data, callback) { - var self = this; - - self.get(data.job_id, function (err, job) { - if (err) { - return callback(err); - } - - try { - job.setQuery(data.query); - self.save(job, callback); - } catch (err) { - return callback(err); - } - }); -}; - JobService.prototype.save = function (job, callback) { var self = this; @@ -164,3 +122,15 @@ JobService.prototype.drain = function (job_id, callback) { }); }); }; + +JobService.prototype.addWorkInProgressJob = function (user, jobId, callback) { + this.jobBackend.addWorkInProgressJob(user, jobId, callback); +}; + +JobService.prototype.clearWorkInProgressJob = function (user, jobId, callback) { + this.jobBackend.clearWorkInProgressJob(user, jobId, callback); +}; + +JobService.prototype.listWorkInProgressJobs = function (callback) { + this.jobBackend.listWorkInProgressJobs(callback); +}; diff --git a/batch/job_status.js b/batch/job_status.js index d212679f2..ef5ad1710 100644 --- a/batch/job_status.js +++ b/batch/job_status.js @@ -6,7 +6,18 @@ var JOB_STATUS_ENUM = { DONE: 'done', CANCELLED: 'cancelled', FAILED: 'failed', + SKIPPED: 'skipped', UNKNOWN: 'unknown' }; module.exports = JOB_STATUS_ENUM; + +var finalStatus = [ + JOB_STATUS_ENUM.CANCELLED, + JOB_STATUS_ENUM.DONE, + JOB_STATUS_ENUM.FAILED, + JOB_STATUS_ENUM.UNKNOWN +]; +module.exports.isFinal = function(status) { + return finalStatus.indexOf(status) !== -1; +}; diff --git a/batch/job_subscriber.js b/batch/job_subscriber.js deleted file mode 100644 index 2c411c29b..000000000 --- a/batch/job_subscriber.js +++ /dev/null @@ -1,45 +0,0 @@ -'use strict'; - -var debug = require('./util/debug')('job-subscriber'); -var SUBSCRIBE_INTERVAL_IN_MILLISECONDS = 10 * 60 * 1000; // 10 minutes - -function _subscribe(client, channel, queueSeeker, onMessage) { - queueSeeker.seek(onMessage, function (err) { - if (err) { - debug(err); - } - - client.removeAllListeners('message'); - client.unsubscribe(channel); - client.subscribe(channel); - client.on('message', onMessage); - }); -} - -function JobSubscriber(redis, queueSeeker) { - this.channel = 'batch:hosts'; - this.client = redis.createClient(global.settings.redis_port, global.settings.redis_host); - this.queueSeeker = queueSeeker; -} - -module.exports = JobSubscriber; - -JobSubscriber.prototype.subscribe = function (onMessage) { - var self = this; - - _subscribe(this.client, this.channel, this.queueSeeker, onMessage); - - this.seekerInterval = setInterval( - _subscribe, - SUBSCRIBE_INTERVAL_IN_MILLISECONDS, - this.client, - this.channel, - self.queueSeeker, - onMessage - ); -}; - -JobSubscriber.prototype.unsubscribe = function () { - clearInterval(this.seekerInterval); - this.client.unsubscribe(this.channel); -}; diff --git a/batch/leader/locker.js b/batch/leader/locker.js new file mode 100644 index 000000000..463e74cee --- /dev/null +++ b/batch/leader/locker.js @@ -0,0 +1,74 @@ +'use strict'; + +var RedisDistlockLocker = require('./provider/redis-distlock'); +var debug = require('../util/debug')('leader-locker'); +var EventEmitter = require('events').EventEmitter; +var util = require('util'); + +var LOCK = { + TTL: 5000 +}; + +function Locker(locker, ttl) { + EventEmitter.call(this); + this.locker = locker; + this.ttl = (Number.isFinite(ttl) && ttl > 0) ? ttl : LOCK.TTL; + this.renewInterval = this.ttl / 5; + this.intervalIds = {}; +} +util.inherits(Locker, EventEmitter); + +module.exports = Locker; + +Locker.prototype.lock = function(resource, callback) { + var self = this; + debug('Locker.lock(%s, %d)', resource, this.ttl); + this.locker.lock(resource, this.ttl, function (err, lock) { + if (!err) { + self.startRenewal(resource); + } + return callback(err, lock); + }); +}; + +Locker.prototype.unlock = function(resource, callback) { + var self = this; + debug('Locker.unlock(%s)', resource); + this.locker.unlock(resource, function(err) { + self.stopRenewal(resource); + return callback(err); + }); +}; + +Locker.prototype.startRenewal = function(resource) { + var self = this; + if (!this.intervalIds.hasOwnProperty(resource)) { + this.intervalIds[resource] = setInterval(function() { + debug('Trying to extend lock resource=%s', resource); + self.locker.lock(resource, self.ttl, function(err, _lock) { + if (err) { + self.emit('error', err, resource); + return self.stopRenewal(resource); + } + if (_lock) { + debug('Extended lock resource=%s', resource); + } + }); + }, this.renewInterval); + } +}; + +Locker.prototype.stopRenewal = function(resource) { + if (this.intervalIds.hasOwnProperty(resource)) { + clearInterval(this.intervalIds[resource]); + delete this.intervalIds[resource]; + } +}; + +module.exports.create = function createLocker(type, config) { + if (type !== 'redis-distlock') { + throw new Error('Invalid type Locker type. Valid types are: "redis-distlock"'); + } + var locker = new RedisDistlockLocker(config.pool); + return new Locker(locker, config.ttl); +}; diff --git a/batch/leader/provider/redis-distlock.js b/batch/leader/provider/redis-distlock.js new file mode 100644 index 000000000..01e944f78 --- /dev/null +++ b/batch/leader/provider/redis-distlock.js @@ -0,0 +1,111 @@ +'use strict'; + +var REDIS_DISTLOCK = { + DB: 5, + PREFIX: 'batch:locks:' +}; + +var Redlock = require('redlock'); +var debug = require('../../util/debug')('leader:redis-distlock'); + +function RedisDistlockLocker(redisPool) { + this.pool = redisPool; + this.redlock = new Redlock([{}], { + // see http://redis.io/topics/distlock + driftFactor: 0.01, // time in ms + // the max number of times Redlock will attempt to lock a resource before failing + retryCount: 3, + // the time in ms between attempts + retryDelay: 100 + }); + this._locks = {}; +} + +module.exports = RedisDistlockLocker; +module.exports.type = 'redis-distlock'; + +function resourceId(resource) { + return REDIS_DISTLOCK.PREFIX + resource; +} + +RedisDistlockLocker.prototype.lock = function(resource, ttl, callback) { + var self = this; + debug('RedisDistlockLocker.lock(%s, %d)', resource, ttl); + var lockId = resourceId(resource); + + var lock = this._getLock(lockId); + function acquireCallback(err, _lock) { + if (err) { + return callback(err); + } + self._setLock(lockId, _lock); + return callback(null, _lock); + } + if (lock) { + return this._tryExtend(lock, ttl, function(err, _lock) { + if (err) { + return self._tryAcquire(lockId, ttl, acquireCallback); + } + + return callback(null, _lock); + }); + } else { + return this._tryAcquire(lockId, ttl, acquireCallback); + } +}; + +RedisDistlockLocker.prototype.unlock = function(resource, callback) { + var self = this; + var lock = this._getLock(resourceId(resource)); + if (lock) { + this.pool.acquire(REDIS_DISTLOCK.DB, function (err, client) { + if (err) { + return callback(err); + } + self.redlock.servers = [client]; + return self.redlock.unlock(lock, function(err) { + self.pool.release(REDIS_DISTLOCK.DB, client); + return callback(err); + }); + }); + } +}; + +RedisDistlockLocker.prototype._getLock = function(resource) { + if (this._locks.hasOwnProperty(resource)) { + return this._locks[resource]; + } + return null; +}; + +RedisDistlockLocker.prototype._setLock = function(resource, lock) { + this._locks[resource] = lock; +}; + +RedisDistlockLocker.prototype._tryExtend = function(lock, ttl, callback) { + var self = this; + this.pool.acquire(REDIS_DISTLOCK.DB, function (err, client) { + if (err) { + return callback(err); + } + self.redlock.servers = [client]; + return lock.extend(ttl, function(err, _lock) { + self.pool.release(REDIS_DISTLOCK.DB, client); + return callback(err, _lock); + }); + }); +}; + +RedisDistlockLocker.prototype._tryAcquire = function(resource, ttl, callback) { + var self = this; + this.pool.acquire(REDIS_DISTLOCK.DB, function (err, client) { + if (err) { + return callback(err); + } + self.redlock.servers = [client]; + return self.redlock.lock(resource, ttl, function(err, _lock) { + self.pool.release(REDIS_DISTLOCK.DB, client); + return callback(err, _lock); + }); + }); +}; diff --git a/batch/maintenance/host-user-queue-mover.js b/batch/maintenance/host-user-queue-mover.js new file mode 100644 index 000000000..c2cec1d33 --- /dev/null +++ b/batch/maintenance/host-user-queue-mover.js @@ -0,0 +1,144 @@ +'use strict'; + +var asyncQ = require('queue-async'); +var debug = require('../util/debug')('queue-mover'); +var forever = require('../util/forever'); + +var QUEUE = { + OLD: { + DB: 5, + PREFIX: 'batch:queues:' // host + }, + NEW: { + DB: 5, + PREFIX: 'batch:queue:' // user + } +}; + +function HostUserQueueMover(jobQueue, jobService, locker, redisPool) { + this.jobQueue = jobQueue; + this.jobService = jobService; + this.locker = locker; + this.pool = redisPool; +} + +module.exports = HostUserQueueMover; + +HostUserQueueMover.prototype.moveOldJobs = function(callback) { + var self = this; + this.getOldQueues(function(err, hosts) { + var async = asyncQ(4); + hosts.forEach(function(host) { + async.defer(self.moveOldQueueJobs.bind(self), host); + }); + + async.awaitAll(function (err) { + if (err) { + debug('Something went wrong moving jobs', err); + } else { + debug('Finished moving all jobs'); + } + + callback(); + }); + }); +}; + +HostUserQueueMover.prototype.moveOldQueueJobs = function(host, callback) { + var self = this; + // do forever, it does not throw a stack overflow + forever( + function (next) { + self.locker.lock(host, function(err) { + // we didn't get the lock for the host + if (err) { + debug('Could not lock host=%s. Reason: %s', host, err.message); + return next(err); + } + debug('Locked host=%s', host); + self.processNextJob(host, next); + }); + }, + function (err) { + if (err) { + debug(err.name === 'EmptyQueue' ? err.message : err); + } + self.locker.unlock(host, callback); + } + ); +}; + +//this.metadataBackend.redisCmd(QUEUE.DB, 'RPOP', [ QUEUE.PREFIX + user ], callback); + +HostUserQueueMover.prototype.processNextJob = function (host, callback) { + var self = this; + this.pool.acquire(QUEUE.OLD.DB, function(err, client) { + if (err) { + return callback(err); + } + + client.lpop(QUEUE.OLD.PREFIX + host, function(err, jobId) { + self.pool.release(QUEUE.OLD.DB, client); + debug('Found jobId=%s at queue=%s', jobId, host); + if (!jobId) { + var emptyQueueError = new Error('Empty queue'); + emptyQueueError.name = 'EmptyQueue'; + return callback(emptyQueueError); + } + self.jobService.get(jobId, function(err, job) { + if (err) { + debug(err); + return callback(); + } + if (job) { + return self.jobQueue.enqueueFirst(job.data.user, jobId, function() { + return callback(); + }); + } + return callback(); + }); + }); + }); +}; + +HostUserQueueMover.prototype.getOldQueues = function(callback) { + var initialCursor = ['0']; + var hosts = {}; + var self = this; + + this.pool.acquire(QUEUE.OLD.DB, function(err, client) { + if (err) { + return callback(err); + } + self._getOldQueues(client, initialCursor, hosts, function(err, hosts) { + self.pool.release(QUEUE.DB, client); + return callback(err, Object.keys(hosts)); + }); + }); +}; + +HostUserQueueMover.prototype._getOldQueues = function (client, cursor, hosts, callback) { + var self = this; + var redisParams = [cursor[0], 'MATCH', QUEUE.OLD.PREFIX + '*']; + + client.scan(redisParams, function(err, currentCursor) { + if (err) { + return callback(null, hosts); + } + + var queues = currentCursor[1]; + if (queues) { + queues.forEach(function (queue) { + var user = queue.substr(QUEUE.OLD.PREFIX.length); + hosts[user] = true; + }); + } + + var hasMore = currentCursor[0] !== '0'; + if (!hasMore) { + return callback(null, hosts); + } + + self._getOldQueues(client, currentCursor, hosts, callback); + }); +}; diff --git a/batch/models/job_base.js b/batch/models/job_base.js index 5169e5607..b2143ac08 100644 --- a/batch/models/job_base.js +++ b/batch/models/job_base.js @@ -1,28 +1,22 @@ 'use strict'; -var assert = require('assert'); +var util = require('util'); var uuid = require('node-uuid'); +var JobStateMachine = require('./job_state_machine'); var jobStatus = require('../job_status'); -var validStatusTransitions = [ - [jobStatus.PENDING, jobStatus.RUNNING], - [jobStatus.PENDING, jobStatus.CANCELLED], - [jobStatus.PENDING, jobStatus.UNKNOWN], - [jobStatus.RUNNING, jobStatus.DONE], - [jobStatus.RUNNING, jobStatus.FAILED], - [jobStatus.RUNNING, jobStatus.CANCELLED], - [jobStatus.RUNNING, jobStatus.PENDING], - [jobStatus.RUNNING, jobStatus.UNKNOWN] -]; var mandatoryProperties = [ 'job_id', 'status', 'query', 'created_at', 'updated_at', + 'host', 'user' ]; function JobBase(data) { + JobStateMachine.call(this); + var now = new Date().toISOString(); this.data = data; @@ -39,24 +33,10 @@ function JobBase(data) { this.data.updated_at = now; } } +util.inherits(JobBase, JobStateMachine); module.exports = JobBase; -JobBase.prototype.isValidStatusTransition = function (initialStatus, finalStatus) { - var transition = [ initialStatus, finalStatus ]; - - for (var i = 0; i < validStatusTransitions.length; i++) { - try { - assert.deepEqual(transition, validStatusTransitions[i]); - return true; - } catch (e) { - continue; - } - } - - return false; -}; - // should be implemented by childs JobBase.prototype.getNextQuery = function () { throw new Error('Unimplemented method'); @@ -104,7 +84,7 @@ JobBase.prototype.setQuery = function (query) { JobBase.prototype.setStatus = function (finalStatus, errorMesssage) { var now = new Date().toISOString(); var initialStatus = this.data.status; - var isValid = this.isValidStatusTransition(initialStatus, finalStatus); + var isValid = this.isValidTransition(initialStatus, finalStatus); if (!isValid) { throw new Error('Cannot set status from ' + initialStatus + ' to ' + finalStatus); @@ -131,3 +111,7 @@ JobBase.prototype.serialize = function () { return data; }; + +JobBase.prototype.log = function(/*logger*/) { + return false; +}; diff --git a/batch/models/job_fallback.js b/batch/models/job_fallback.js index abc4a1b09..31317c2b5 100644 --- a/batch/models/job_fallback.js +++ b/batch/models/job_fallback.js @@ -2,35 +2,30 @@ var util = require('util'); var JobBase = require('./job_base'); -var jobStatus = require('../job_status'); -var breakStatus = [ - jobStatus.CANCELLED, - jobStatus.FAILED, - jobStatus.UNKNOWN -]; -function isBreakStatus(status) { - return breakStatus.indexOf(status) !== -1; -} -var finalStatus = [ - jobStatus.CANCELLED, - jobStatus.DONE, - jobStatus.FAILED, - jobStatus.UNKNOWN -]; -function isFinalStatus(status) { - return finalStatus.indexOf(status) !== -1; -} +var JobStatus = require('../job_status'); +var QueryFallback = require('./query/query_fallback'); +var MainFallback = require('./query/main_fallback'); +var QueryFactory = require('./query/query_factory'); function JobFallback(jobDefinition) { JobBase.call(this, jobDefinition); this.init(); + + this.queries = []; + for (var i = 0; i < this.data.query.query.length; i++) { + this.queries[i] = QueryFactory.create(this.data, i); + } + + if (MainFallback.is(this.data)) { + this.fallback = new MainFallback(); + } } util.inherits(JobFallback, JobBase); module.exports = JobFallback; -// from user: { +// 1. from user: { // query: { // query: [{ // query: 'select ...', @@ -39,7 +34,8 @@ module.exports = JobFallback; // onerror: 'select ...' // } // } -// from redis: { +// +// 2. from redis: { // status: 'pending', // fallback_status: 'pending' // query: { @@ -63,11 +59,7 @@ JobFallback.is = function (query) { } for (var i = 0; i < query.query.length; i++) { - if (!query.query[i].query) { - return false; - } - - if (typeof query.query[i].query !== 'string') { + if (!QueryFallback.is(query.query[i])) { return false; } } @@ -76,96 +68,63 @@ JobFallback.is = function (query) { }; JobFallback.prototype.init = function () { - // jshint maxcomplexity: 8 for (var i = 0; i < this.data.query.query.length; i++) { - if ((this.data.query.query[i].onsuccess || this.data.query.query[i].onerror) && - !this.data.query.query[i].status) { - this.data.query.query[i].status = jobStatus.PENDING; - this.data.query.query[i].fallback_status = jobStatus.PENDING; - } else if (!this.data.query.query[i].status){ - this.data.query.query[i].status = jobStatus.PENDING; + if (shouldInitStatus(this.data.query.query[i])){ + this.data.query.query[i].status = JobStatus.PENDING; + } + if (shouldInitQueryFallbackStatus(this.data.query.query[i])) { + this.data.query.query[i].fallback_status = JobStatus.PENDING; } } - if ((this.data.query.onsuccess || this.data.query.onerror) && !this.data.status) { - this.data.status = jobStatus.PENDING; - this.data.fallback_status = jobStatus.PENDING; - - } else if (!this.data.status) { - this.data.status = jobStatus.PENDING; + if (shouldInitStatus(this.data)) { + this.data.status = JobStatus.PENDING; } -}; - -JobFallback.prototype.getNextQuery = function () { - var query = this._getNextQueryFromQuery(); - if (!query) { - query = this._getNextQueryFromJobFallback(); + if (shouldInitFallbackStatus(this.data)) { + this.data.fallback_status = JobStatus.PENDING; } - - return query; }; -JobFallback.prototype._hasNextQueryFromQuery = function () { - return !!this._getNextQueryFromQuery(); -}; +function shouldInitStatus(jobOrQuery) { + return !jobOrQuery.status; +} -JobFallback.prototype._getNextQueryFromQuery = function () { - // jshint maxcomplexity: 8 - for (var i = 0; i < this.data.query.query.length; i++) { +function shouldInitQueryFallbackStatus(query) { + return (query.onsuccess || query.onerror) && !query.fallback_status; +} - if (this.data.query.query[i].fallback_status) { - if (this._isNextQuery(i)) { - return this.data.query.query[i].query; - } else if (this._isNextQueryOnSuccess(i)) { - return this.data.query.query[i].onsuccess; - } else if (this._isNextQueryOnError(i)) { - return this.data.query.query[i].onerror; - } else if (isBreakStatus(this.data.query.query[i].status)) { - return; - } - } else if (this.data.query.query[i].status === jobStatus.PENDING) { - return this.data.query.query[i].query; - } - } -}; +function shouldInitFallbackStatus(job) { + return (job.query.onsuccess || job.query.onerror) && !job.fallback_status; +} -JobFallback.prototype._getNextQueryFromJobFallback = function () { - if (this.data.fallback_status) { - if (this._isNextQueryOnSuccessJob()) { - return this.data.query.onsuccess; - } else if (this._isNextQueryOnErrorJob()) { - return this.data.query.onerror; +JobFallback.prototype.getNextQueryFromQueries = function () { + for (var i = 0; i < this.queries.length; i++) { + if (this.queries[i].hasNextQuery(this.data)) { + return this.queries[i].getNextQuery(this.data); } } }; -JobFallback.prototype._isNextQuery = function (index) { - return this.data.query.query[index].status === jobStatus.PENDING; +JobFallback.prototype.hasNextQueryFromQueries = function () { + return !!this.getNextQueryFromQueries(); }; -JobFallback.prototype._isNextQueryOnSuccess = function (index) { - return this.data.query.query[index].status === jobStatus.DONE && - this.data.query.query[index].onsuccess && - this.data.query.query[index].fallback_status === jobStatus.PENDING; -}; +JobFallback.prototype.getNextQueryFromFallback = function () { + if (this.fallback && this.fallback.hasNextQuery(this.data)) { -JobFallback.prototype._isNextQueryOnError = function (index) { - return this.data.query.query[index].status === jobStatus.FAILED && - this.data.query.query[index].onerror && - this.data.query.query[index].fallback_status === jobStatus.PENDING; + return this.fallback.getNextQuery(this.data); + } }; -JobFallback.prototype._isNextQueryOnSuccessJob = function () { - return this.data.status === jobStatus.DONE && - this.data.query.onsuccess && - this.data.fallback_status === jobStatus.PENDING; -}; +JobFallback.prototype.getNextQuery = function () { + var query = this.getNextQueryFromQueries(); -JobFallback.prototype._isNextQueryOnErrorJob = function () { - return this.data.status === jobStatus.FAILED && - this.data.query.onerror && - this.data.fallback_status === jobStatus.PENDING; + if (!query) { + query = this.getNextQueryFromFallback(); + } + + return query; }; JobFallback.prototype.setQuery = function (query) { @@ -178,121 +137,142 @@ JobFallback.prototype.setQuery = function (query) { JobFallback.prototype.setStatus = function (status, errorMesssage) { var now = new Date().toISOString(); - var resultFromQuery = this._setQueryStatus(status, errorMesssage); - var resultFromJob = this._setJobStatus(status, resultFromQuery.isChangeAppliedToQueryFallback, errorMesssage); - if (!resultFromJob.isValid && !resultFromQuery.isValid) { - throw new Error('Cannot set status from ' + this.data.status + ' to ' + status); + var hasChanged = this.setQueryStatus(status, this.data, errorMesssage); + hasChanged = this.setJobStatus(status, this.data, hasChanged, errorMesssage); + hasChanged = this.setFallbackStatus(status, this.data, hasChanged); + + if (!hasChanged.isValid) { + throw new Error('Cannot set status to ' + status); } this.data.updated_at = now; }; -JobFallback.prototype._getLastStatusFromFinishedQuery = function () { - var lastStatus = jobStatus.DONE; - - for (var i = 0; i < this.data.query.query.length; i++) { - if (this.data.query.query[i].fallback_status) { - if (isFinalStatus(this.data.query.query[i].status)) { - lastStatus = this.data.query.query[i].status; - } else { - break; - } - } else { - if (isFinalStatus(this.data.query.query[i].status)) { - lastStatus = this.data.query.query[i].status; - } else { - break; - } - } - } - - return lastStatus; +JobFallback.prototype.setQueryStatus = function (status, job, errorMesssage) { + return this.queries.reduce(function (hasChanged, query) { + var result = query.setStatus(status, this.data, hasChanged, errorMesssage); + return result.isValid ? result : hasChanged; + }.bind(this), { isValid: false, appliedToFallback: false }); }; -JobFallback.prototype._setJobStatus = function (status, isChangeAppliedToQueryFallback, errorMesssage) { - var isValid = false; - - status = this._shiftJobStatus(status, isChangeAppliedToQueryFallback); - - isValid = this.isValidStatusTransition(this.data.status, status); - - if (isValid) { - this.data.status = status; - } else if (this.data.fallback_status) { +JobFallback.prototype.setJobStatus = function (status, job, hasChanged, errorMesssage) { + var result = { + isValid: false, + appliedToFallback: false + }; - isValid = this.isValidStatusTransition(this.data.fallback_status, status); + status = this.shiftStatus(status, hasChanged); - if (isValid) { - this.data.fallback_status = status; + result.isValid = this.isValidTransition(job.status, status); + if (result.isValid) { + job.status = status; + if (status === JobStatus.FAILED && errorMesssage && !hasChanged.appliedToFallback) { + job.failed_reason = errorMesssage; } } - if (status === jobStatus.FAILED && errorMesssage && !isChangeAppliedToQueryFallback) { - this.data.failed_reason = errorMesssage; + return result.isValid ? result : hasChanged; +}; + +JobFallback.prototype.setFallbackStatus = function (status, job, hasChanged) { + var result = hasChanged; + + if (this.fallback && !this.hasNextQueryFromQueries()) { + result = this.fallback.setStatus(status, job, hasChanged); } - return { - isValid: isValid - }; + return result.isValid ? result : hasChanged; }; -JobFallback.prototype._shiftJobStatus = function (status, isChangeAppliedToQueryFallback) { +JobFallback.prototype.shiftStatus = function (status, hasChanged) { // jshint maxcomplexity: 7 - - // In some scenarios we have to change the normal flow in order to keep consistency - // between query's status and job's status. - - if (isChangeAppliedToQueryFallback) { - if (!this._hasNextQueryFromQuery() && (status === jobStatus.DONE || status === jobStatus.FAILED)) { - status = this._getLastStatusFromFinishedQuery(); - } else if (status === jobStatus.DONE || status === jobStatus.FAILED){ - status = jobStatus.PENDING; + if (hasChanged.appliedToFallback) { + if (!this.hasNextQueryFromQueries() && (status === JobStatus.DONE || status === JobStatus.FAILED)) { + status = this.getLastFinishedStatus(); + } else if (status === JobStatus.DONE || status === JobStatus.FAILED){ + status = JobStatus.PENDING; } - } else if (this._hasNextQueryFromQuery() && status !== jobStatus.RUNNING) { - status = jobStatus.PENDING; + } else if (this.hasNextQueryFromQueries() && status !== JobStatus.RUNNING) { + status = JobStatus.PENDING; } return status; }; +JobFallback.prototype.getLastFinishedStatus = function () { + return this.queries.reduce(function (lastFinished, query) { + var status = query.getStatus(this.data); + return this.isFinalStatus(status) ? status : lastFinished; + }.bind(this), JobStatus.DONE); +}; -JobFallback.prototype._setQueryStatus = function (status, errorMesssage) { - // jshint maxcomplexity: 7 - var isValid = false; - var isChangeAppliedToQueryFallback = false; - - for (var i = 0; i < this.data.query.query.length; i++) { - isValid = this.isValidStatusTransition(this.data.query.query[i].status, status); - - if (isValid) { - this.data.query.query[i].status = status; +JobFallback.prototype.log = function(logger) { + if (!isFinished(this)) { + return false; + } - if (status === jobStatus.FAILED && errorMesssage) { - this.data.query.query[i].failed_reason = errorMesssage; + var queries = this.data.query.query; + + for (var i = 0; i < queries.length; i++) { + var query = queries[i]; + + var logEntry = { + created: this.data.created_at, + waiting: elapsedTime(this.data.created_at, query.started_at), + time: query.started_at, + endtime: query.ended_at, + username: this.data.user, + dbhost: this.data.host, + job: this.data.job_id, + elapsed: elapsedTime(query.started_at, query.ended_at) + }; + + var queryId = query.id; + + var tag = 'query'; + if (queryId) { + logEntry.query_id = queryId; + + var node = parseQueryId(queryId); + if (node) { + logEntry.analysis = node.analysisId; + logEntry.node = node.nodeId; + logEntry.type = node.nodeType; + tag = 'analysis'; } - - break; } - if (this.data.query.query[i].fallback_status) { - isValid = this.isValidStatusTransition(this.data.query.query[i].fallback_status, status); + logger.info(logEntry, tag); + } + + return true; +}; + +function isFinished (job) { + return JobStatus.isFinal(job.data.status) && + (!job.data.fallback_status || JobStatus.isFinal(job.data.fallback_status)); +} - if (isValid) { - this.data.query.query[i].fallback_status = status; +function parseQueryId (queryId) { + var data = queryId.split(':'); - if (status === jobStatus.FAILED && errorMesssage) { - this.data.query.query[i].failed_reason = errorMesssage; - } + if (data.length === 3) { + return { + analysisId: data[0], + nodeId: data[1], + nodeType: data[2] + }; + } + return null; +} - isChangeAppliedToQueryFallback = true; - break; - } - } +function elapsedTime (started_at, ended_at) { + if (!started_at || !ended_at) { + return; } - return { - isValid: isValid, - isChangeAppliedToQueryFallback: isChangeAppliedToQueryFallback - }; -}; + var start = new Date(started_at); + var end = new Date(ended_at); + return end.getTime() - start.getTime(); +} diff --git a/batch/models/job_multiple.js b/batch/models/job_multiple.js index a1fb8317b..85cf1d872 100644 --- a/batch/models/job_multiple.js +++ b/batch/models/job_multiple.js @@ -76,7 +76,7 @@ JobMultiple.prototype.setStatus = function (finalStatus, errorMesssage) { } for (var i = 0; i < this.data.query.length; i++) { - var isValid = JobMultiple.super_.prototype.isValidStatusTransition(this.data.query[i].status, finalStatus); + var isValid = JobMultiple.super_.prototype.isValidTransition(this.data.query[i].status, finalStatus); if (isValid) { this.data.query[i].status = finalStatus; diff --git a/batch/models/job_state_machine.js b/batch/models/job_state_machine.js new file mode 100644 index 000000000..f476b8a8f --- /dev/null +++ b/batch/models/job_state_machine.js @@ -0,0 +1,39 @@ +'use strict'; + +var assert = require('assert'); +var JobStatus = require('../job_status'); +var validStatusTransitions = [ + [JobStatus.PENDING, JobStatus.RUNNING], + [JobStatus.PENDING, JobStatus.CANCELLED], + [JobStatus.PENDING, JobStatus.UNKNOWN], + [JobStatus.PENDING, JobStatus.SKIPPED], + [JobStatus.RUNNING, JobStatus.DONE], + [JobStatus.RUNNING, JobStatus.FAILED], + [JobStatus.RUNNING, JobStatus.CANCELLED], + [JobStatus.RUNNING, JobStatus.PENDING], + [JobStatus.RUNNING, JobStatus.UNKNOWN] +]; + +function JobStateMachine () { +} + +module.exports = JobStateMachine; + +JobStateMachine.prototype.isValidTransition = function (initialStatus, finalStatus) { + var transition = [ initialStatus, finalStatus ]; + + for (var i = 0; i < validStatusTransitions.length; i++) { + try { + assert.deepEqual(transition, validStatusTransitions[i]); + return true; + } catch (e) { + continue; + } + } + + return false; +}; + +JobStateMachine.prototype.isFinalStatus = function (status) { + return JobStatus.isFinal(status); +}; diff --git a/batch/models/query/fallback.js b/batch/models/query/fallback.js new file mode 100644 index 000000000..1ce5f22f0 --- /dev/null +++ b/batch/models/query/fallback.js @@ -0,0 +1,78 @@ +'use strict'; + +var util = require('util'); +var QueryBase = require('./query_base'); +var jobStatus = require('../../job_status'); + +function Fallback(index) { + QueryBase.call(this, index); +} +util.inherits(Fallback, QueryBase); + +module.exports = Fallback; + +Fallback.is = function (query) { + if (query.onsuccess || query.onerror) { + return true; + } + return false; +}; + +Fallback.prototype.getNextQuery = function (job) { + if (this.hasOnSuccess(job)) { + return this.getOnSuccess(job); + } + if (this.hasOnError(job)) { + return this.getOnError(job); + } +}; + +Fallback.prototype.getOnSuccess = function (job) { + if (job.query.query[this.index].status === jobStatus.DONE && + job.query.query[this.index].fallback_status === jobStatus.PENDING) { + var onsuccessQuery = job.query.query[this.index].onsuccess; + if (onsuccessQuery) { + onsuccessQuery = onsuccessQuery.replace(/<%=\s*job_id\s*%>/g, job.job_id); + } + return onsuccessQuery; + } +}; + +Fallback.prototype.hasOnSuccess = function (job) { + return !!this.getOnSuccess(job); +}; + +Fallback.prototype.getOnError = function (job) { + if (job.query.query[this.index].status === jobStatus.FAILED && + job.query.query[this.index].fallback_status === jobStatus.PENDING) { + var onerrorQuery = job.query.query[this.index].onerror; + if (onerrorQuery) { + onerrorQuery = onerrorQuery.replace(/<%=\s*job_id\s*%>/g, job.job_id); + onerrorQuery = onerrorQuery.replace(/<%=\s*error_message\s*%>/g, job.query.query[this.index].failed_reason); + } + return onerrorQuery; + } +}; + +Fallback.prototype.hasOnError = function (job) { + return !!this.getOnError(job); +}; + +Fallback.prototype.setStatus = function (status, job, errorMessage) { + var isValid = false; + + isValid = this.isValidTransition(job.query.query[this.index].fallback_status, status); + + if (isValid) { + job.query.query[this.index].fallback_status = status; + if (status === jobStatus.FAILED && errorMessage) { + job.query.query[this.index].failed_reason = errorMessage; + } + } + + return isValid; +}; + +Fallback.prototype.getStatus = function (job) { + return job.query.query[this.index].fallback_status; +}; diff --git a/batch/models/query/main_fallback.js b/batch/models/query/main_fallback.js new file mode 100644 index 000000000..7c52194b7 --- /dev/null +++ b/batch/models/query/main_fallback.js @@ -0,0 +1,74 @@ +'use strict'; + +var util = require('util'); +var QueryBase = require('./query_base'); +var jobStatus = require('../../job_status'); + +function MainFallback() { + QueryBase.call(this); +} +util.inherits(MainFallback, QueryBase); + +module.exports = MainFallback; + +MainFallback.is = function (job) { + if (job.query.onsuccess || job.query.onerror) { + return true; + } + return false; +}; + +MainFallback.prototype.getNextQuery = function (job) { + if (this.hasOnSuccess(job)) { + return this.getOnSuccess(job); + } + + if (this.hasOnError(job)) { + return this.getOnError(job); + } +}; + +MainFallback.prototype.getOnSuccess = function (job) { + if (job.status === jobStatus.DONE && job.fallback_status === jobStatus.PENDING) { + return job.query.onsuccess; + } +}; + +MainFallback.prototype.hasOnSuccess = function (job) { + return !!this.getOnSuccess(job); +}; + +MainFallback.prototype.getOnError = function (job) { + if (job.status === jobStatus.FAILED && job.fallback_status === jobStatus.PENDING) { + return job.query.onerror; + } +}; + +MainFallback.prototype.hasOnError = function (job) { + return !!this.getOnError(job); +}; + +MainFallback.prototype.setStatus = function (status, job, previous) { + var isValid = false; + var appliedToFallback = false; + + if (previous.isValid && !previous.appliedToFallback) { + if (this.isFinalStatus(status) && !this.hasNextQuery(job)) { + isValid = this.isValidTransition(job.fallback_status, jobStatus.SKIPPED); + + if (isValid) { + job.fallback_status = jobStatus.SKIPPED; + appliedToFallback = true; + } + } + } else if (!previous.isValid) { + isValid = this.isValidTransition(job.fallback_status, status); + + if (isValid) { + job.fallback_status = status; + appliedToFallback = true; + } + } + + return { isValid: isValid, appliedToFallback: appliedToFallback }; +}; diff --git a/batch/models/query/query.js b/batch/models/query/query.js new file mode 100644 index 000000000..6970ae97e --- /dev/null +++ b/batch/models/query/query.js @@ -0,0 +1,57 @@ +'use strict'; + +var util = require('util'); +var QueryBase = require('./query_base'); +var jobStatus = require('../../job_status'); + +function Query(index) { + QueryBase.call(this, index); +} +util.inherits(Query, QueryBase); + +module.exports = Query; + +Query.is = function (query) { + if (query.query && typeof query.query === 'string') { + return true; + } + + return false; +}; + +Query.prototype.getNextQuery = function (job) { + if (job.query.query[this.index].status === jobStatus.PENDING) { + var query = { + query: job.query.query[this.index].query + }; + if (Number.isFinite(job.query.query[this.index].timeout)) { + query.timeout = job.query.query[this.index].timeout; + } + return query; + } +}; + +Query.prototype.setStatus = function (status, job, errorMesssage) { + var isValid = false; + + isValid = this.isValidTransition(job.query.query[this.index].status, status); + + if (isValid) { + job.query.query[this.index].status = status; + if (status === jobStatus.RUNNING) { + job.query.query[this.index].started_at = new Date().toISOString(); + } + if (this.isFinalStatus(status)) { + job.query.query[this.index].ended_at = new Date().toISOString(); + } + if (status === jobStatus.FAILED && errorMesssage) { + job.query.query[this.index].failed_reason = errorMesssage; + } + } + + return isValid; +}; + +Query.prototype.getStatus = function (job) { + return job.query.query[this.index].status; +}; diff --git a/batch/models/query/query_base.js b/batch/models/query/query_base.js new file mode 100644 index 000000000..737e1bf59 --- /dev/null +++ b/batch/models/query/query_base.js @@ -0,0 +1,31 @@ +'use strict'; + +var util = require('util'); +var JobStateMachine = require('../job_state_machine'); + +function QueryBase(index) { + JobStateMachine.call(this); + + this.index = index; +} +util.inherits(QueryBase, JobStateMachine); + +module.exports = QueryBase; + +// should be implemented +QueryBase.prototype.setStatus = function () { + throw new Error('Unimplemented method'); +}; + +// should be implemented +QueryBase.prototype.getNextQuery = function () { + throw new Error('Unimplemented method'); +}; + +QueryBase.prototype.hasNextQuery = function (job) { + return !!this.getNextQuery(job); +}; + +QueryBase.prototype.getStatus = function () { + throw new Error('Unimplemented method'); +}; diff --git a/batch/models/query/query_factory.js b/batch/models/query/query_factory.js new file mode 100644 index 000000000..c33534e09 --- /dev/null +++ b/batch/models/query/query_factory.js @@ -0,0 +1,16 @@ +'use strict'; + +var QueryFallback = require('./query_fallback'); + +function QueryFactory() { +} + +module.exports = QueryFactory; + +QueryFactory.create = function (job, index) { + if (QueryFallback.is(job.query.query[index])) { + return new QueryFallback(job, index); + } + + throw new Error('there is no query class for the provided query'); +}; diff --git a/batch/models/query/query_fallback.js b/batch/models/query/query_fallback.js new file mode 100644 index 000000000..cb0579a39 --- /dev/null +++ b/batch/models/query/query_fallback.js @@ -0,0 +1,75 @@ +'use strict'; + +var util = require('util'); +var QueryBase = require('./query_base'); +var Query = require('./query'); +var Fallback = require('./fallback'); +var jobStatus = require('../../job_status'); + +function QueryFallback(job, index) { + QueryBase.call(this, index); + + this.init(job, index); +} + +util.inherits(QueryFallback, QueryBase); + +QueryFallback.is = function (query) { + if (Query.is(query)) { + return true; + } + return false; +}; + +QueryFallback.prototype.init = function (job, index) { + this.query = new Query(index); + + if (Fallback.is(job.query.query[index])) { + this.fallback = new Fallback(index); + } +}; + +QueryFallback.prototype.getNextQuery = function (job) { + if (this.query.hasNextQuery(job)) { + return this.query.getNextQuery(job); + } + + if (this.fallback && this.fallback.hasNextQuery(job)) { + return this.fallback.getNextQuery(job); + } +}; + +QueryFallback.prototype.setStatus = function (status, job, previous, errorMesssage) { + // jshint maxcomplexity: 9 + var isValid = false; + var appliedToFallback = false; + + if (previous.isValid && !previous.appliedToFallback) { + if (status === jobStatus.FAILED || status === jobStatus.CANCELLED) { + this.query.setStatus(jobStatus.SKIPPED, job, errorMesssage); + + if (this.fallback) { + this.fallback.setStatus(jobStatus.SKIPPED, job); + } + } + } else if (!previous.isValid) { + isValid = this.query.setStatus(status, job, errorMesssage); + + if (this.fallback) { + if (!isValid) { + isValid = this.fallback.setStatus(status, job, errorMesssage); + appliedToFallback = true; + } else if (isValid && this.isFinalStatus(status) && !this.fallback.hasNextQuery(job)) { + this.fallback.setStatus(jobStatus.SKIPPED, job); + } + } + } + + return { isValid: isValid, appliedToFallback: appliedToFallback }; +}; + +QueryFallback.prototype.getStatus = function (job) { + return this.query.getStatus(job); +}; + +module.exports = QueryFallback; diff --git a/batch/pubsub/channel.js b/batch/pubsub/channel.js new file mode 100644 index 000000000..f719c84fc --- /dev/null +++ b/batch/pubsub/channel.js @@ -0,0 +1,4 @@ +module.exports = { + DB: 0, + NAME: 'batch:users' +}; diff --git a/batch/pubsub/job-publisher.js b/batch/pubsub/job-publisher.js new file mode 100644 index 000000000..4230a71de --- /dev/null +++ b/batch/pubsub/job-publisher.js @@ -0,0 +1,31 @@ +'use strict'; + +var Channel = require('./channel'); +var debug = require('./../util/debug')('pubsub:publisher'); +var error = require('./../util/debug')('pubsub:publisher:error'); + +function JobPublisher(pool) { + this.pool = pool; +} + +JobPublisher.prototype.publish = function (user) { + var self = this; + + this.pool.acquire(Channel.DB, function (err, client) { + if (err) { + return error('Error adquiring redis client: ' + err.message); + } + + client.publish(Channel.NAME, user, function (err) { + self.pool.release(Channel.DB, client); + + if (err) { + return error('Error publishing to ' + Channel.NAME + ':' + user + ', ' + err.message); + } + + debug('publish to ' + Channel.NAME + ':' + user); + }); + }); +}; + +module.exports = JobPublisher; diff --git a/batch/pubsub/job-subscriber.js b/batch/pubsub/job-subscriber.js new file mode 100644 index 000000000..f0a3c6414 --- /dev/null +++ b/batch/pubsub/job-subscriber.js @@ -0,0 +1,89 @@ +'use strict'; + +var Channel = require('./channel'); +var QueueSeeker = require('./queue-seeker'); +var debug = require('./../util/debug')('pubsub:subscriber'); +var error = require('./../util/debug')('pubsub:subscriber:error'); + +var MINUTE = 60 * 1000; +var SUBSCRIBE_INTERVAL = 5 * MINUTE; + +function JobSubscriber(pool, userDatabaseMetadataService) { + this.pool = pool; + this.userDatabaseMetadataService = userDatabaseMetadataService; + this.queueSeeker = new QueueSeeker(pool); +} + +module.exports = JobSubscriber; + +function seeker(queueSeeker, onJobHandler, callback) { + queueSeeker.seek(function (err, users) { + if (err) { + if (callback) { + callback(err); + } + return error(err); + } + debug('queues found successfully'); + users.forEach(onJobHandler); + + if (callback) { + return callback(null); + } + }); +} + +JobSubscriber.prototype.subscribe = function (onJobHandler, callback) { + var self = this; + + function wrappedJobHandlerListener(user) { + self.userDatabaseMetadataService.getUserMetadata(user, function (err, userDatabaseMetadata) { + if (err) { + return callback(err); + } + return onJobHandler(user, userDatabaseMetadata.host); + }); + } + + seeker(this.queueSeeker, wrappedJobHandlerListener, function(err) { + if (callback) { + callback(err); + } + + // do not start any pooling until first seek has finished + self.seekerInterval = setInterval(seeker, SUBSCRIBE_INTERVAL, self.queueSeeker, wrappedJobHandlerListener); + + self.pool.acquire(Channel.DB, function (err, client) { + if (err) { + return error('Error adquiring redis client: ' + err.message); + } + + self.client = client; + client.removeAllListeners('message'); + client.unsubscribe(Channel.NAME); + client.subscribe(Channel.NAME); + + client.on('message', function (channel, user) { + debug('message received in channel=%s from user=%s', channel, user); + wrappedJobHandlerListener(user); + }); + + client.on('error', function () { + self.unsubscribe(); + self.pool.release(Channel.DB, client); + self.subscribe(onJobHandler); + }); + }); + }); +}; + +JobSubscriber.prototype.unsubscribe = function (callback) { + clearInterval(this.seekerInterval); + if (this.client && this.client.connected) { + this.client.unsubscribe(Channel.NAME, callback); + } else { + if (callback) { + return callback(null); + } + } +}; diff --git a/batch/pubsub/queue-seeker.js b/batch/pubsub/queue-seeker.js new file mode 100644 index 000000000..0e1b7a4b9 --- /dev/null +++ b/batch/pubsub/queue-seeker.js @@ -0,0 +1,51 @@ +'use strict'; + +var QUEUE = require('../job_queue').QUEUE; + +function QueueSeeker(pool) { + this.pool = pool; +} + +module.exports = QueueSeeker; + +QueueSeeker.prototype.seek = function (callback) { + var initialCursor = ['0']; + var users = {}; + var self = this; + + this.pool.acquire(QUEUE.DB, function(err, client) { + if (err) { + return callback(err); + } + self._seek(client, initialCursor, users, function(err, users) { + self.pool.release(QUEUE.DB, client); + return callback(err, Object.keys(users)); + }); + }); +}; + +QueueSeeker.prototype._seek = function (client, cursor, users, callback) { + var self = this; + var redisParams = [cursor[0], 'MATCH', QUEUE.PREFIX + '*']; + + client.scan(redisParams, function(err, currentCursor) { + if (err) { + return callback(null, users); + } + + var queues = currentCursor[1]; + if (queues) { + queues.forEach(function (queue) { + var user = queue.substr(QUEUE.PREFIX.length); + users[user] = true; + }); + } + + var hasMore = currentCursor[0] !== '0'; + if (!hasMore) { + return callback(null, users); + } + + self._seek(client, currentCursor, users, callback); + }); +}; diff --git a/batch/query_runner.js b/batch/query_runner.js index 1d5701ff3..e91469290 100644 --- a/batch/query_runner.js +++ b/batch/query_runner.js @@ -1,38 +1,46 @@ 'use strict'; var PSQL = require('cartodb-psql'); +var debug = require('./util/debug')('query-runner'); -function QueryRunner() { +function QueryRunner(userDatabaseMetadataService) { + this.userDatabaseMetadataService = userDatabaseMetadataService; } module.exports = QueryRunner; -QueryRunner.prototype.run = function (job_id, sql, userDatabaseMetadata, callback) { - var pg = new PSQL(userDatabaseMetadata, {}, { destroyOnError: true }); - - pg.query('SET statement_timeout=0', function (err) { - if(err) { +QueryRunner.prototype.run = function (job_id, sql, user, timeout, callback) { + this.userDatabaseMetadataService.getUserMetadata(user, function (err, userDatabaseMetadata) { + if (err) { return callback(err); } - // mark query to allow to users cancel their queries - sql = '/* ' + job_id + ' */ ' + sql; + var pg = new PSQL(userDatabaseMetadata, {}, { destroyOnError: true }); - pg.eventedQuery(sql, function (err, query) { - if (err) { + pg.query('SET statement_timeout=' + timeout, function (err) { + if(err) { return callback(err); } - query.on('error', callback); + // mark query to allow to users cancel their queries + sql = '/* ' + job_id + ' */ ' + sql; - query.on('end', function (result) { - // only if result is present then query is done sucessfully otherwise an error has happened - // and it was handled by error listener - if (result) { - callback(null, result); + debug('Running query [timeout=%d] %s', timeout, sql); + pg.eventedQuery(sql, function (err, query) { + if (err) { + return callback(err); } + + query.on('error', callback); + + query.on('end', function (result) { + // only if result is present then query is done sucessfully otherwise an error has happened + // and it was handled by error listener + if (result) { + callback(null, result); + } + }); }); }); }); - }; diff --git a/batch/queue_seeker.js b/batch/queue_seeker.js deleted file mode 100644 index 81f6fae47..000000000 --- a/batch/queue_seeker.js +++ /dev/null @@ -1,47 +0,0 @@ -'use strict'; - -function QueueSeeker(metadataBackend) { - this.db = 5; - this.channel = 'batch:hosts'; - this.redisPrefix = 'batch:queues:'; - this.pattern = this.redisPrefix + '*'; - this.metadataBackend = metadataBackend; -} - -module.exports = QueueSeeker; - -QueueSeeker.prototype.seek = function (onMessage, callback) { - var initialCursor = ['0']; - this.onMessage = onMessage; - - this._seek(initialCursor, callback); -}; - -QueueSeeker.prototype._seek = function (cursor, callback) { - var self = this; - var redisParams = [cursor[0], 'MATCH', self.pattern]; - - self.metadataBackend.redisCmd(self.db, 'SCAN', redisParams, function (err, currentCursor) { - if (err) { - return callback(err); - } - - // checks if iteration has ended - if (currentCursor[0] === '0') { - return callback(null); - } - - var queues = currentCursor[1]; - - if (!queues) { - return callback(null); - } - - queues.forEach(function (queue) { - var host = queue.substr(self.redisPrefix.length); - self.onMessage(self.channel, host); - }); - - self._seek(currentCursor, callback); - }); -}; diff --git a/batch/scheduler/capacity/fixed.js b/batch/scheduler/capacity/fixed.js new file mode 100644 index 000000000..aef83be81 --- /dev/null +++ b/batch/scheduler/capacity/fixed.js @@ -0,0 +1,11 @@ +'use strict'; + +function FixedCapacity(capacity) { + this.capacity = Math.max(1, capacity); +} + +module.exports = FixedCapacity; + +FixedCapacity.prototype.getCapacity = function(callback) { + return callback(null, this.capacity); +}; diff --git a/batch/scheduler/capacity/http-load.js b/batch/scheduler/capacity/http-load.js new file mode 100644 index 000000000..4fd6c1ae0 --- /dev/null +++ b/batch/scheduler/capacity/http-load.js @@ -0,0 +1,32 @@ +'use strict'; + +var util = require('util'); +var debug = require('../../util/debug')('capacity-http-load'); +var HttpSimpleCapacity = require('./http-simple'); + +function HttpLoadCapacity(host, capacityEndpoint) { + HttpSimpleCapacity.call(this, host, capacityEndpoint); +} +util.inherits(HttpLoadCapacity, HttpSimpleCapacity); + +module.exports = HttpLoadCapacity; + +HttpLoadCapacity.prototype.getCapacity = function(callback) { + this.getResponse(function(err, values) { + var capacity = 1; + + if (err) { + return callback(null, capacity); + } + + var cores = parseInt(values.cores, 10); + var relativeLoad = parseFloat(values.relative_load); + + capacity = Math.max(1, Math.floor(((1 - relativeLoad) * cores) - 1)); + + capacity = Number.isFinite(capacity) ? capacity : 1; + + debug('host=%s, capacity=%s', this.host, capacity); + return callback(null, capacity); + }.bind(this)); +}; diff --git a/batch/scheduler/capacity/http-simple.js b/batch/scheduler/capacity/http-simple.js new file mode 100644 index 000000000..fe73a7a3f --- /dev/null +++ b/batch/scheduler/capacity/http-simple.js @@ -0,0 +1,62 @@ +'use strict'; + +var request = require('request'); +var debug = require('../../util/debug')('capacity-http-simple'); + +function HttpSimpleCapacity(host, capacityEndpoint) { + this.host = host; + this.capacityEndpoint = capacityEndpoint; + + this.lastResponse = null; + this.lastResponseTime = 0; +} + +module.exports = HttpSimpleCapacity; + +HttpSimpleCapacity.prototype.getCapacity = function(callback) { + this.getResponse(function(err, values) { + var capacity = 1; + + if (err) { + return callback(null, capacity); + } + + var availableCores = parseInt(values.available_cores, 10); + + capacity = Math.max(availableCores, 1); + capacity = Number.isFinite(capacity) ? capacity : 1; + + debug('host=%s, capacity=%s', this.host, capacity); + return callback(null, capacity); + }.bind(this)); +}; + +HttpSimpleCapacity.prototype.getResponse = function(callback) { + var requestParams = { + method: 'POST', + url: this.capacityEndpoint, + timeout: 2000, + json: true + }; + debug('getCapacity(%s)', this.host); + + // throttle requests for 500 ms + var now = Date.now(); + if (this.lastResponse !== null && ((now - this.lastResponseTime) < 500)) { + return callback(null, this.lastResponse); + } + + request.post(requestParams, function(err, res, jsonRes) { + if (err) { + return callback(err); + } + if (jsonRes && jsonRes.retcode === 0) { + this.lastResponse = jsonRes.return_values || {}; + // We could go more aggressive by updating lastResponseTime on failures. + this.lastResponseTime = now; + + return callback(null, this.lastResponse); + } + return callback(new Error('Could not retrieve information from endpoint')); + }.bind(this)); +}; diff --git a/batch/scheduler/host-scheduler.js b/batch/scheduler/host-scheduler.js new file mode 100644 index 000000000..99526e81d --- /dev/null +++ b/batch/scheduler/host-scheduler.js @@ -0,0 +1,85 @@ +'use strict'; + +var _ = require('underscore'); +var debug = require('../util/debug')('host-scheduler'); +var Scheduler = require('./scheduler'); +var Locker = require('../leader/locker'); +var FixedCapacity = require('./capacity/fixed'); +var HttpSimpleCapacity = require('./capacity/http-simple'); +var HttpLoadCapacity = require('./capacity/http-load'); + +function HostScheduler(name, taskRunner, redisPool) { + this.name = name || 'scheduler'; + this.taskRunner = taskRunner; + this.locker = Locker.create('redis-distlock', { pool: redisPool }); + this.locker.on('error', function(err, host) { + debug('[%s] Locker.error %s', this.name, err.message); + this.unlock(host); + }.bind(this)); + // host => Scheduler + this.schedulers = {}; +} + +module.exports = HostScheduler; + +HostScheduler.prototype.add = function(host, user, callback) { + this.lock(host, function(err, scheduler) { + if (err) { + debug('[%s] Could not lock host=%s', this.name, host); + return callback(err); + } + scheduler.add(user); + var wasRunning = scheduler.schedule(); + debug('[%s] Scheduler host=%s was running=%s', this.name, host, wasRunning); + return callback(err, wasRunning); + }.bind(this)); +}; + +HostScheduler.prototype.getCapacityProvider = function(host) { + var strategy = global.settings.batch_capacity_strategy; + + if (strategy === 'http-simple' || strategy === 'http-load') { + if (global.settings.batch_capacity_http_url_template) { + var endpoint = _.template(global.settings.batch_capacity_http_url_template, { dbhost: host }); + debug('Using strategy=%s capacity. Endpoint=%s', strategy, endpoint); + + if (strategy === 'http-simple') { + return new HttpSimpleCapacity(host, endpoint); + } + return new HttpLoadCapacity(host, endpoint); + } + } + + var fixedCapacity = global.settings.batch_capacity_fixed_amount || 4; + debug('Using strategy=fixed capacity=%d', fixedCapacity); + return new FixedCapacity(fixedCapacity); +}; + +HostScheduler.prototype.lock = function(host, callback) { + debug('[%s] lock(%s)', this.name, host); + var self = this; + this.locker.lock(host, function(err) { + if (err) { + debug('[%s] Could not lock host=%s. Reason: %s', self.name, host, err.message); + return callback(err); + } + + if (!self.schedulers.hasOwnProperty(host)) { + var scheduler = new Scheduler(self.getCapacityProvider(host), self.taskRunner); + scheduler.on('done', self.unlock.bind(self, host)); + self.schedulers[host] = scheduler; + } + + debug('[%s] Locked host=%s', self.name, host); + return callback(null, self.schedulers[host]); + }); +}; + +HostScheduler.prototype.unlock = function(host) { + debug('[%s] unlock(%s)', this.name, host); + if (this.schedulers.hasOwnProperty(host)) { + // TODO stop scheduler? + delete this.schedulers[host]; + } + this.locker.unlock(host, debug); +}; diff --git a/batch/scheduler/scheduler.js b/batch/scheduler/scheduler.js new file mode 100644 index 000000000..a8e18c9a0 --- /dev/null +++ b/batch/scheduler/scheduler.js @@ -0,0 +1,201 @@ +'use strict'; + +// Inspiration from: +// - https://www.kernel.org/doc/Documentation/scheduler/sched-design-CFS.txt +// - https://www.kernel.org/doc/Documentation/rbtree.txt +// - http://www.ibm.com/developerworks/linux/library/l-completely-fair-scheduler/ + +var util = require('util'); +var EventEmitter = require('events').EventEmitter; +var RBTree = require('bintrees').RBTree; + +var debug = require('../util/debug')('scheduler'); + +var forever = require('../util/forever'); + +function Scheduler(capacity, taskRunner) { + EventEmitter.call(this); + debug('new Scheduler'); + this.taskRunner = taskRunner; + this.capacity = capacity; + this.tasks = []; + this.users = {}; + this.tasksTree = new RBTree(function(taskEntityA, taskEntityB) { + // if the user is the same it's the same entity + if (taskEntityA.user === taskEntityB.user) { + return 0; + } + + // priority for entity with less executed jobs + if (taskEntityA.jobs !== taskEntityB.jobs) { + return taskEntityA.jobs - taskEntityB.jobs; + } + + // priority for oldest job + if (taskEntityA.createdAt !== taskEntityB.createdAt) { + return taskEntityA.createdAt - taskEntityB.createdAt; + } + + // we don't care if we arrive here + return -1; + }); +} +util.inherits(Scheduler, EventEmitter); + +module.exports = Scheduler; + +Scheduler.prototype.add = function(user) { + debug('add(%s)', user); + var taskEntity = this.users[user]; + if (taskEntity) { + if (taskEntity.status === STATUS.DONE) { + taskEntity.status = STATUS.PENDING; + this.tasksTree.insert(taskEntity); + this.emit('add'); + } + + return true; + } else { + taskEntity = new TaskEntity(user, this.tasks.length); + this.tasks.push(taskEntity); + this.users[user] = taskEntity; + this.tasksTree.insert(taskEntity); + + this.emit('add'); + + return false; + } +}; + +Scheduler.prototype.schedule = function() { + if (this.running) { + return true; + } + this.running = true; + + var self = this; + forever( + function (next) { + debug('Waiting for task'); + self.acquire(function(err, taskEntity) { + debug('Acquired user=%j', taskEntity); + + if (!taskEntity) { + return next(new Error('all users finished')); + } + + self.tasksTree.remove(taskEntity); + taskEntity.running(); + + debug('Running task for user=%s', taskEntity.user); + self.taskRunner.run(taskEntity.user, function(err, userQueueIsEmpty) { + debug('Run task=%j, done=%s', taskEntity, userQueueIsEmpty); + taskEntity.ran(userQueueIsEmpty); + self.release(err, taskEntity); + }); + + // try to acquire next user + // will block until capacity slot is available + next(); + }); + }, + function (err) { + debug('done: %s', err.message); + self.running = false; + self.emit('done'); + self.removeAllListeners(); + } + ); + + return false; +}; + +Scheduler.prototype.acquire = function(callback) { + this.removeAllListeners('add'); + this.removeAllListeners('release'); + + if (this.tasks.every(is(STATUS.DONE))) { + return callback(null, null); + } + + var self = this; + this.capacity.getCapacity(function(err, capacity) { + if (err) { + return callback(err); + } + + debug('Trying to acquire task'); + var running = self.tasks.filter(is(STATUS.RUNNING)); + debug('[capacity=%d, running=%d] candidates=%j', capacity, running.length, self.tasks); + + self.once('add', function() { + debug('Got a new task'); + self.acquire(callback); + }); + self.once('release', function() { + debug('Slot was released'); + self.acquire(callback); + }); + + if (running.length >= capacity) { + debug('Not enough capacity'); + return null; + } + + var isRunningAny = self.tasks.some(is(STATUS.RUNNING)); + var candidate = self.tasksTree.min(); + if (isRunningAny && candidate === null) { + debug('Waiting for last task to finish'); + return null; + } + + if (candidate) { + self.emit('acquired', candidate.user); + } + + return callback(null, candidate); + }); +}; + +Scheduler.prototype.release = function(err, taskEntity) { + debug('Released %j', taskEntity); + if (taskEntity.is(STATUS.PENDING)) { + this.tasksTree.insert(taskEntity); + } + this.emit('release'); +}; + + +/* Task entities */ + +var STATUS = { + PENDING: 'pending', + RUNNING: 'running', + DONE: 'done' +}; + +function TaskEntity(user, createdAt) { + this.user = user; + this.createdAt = createdAt; + this.status = STATUS.PENDING; + this.jobs = 0; +} + +TaskEntity.prototype.is = function(status) { + return this.status === status; +}; + +TaskEntity.prototype.running = function() { + this.status = STATUS.RUNNING; +}; + +TaskEntity.prototype.ran = function(userQueueIsEmpty) { + this.jobs++; + this.status = userQueueIsEmpty ? STATUS.DONE : STATUS.PENDING; +}; + +function is(status) { + return function(taskEntity) { + return taskEntity.is(status); + }; +} diff --git a/batch/user_indexer.js b/batch/user_indexer.js deleted file mode 100644 index 7a7de7f83..000000000 --- a/batch/user_indexer.js +++ /dev/null @@ -1,21 +0,0 @@ -'use strict'; - -function UserIndexer(metadataBackend) { - this.metadataBackend = metadataBackend; - this.db = 5; - this.redisPrefix = 'batch:users:'; -} - -UserIndexer.prototype.add = function (username, job_id, callback) { - this.metadataBackend.redisCmd(this.db, 'RPUSH', [ this.redisPrefix + username, job_id ] , callback); -}; - -UserIndexer.prototype.list = function (username, callback) { - this.metadataBackend.redisCmd(this.db, 'LRANGE', [ this.redisPrefix + username, -100, -1 ] , callback); -}; - -UserIndexer.prototype.remove = function (username, job_id, callback) { - this.metadataBackend.redisCmd(this.db, 'LREM', [ this.redisPrefix + username, 0, job_id] , callback); -}; - -module.exports = UserIndexer; diff --git a/config/environments/development.js.example b/config/environments/development.js.example index 3f46b9ec1..0e2bce872 100644 --- a/config/environments/development.js.example +++ b/config/environments/development.js.example @@ -29,7 +29,28 @@ module.exports.db_pubuser_pass = 'public'; module.exports.db_host = 'localhost'; module.exports.db_port = '5432'; module.exports.db_batch_port = '5432'; -module.exports.jobs_ttl_in_seconds = 48 * 3600; // 48 hours +module.exports.finished_jobs_ttl_in_seconds = 2 * 3600; // 2 hours +module.exports.batch_query_timeout = 12 * 3600 * 1000; // 12 hours in milliseconds +module.exports.batch_log_filename = 'logs/batch-queries.log'; +// Max number of queued jobs a user can have at a given time +module.exports.batch_max_queued_jobs = 64; +// Capacity strategy to use. +// It allows to tune how many queries run at a db host at the same time. +// Options: 'fixed', 'http-simple', 'http-load' +module.exports.batch_capacity_strategy = 'fixed'; +// Applies when strategy='fixed'. +// Number of simultaneous users running queries in the same host. +// It will use 1 as min. +// Default 4. +module.exports.batch_capacity_fixed_amount = 4; +// Applies when strategy='http-simple' or strategy='http-load'. +// HTTP endpoint to check db host load. +// Helps to decide the number of simultaneous users running queries in that host. +// 'http-simple' will use 'available_cores' to decide the number. +// 'http-load' will use 'cores' and 'relative_load' to decide the number. +// It will use 1 as min. +// If no template is provided it will default to 'fixed' strategy. +module.exports.batch_capacity_http_url_template = 'http://<%= dbhost %>:9999/load'; // Max database connections in the pool // Subsequent connections will wait for a free slot. // NOTE: not used by OGR-mediated accesses diff --git a/config/environments/production.js.example b/config/environments/production.js.example index 09e1abe34..2411a6bb8 100644 --- a/config/environments/production.js.example +++ b/config/environments/production.js.example @@ -30,7 +30,28 @@ module.exports.db_pubuser_pass = 'public'; module.exports.db_host = 'localhost'; module.exports.db_port = '6432'; module.exports.db_batch_port = '5432'; -module.exports.jobs_ttl_in_seconds = 48 * 3600; // 48 hours +module.exports.finished_jobs_ttl_in_seconds = 2 * 3600; // 2 hours +module.exports.batch_query_timeout = 12 * 3600 * 1000; // 12 hours in milliseconds +module.exports.batch_log_filename = 'logs/batch-queries.log'; +// Max number of queued jobs a user can have at a given time +module.exports.batch_max_queued_jobs = 64; +// Capacity strategy to use. +// It allows to tune how many queries run at a db host at the same time. +// Options: 'fixed', 'http-simple', 'http-load' +module.exports.batch_capacity_strategy = 'fixed'; +// Applies when strategy='fixed'. +// Number of simultaneous users running queries in the same host. +// It will use 1 as min. +// Default 4. +module.exports.batch_capacity_fixed_amount = 4; +// Applies when strategy='http-simple' or strategy='http-load'. +// HTTP endpoint to check db host load. +// Helps to decide the number of simultaneous users running queries in that host. +// 'http-simple' will use 'available_cores' to decide the number. +// 'http-load' will use 'cores' and 'relative_load' to decide the number. +// It will use 1 as min. +// If no template is provided it will default to 'fixed' strategy. +module.exports.batch_capacity_http_url_template = 'http://<%= dbhost %>:9999/load'; // Max database connections in the pool // Subsequent connections will wait for a free slot.i // NOTE: not used by OGR-mediated accesses @@ -64,14 +85,6 @@ module.exports.tableCacheMaxAge = 1000*60*10; module.exports.tmpDir = '/tmp'; // change ogr2ogr command or path module.exports.ogr2ogrCommand = 'ogr2ogr'; -// Optional rollbar support -module.exports.rollbar = { - token: 'secret', - // See http://github.com/rollbar/node_rollbar#configuration-reference - options: { - handler: 'inline' - } -} // Optional statsd support module.exports.statsd = { host: 'localhost', @@ -85,4 +98,7 @@ module.exports.health = { username: 'development', query: 'select 1' }; +module.exports.oauth = { + allowedHosts: ['carto.com', 'cartodb.com'] +}; module.exports.disabled_file = 'pids/disabled'; diff --git a/config/environments/staging.js.example b/config/environments/staging.js.example index 51eecc7df..4fa2f10d8 100644 --- a/config/environments/staging.js.example +++ b/config/environments/staging.js.example @@ -30,7 +30,28 @@ module.exports.db_pubuser_pass = 'public'; module.exports.db_host = 'localhost'; module.exports.db_port = '6432'; module.exports.db_batch_port = '5432'; -module.exports.jobs_ttl_in_seconds = 48 * 3600; // 48 hours +module.exports.finished_jobs_ttl_in_seconds = 2 * 3600; // 2 hours +module.exports.batch_query_timeout = 12 * 3600 * 1000; // 12 hours in milliseconds +module.exports.batch_log_filename = 'logs/batch-queries.log'; +// Max number of queued jobs a user can have at a given time +module.exports.batch_max_queued_jobs = 64; +// Capacity strategy to use. +// It allows to tune how many queries run at a db host at the same time. +// Options: 'fixed', 'http-simple', 'http-load' +module.exports.batch_capacity_strategy = 'fixed'; +// Applies when strategy='fixed'. +// Number of simultaneous users running queries in the same host. +// It will use 1 as min. +// Default 4. +module.exports.batch_capacity_fixed_amount = 4; +// Applies when strategy='http-simple' or strategy='http-load'. +// HTTP endpoint to check db host load. +// Helps to decide the number of simultaneous users running queries in that host. +// 'http-simple' will use 'available_cores' to decide the number. +// 'http-load' will use 'cores' and 'relative_load' to decide the number. +// It will use 1 as min. +// If no template is provided it will default to 'fixed' strategy. +module.exports.batch_capacity_http_url_template = 'http://<%= dbhost %>:9999/load'; // Max database connections in the pool // Subsequent connections will wait for a free slot. // NOTE: not used by OGR-mediated accesses @@ -64,14 +85,6 @@ module.exports.tableCacheMaxAge = 1000*60*10; module.exports.tmpDir = '/tmp'; // change ogr2ogr command or path module.exports.ogr2ogrCommand = 'ogr2ogr'; -// Optional rollbar support -module.exports.rollbar = { - token: 'secret', - // See http://github.com/rollbar/node_rollbar#configuration-reference - options: { - handler: 'inline' - } -} // Optional statsd support module.exports.statsd = { host: 'localhost', diff --git a/config/environments/test.js.example b/config/environments/test.js.example index 314983a53..203e5e39a 100644 --- a/config/environments/test.js.example +++ b/config/environments/test.js.example @@ -27,7 +27,28 @@ module.exports.db_pubuser_pass = 'public'; module.exports.db_host = 'localhost'; module.exports.db_port = '5432'; module.exports.db_batch_port = '5432'; -module.exports.jobs_ttl_in_seconds = 48 * 3600; // 48 hours +module.exports.finished_jobs_ttl_in_seconds = 2 * 3600; // 2 hours +module.exports.batch_query_timeout = 5 * 1000; // 5 seconds in milliseconds +module.exports.batch_log_filename = 'logs/batch-queries.log'; +// Max number of queued jobs a user can have at a given time +module.exports.batch_max_queued_jobs = 64; +// Capacity strategy to use. +// It allows to tune how many queries run at a db host at the same time. +// Options: 'fixed', 'http-simple', 'http-load' +module.exports.batch_capacity_strategy = 'fixed'; +// Applies when strategy='fixed'. +// Number of simultaneous users running queries in the same host. +// It will use 1 as min. +// Default 4. +module.exports.batch_capacity_fixed_amount = 4; +// Applies when strategy='http-simple' or strategy='http-load'. +// HTTP endpoint to check db host load. +// Helps to decide the number of simultaneous users running queries in that host. +// 'http-simple' will use 'available_cores' to decide the number. +// 'http-load' will use 'cores' and 'relative_load' to decide the number. +// It will use 1 as min. +// If no template is provided it will default to 'fixed' strategy. +module.exports.batch_capacity_http_url_template = 'http://<%= dbhost %>:9999/load'; // Max database connections in the pool // Subsequent connections will wait for a free slot. // NOTE: not used by OGR-mediated accesses @@ -74,4 +95,7 @@ module.exports.health = { username: 'vizzuality', query: 'select 1' }; +module.exports.oauth = { + allowedHosts: ['localhost.lan:8080', 'localhostdb.lan:8080'] +}; module.exports.disabled_file = 'pids/disabled'; diff --git a/config/settings.js b/config/settings.js deleted file mode 100644 index 4738663df..000000000 --- a/config/settings.js +++ /dev/null @@ -1,4 +0,0 @@ -var path = require('path'); - -module.exports.app_root = path.join(__dirname, '..'); - diff --git a/configure b/configure index cbf3e9ee4..3ae5ca060 100755 --- a/configure +++ b/configure @@ -20,9 +20,8 @@ while test -n "$1"; do PGPORT=`echo "$1" | cut -d= -f2` ;; *) - echo "Unknown option '$1'" >&2 - usage >&2 - exit 1 + echo "Unused option '$1'" >&2 + ;; esac shift done diff --git a/doc/API.md b/doc/API.md index 7771c83a9..3e1fe2e9a 100644 --- a/doc/API.md +++ b/doc/API.md @@ -1,6 +1,6 @@ # SQL API -CartoDB's SQL API allows you to interact with your tables and data inside CartoDB, as if you were running SQL statements against a normal database. The database behind CartoDB is PostgreSQL so if you need help with specific SQL statements or you want to learn more about it, visit the [official documentation](http://www.postgresql.org/docs/9.1/static/sql.html). +CARTO's SQL API allows you to interact with your tables and data inside CARTO, as if you were running SQL statements against a normal database. The database behind CARTO is PostgreSQL so if you need help with specific SQL statements or you want to learn more about it, visit the [official documentation](http://www.postgresql.org/docs/9.1/static/sql.html). There are two main situations in which you would want to use the SQL API: @@ -14,9 +14,9 @@ Remember that in order to access, read or modify data in private tables, you wil * [Authentication](authentication.md) * [Making calls to the SQL API](making_calls.md) -* [Handling geospatial data](handling_geospatial_data.md) -* [Query optimizations](query_optimizations.md) -* [SQL Batch API](sql_batch_api.md) -* [API version number](version.md) -* [Libraries in different languages](libraries_support.md) +* [Batch Queries](batch_queries.md) +* [Handling Geospatial Data](handling_geospatial_data.md) +* [Query Optimizations](query_optimizations.md) +* [API Version Vumber](version.md) +* [Libraries in Different Languages](libraries_support.md) * [Other Tips and Questions](tips_and_tricks.md) diff --git a/doc/authentication.md b/doc/authentication.md index e20c049f6..818f0d5a4 100644 --- a/doc/authentication.md +++ b/doc/authentication.md @@ -1,17 +1,17 @@ # Authentication -For all access to private tables, and for write access to public tables, CartoDB enforces secure API access that requires you to authorize your queries. In order to authorize queries, you can use an API Key or a Consumer Key. +For all access to private tables, and for write access to public tables, CARTO enforces secure API access that requires you to authorize your queries. In order to authorize queries, you can use an API Key or a Consumer Key. ## API Key -The API Key offers the simplest way to access private data, or perform writes and updates to your public data. Remember that your API Key protects access to your data, so keep it confidential and only share it if you want others to have this access. If necessary, you can reset your API Key from your CartoDB dashboard. +The API Key offers the simplest way to access private data, or perform writes and updates to your public data. Remember that your API Key protects access to your data, so keep it confidential and only share it if you want others to have this access. If necessary, you can reset your API Key from your CARTO dashboard. -**Tip:** For details about how to access, or reset, your API Key, see [Your Account](http://docs.cartodb.com/cartodb-editor/your-account/#api-key) details. +**Tip:** For details about how to access, or reset, your API Key, see [Your Account](http://docs.carto.com/carto-editor/your-account/#api-key) details. -To use your API Key, pass it as a parameter in an URL call to the CartoDB API. For example, to perform an insert into your table, you would use the following URL structure. +To use your API Key, pass it as a parameter in an URL call to the CARTO API. For example, to perform an insert into your table, you would use the following URL structure. #### Example ```bash -https://{username}.cartodb.com/api/v2/sql?q={SQL statement}&api_key={api_key} +https://{username}.carto.com/api/v2/sql?q={SQL statement}&api_key={api_key} ``` diff --git a/doc/batch_queries.md b/doc/batch_queries.md new file mode 100644 index 000000000..d115edfa9 --- /dev/null +++ b/doc/batch_queries.md @@ -0,0 +1,491 @@ +# Batch Queries + +A Batch Query enables you to request queries with long-running CPU processing times. Typically, these kind of requests raise timeout errors when using the SQL API. In order to avoid timeouts, you can use Batch Queries to [create](#create-a-job), [read](#read-a-job) and [cancel](#cancel-a-job) queries. You can also run a [chained batch query](#chaining-batch-queries) to chain several SQL queries into one job. A Batch Query schedules the incoming jobs and allows you to request the job status for each query. + +_Batch Queries are not intended to be used for large query payloads that contain over 16384 characters (16kb). For instance, if you are inserting a large number of rows into your table, you still need to use the [Import API](https://carto.com/docs/carto-engine/import-api/) or [SQL API](https://carto.com/docs/carto-engine/sql-api/) for this type of data management. Batch Queries are specific to queries and CPU usage._ + +**Note:** In order to use Batch Queries, you **must** be [authenticated](https://carto.com/docs/carto-engine/sql-api/authentication/) using API keys. + +## Authentication + +An API Key is required to manage your jobs. The following error message appears if you are not [authenticated](https://carto.com/docs/carto-engine/sql-api/authentication/): + +```bash +{ + "error": [ + "permission denied" + ] +} +``` + +In order to get full access, you must use your API Key. + +Using cURL tool: + +```bash +curl -X POST -H "Content-Type: application/json" -d '{ + "query": "{query}" +}' "http://{username}.carto.com/api/v2/sql/job?api_key={api_key}" +``` + +Using Node.js request client: + +```bash +var request = require("request"); + +var options = { + method: "POST", + url: "http://{username}.carto.com/api/v2/sql/job", + qs: { + "api_key": "{api_key}" + }, + headers: { + "content-type": "application/json" + }, + body: { + query: "{query}" + }, + json: true +}; + +request(options, function (error, response, body) { + if (error) throw new Error(error); + + console.log(body); +}); +``` + +## Batch Queries Job Schema + +A Batch Query request to your CARTO account includes the following job schema elements. _Only the `query` element can be modified._ All other elements of the job schema are defined by the Batch Query and are read-only. + +Name | Description +--- | --- +`job_id` | a universally unique identifier (uuid). +`user` | user identifier, as displayed by the username. +`status` | displays the result of the long-running query. The possible status results are: +--- | --- +|_ `pending` | job waiting to be executed. +|_ `running` | indicates that the job is currently running. +|_ `done` | job executed successfully. +|_ `failed` | job executed but failed, with errors. +|_ `canceled` | job canceled by user request. +|_ `unknown` | appears when it is not possible to determine what exactly happened with the job. +`query` | the SQL statement to be executed in a database. _You can modify the select SQL statement to be used in the job schema._

**Tip:** In some scenarios, you may need to retrieve the query results from a finished job. See [Fetching Job Results](#fetching-job-results) for details. +`created_at` | the date and time when the job schema was created. +`updated_at` | the date and time of when the job schema was last updated, or modified. +`failed_reason` | displays the database error message, if something went wrong. + +#### Example + +```bash +HEADERS: 201 CREATED; application/json +BODY: { + "job_id": "de305d54-75b4-431b-adb2-eb6b9e546014", + "user": "cartofante", + "query": "UPDATE airports SET type = 'international'", + "status": "pending", + "created_at": "2015-12-15T07:36:25Z", + "updated_at": "2015-12-15T07:36:25Z" +} +``` + +### Create a Job + +To create a Batch Query job, make a POST request with the following parameters. + +Creates a Batch Query job request. + +```bash +HEADERS: POST /api/v2/sql/job +BODY: { + "query": "UPDATE airports SET type = 'international'" +} +``` + +#### Response + +```bash +HEADERS: 201 CREATED; application/json +BODY: { + "job_id": "de305d54-75b4-431b-adb2-eb6b9e546014", + "user": "cartofante" + "query": "UPDATE airports SET type = 'international'", + "status": "pending", + "created_at": "2015-12-15T07:36:25Z", + "updated_at": "2015-12-15T07:36:25Z" +} +``` + +##### POST Examples + +If you are using the Batch Query create operation for a cURL POST request, use the following code: + +```bash +curl -X POST -H "Content-Type: application/json" -d '{ + "query": "CREATE TABLE world_airports AS SELECT a.cartodb_id, a.the_geom, a.the_geom_webmercator, a.name airport, b.name country FROM world_borders b JOIN airports a ON ST_Contains(b.the_geom, a.the_geom)" +}' "http://{username}.carto.com/api/v2/sql/job" +``` + +If you are using the Batch Query create operation for a Node.js client POST request, use the following code: + +```bash +var request = require("request"); + +var options = { + method: "POST", + url: "http://{username}.carto.com/api/v2/sql/job", + headers: { "content-type": "application/json" }, + body: { + query: "CREATE TABLE world_airports AS SELECT a.cartodb_id, a.the_geom, a.the_geom_webmercator, a.name airport, b.name country FROM world_borders b JOIN airports a ON ST_Contains(b.the_geom, a.the_geom)" + }, + json: true +}; + +request(options, function (error, response, body) { + if (error) throw new Error(error); + + console.log(body); +}); +``` + +### Read a Job + +To read a Batch Query job, make a GET request with the following parameters. + +```bash +HEADERS: GET /api/v2/sql/job/de305d54-75b4-431b-adb2-eb6b9e546014 +BODY: {} +``` + +#### Response + +```bash +HEADERS: 200 OK; application/json +BODY: { + "job_id": "de305d54-75b4-431b-adb2-eb6b9e546014", + "user": "cartofante" + "query": "UPDATE airports SET type = 'international'", + "status": "pending", + "created_at": "2015-12-15T07:36:25Z", + "updated_at": "2015-12-15T07:36:25Z" +} +``` + +##### GET Examples + +If you are using the Batch Query read operation for a cURL GET request, use the following code: + +```bash +curl -X GET "http://{username}.carto.com/api/v2/sql/job/{job_id}" +``` + +If you are a Batch Query read operation for a Node.js client GET request, use the following code: + +```bash +var request = require("request"); + +var options = { + method: "GET", + url: "http://{username}.carto.com/api/v2/sql/job/{job_id}" +}; + +request(options, function (error, response, body) { + if (error) throw new Error(error); + + console.log(body); +}); +``` + +### Cancel a Job + +To cancel a Batch Query, make a DELETE request with the following parameters. + +```bash +HEADERS: DELETE /api/v2/sql/job/de305d54-75b4-431b-adb2-eb6b9e546014 +BODY: {} +``` + +**Note:** Be mindful when canceling a job when the status: `pending` or `running`. + +- If the job is `pending`, the job will never be executed +- If the job is `running`, the job will be terminated immediately + +#### Response + +```bash +HEADERS: 200 OK; application/json +BODY: { + "job_id": "de305d54-75b4-431b-adb2-eb6b9e546014", + "user": "cartofante" + "query": "UPDATE airports SET type = 'international'", + "status": "cancelled", + "created_at": "2015-12-15T07:36:25Z", + "updated_at": "2015-12-17T06:22:42Z" +} +``` + +**Note:** Jobs can only be canceled while the `status: "running"` or `status: "pending"`, otherwise the Batch Query operation is not allowed. You will receive an error if the job status is anything but "running" or "pending". + +```bash +errors: [ + "The job status is done, cancel is not allowed" +] +``` + +##### DELETE Examples + +If you are using the Batch Query cancel operation for cURL DELETE request, use the following code: + +```bash +curl -X DELETE "http://{username}.carto.com/api/v2/sql/job/{job_id}" +``` + +If you are using the Batch Query cancel operation for a Node.js client DELETE request, use the following code: + +```bash +var request = require("request"); + +var options = { + method: "DELETE", + url: "http://{username}.carto.com/api/v2/sql/job/{job_id}", +}; + +request(options, function (error, response, body) { + if (error) throw new Error(error); + + console.log(body); +}); +``` + +### Chaining Batch Queries + +In some cases, you may need to chain queries into one job. The Chaining Batch Query option enables you run an array of SQL statements, and define the order in which the queries are executed. You can use any of the operations (create, read, list, update, cancel) for the queries in a chained batch query. + +```bash +HEADERS: POST /api/v2/sql/job +BODY: { + query: [ + "CREATE TABLE world_airports AS SELECT a.cartodb_id, a.the_geom, a.the_geom_webmercator, a.name airport, b.name country FROM world_borders b JOIN airports a ON ST_Contains(b.the_geom, a.the_geom)", + "DROP TABLE airports", + "ALTER TABLE world_airports RENAME TO airport" + ] +} +``` + +#### Response + +```bash +HEADERS: 201 CREATED; application/json +BODY: { + "job_id": "de305d54-75b4-431b-adb2-eb6b9e546014", + "user": "cartofante" + "query": [{ + "query": "CREATE TABLE world_airports AS SELECT a.cartodb_id, a.the_geom, a.the_geom_webmercator, a.name airport, b.name country FROM world_borders b JOIN airports a ON ST_Contains(b.the_geom, a.the_geom)", + "status": "pending" + }, { + "query": "DROP TABLE airports", + "status": "pending" + }, { + "query": "ALTER TABLE world_airports RENAME TO airport", + "status": "pending" + }], + "status": "pending", + "created_at": "2015-12-15T07:36:25Z", + "updated_at": "2015-12-15T07:36:25Z" +} +``` + +**Note:** The Batch Query returns a job status for both the parent Chained Batch Query request, and for each child query within the request. The order in which each query is executed is guaranteed. Here are the possible status results for Chained Batch Queries: + +- If one query within the Chained Batch Query fails, the `"status": "failed"` is returned for both the job and the query, and any "pending" queries will not be processed + +- If you cancel the Chained Batch Query job, the job status changes to `"status": "cancelled"`. Any running queries within the job will be stopped and changed to `"status": "pending"`, and will not be processed + +- Suppose the first query job status is `"status": "done"`, the second query is `"status": "running"`, and the third query `"status": "pending"`. If the second query fails for some reason, the job status changes to `"status": "failed"` and the last query will not be processed. It is indicated which query failed in the Chained Batch Query job + +- Creating several jobs does not guarantee that jobs are going to be executed in the same order that they were created. If you need run queries in a specific order, you may want use [Chaining Batch Queries](#chaining-batch-queries). + +##### POST Examples + +If you are using the Chained Batch Query operation for cURL POST request, use the following code: + +```bash +curl -X POST -H "Content-Type: application/json" -d '{ + "query": [ + "CREATE TABLE world_airports AS SELECT a.cartodb_id, a.the_geom, a.the_geom_webmercator, a.name airport, b.name country FROM world_borders b JOIN airports a ON ST_Contains(b.the_geom, a.the_geom)", + "DROP TABLE airports", + "ALTER TABLE world_airports RENAME TO airport" + ] +}' "http://{username}.carto.com/api/v2/sql/job" +``` + +If you are using the Chained Batch Query operation for a Node.js client POST request, use the following code: + +```bash +var request = require("request"); + +var options = { + method: "POST", + url: "http://{username}.carto.com/api/v2/sql/job", + headers: { "content-type": "application/json" }, + body: { + "query": [ + "CREATE TABLE world_airports AS SELECT a.cartodb_id, a.the_geom, a.the_geom_webmercator, a.name airport, b.name country FROM world_borders b JOIN airports a ON ST_Contains(b.the_geom, a.the_geom)", + "DROP TABLE airports", + "ALTER TABLE world_airports RENAME TO airport" + ] + }, + json: true +}; + +request(options, function (error, response, body) { + if (error) throw new Error(error); + + console.log(body); +}); +``` + +##### PUT Examples + +If you are using the Chained Batch Query operation for cURL PUT request, use the following code: + +```bash +curl -X PUT -H "Content-Type: application/json" -d '{ + "query": [ + "CREATE TABLE world_airports AS SELECT a.cartodb_id, a.the_geom, a.the_geom_webmercator, a.name airport, b.name country FROM world_borders b JOIN airports a ON ST_Contains(b.the_geom, a.the_geom)", + "DROP TABLE airports", + "ALTER TABLE world_airports RENAME TO airport", + "UPDATE airports SET airport = upper(airport)" + ] +}' "http://{username}.carto.com/api/v2/sql/job/{job_id}" +``` + +If you are using the Chained Batch Query operation for a Node.js client PUT request, use the following code: + +```bash +var request = require("request"); + +var options = { + method: "PUT", + url: "http://{username}.carto.com/api/v2/sql/job/{job_id}", + headers: { "content-type": "application/json" }, + body: { + query: [ + "CREATE TABLE world_airports AS SELECT a.cartodb_id, a.the_geom, a.the_geom_webmercator, a.name airport, b.name country FROM world_borders b JOIN airports a ON ST_Contains(b.the_geom, a.the_geom)", + "DROP TABLE airports", + "ALTER TABLE world_airports RENAME TO airport", + "UPDATE airports SET airport = upper(airport)" + ] + }, + json: true +}; + +request(options, function (error, response, body) { + if (error) throw new Error(error); + + console.log(body); +}); +``` + +## Chaining Batch Queries with fallbacks + +When you need to run an extra query based on how a chaining query finished, Batch Queries enable you to define onerror and onsuccess fallbacks. This powerful feature opens a huge range of possibilities, for instance: + +- You can create jobs periodically in order to get updated data and create a new table where you can check the status of your tables. + +For this example, you can create the following job: + +```bash +curl -X POST -H "Content-Type: application/json" -d '{ + "query": { + "query": [{ + "query": "UPDATE nasdaq SET price = '$100.00' WHERE company = 'CARTO'", + "onsuccess": "UPDATE market_status SET status = 'updated', updated_at = NOW() WHERE table_name = 'nasdaq'" + "onerror": "UPDATE market_status SET status = 'outdated' WHERE table_name = 'nasdaq'" + }] + } +}' "http://{username}.carto.com/api/v2/sql/job" +``` + +If query finishes successfully, then onsuccess fallback will be fired. Otherwise, onerror will be fired. You can define fallbacks per query: + +```bash +curl -X POST -H "Content-Type: application/json" -d '{ + "query": { + "query": [{ + "query": "UPDATE nasdaq SET price = '$101.00' WHERE company = 'CARTO'", + "onsuccess": "UPDATE market_status SET status = 'updated', updated_at = NOW() WHERE table_name = 'nasdaq'", + "onerror": "UPDATE market_status SET status = 'outdated' WHERE table_name = 'nasdaq'" + }, { + "query": "UPDATE down_jones SET price = '$100.00' WHERE company = 'Esri'", + "onsuccess": "UPDATE market_status SET status = 'updated', updated_at = NOW() WHERE table_name = 'down_jones'", + "onerror": "UPDATE market_status SET status = 'outdated' WHERE table_name = 'down_jones'" + }] + } +}' "http://{username}.carto.com/api/v2/sql/job" +``` + +...at the job level.. + +```bash +curl -X POST -H "Content-Type: application/json" -d '{ + "query": { + "query": [{ + "query": "UPDATE nasdaq SET price = '$101.00' WHERE company = 'CARTO'" + }, { + "query": "UPDATE down_jones SET price = '$100.00' WHERE company = 'Esri'" + }], + "onsuccess": "UPDATE market_status SET status = 'updated', updated_at = NOW()", + "onerror": "UPDATE market_status SET status = 'outdated'" + } +}' "http://{username}.carto.com/api/v2/sql/job" +``` + +If a query of a job fails (and onerror fallbacks for that query and job are defined), then Batch Queries runs the first fallback for that query. The job fallback runs next and sets the job as failed. Remaining queries will not be executed. Furthermore, Batch Queries will run the onsuccess fallback at the job level, if (and only if), every query has finished successfully. + +### Templates + +Batch Queries provide a simple way to get the error message and the job identifier to be used in your fallbacks, by using the following templates: + + - `<%= error_message %>`: will be replaced by the error message raised by the database. + - `<%= job_id %>`: will be replaced by the job identifier that Batch Queries provides. + +This is helpful when you want to save error messages into a table: + +```bash +curl -X POST -H "Content-Type: application/json" -d '{ + "query": { + "query": [{ + "query": "UPDATE wrong_table SET price = '$100.00' WHERE company = 'CARTO'" + }], + "onerror": "INSERT INTO errors_log (job_id, error_message, date) VALUES ('<%= job_id %>', '<%= error_message %>', NOW())" + } +}' "http://{username}.carto.com/api/v2/sql/job" +``` + +More templates are coming soon. + +## Fetching Job Results + +In some scenarios, you may need to fetch the output of a job. If that is the case, wrap the query with `SELECT * INTO`, or `CREATE TABLE AS`. The output is stored in a new table in your database. For example, if using the query `SELECT * FROM airports`: + +1. Wrap the query `SELECT * INTO job_result FROM (SELECT * FROM airports) AS job` + +2. [Create a job](#create-a-job), as described previously + +3. Once the job is done, fetch the results through the [CARTO SQL API](https://carto.com/docs/carto-engine/sql-api/), `SELECT * FROM job_result` + +**Note:** If you need to create a map or analysis with the new table, use the [CDB_CartodbfyTable function](https://github.com/CartoDB/cartodb-postgresql/blob/master/doc/cartodbfy-requirements.rst). + +## Best Practices + +For best practices, follow these recommended usage notes when using Batch Queries: + +- Batch Queries are recommended for INSERT, UPDATE, and CREATE queries that manipulate and create new data, such as creating expensive indexes, applying updates over large tables, and creating tables from complex queries. Batch queries have no effect for SELECT queries that retrieve data but do not store the results in a table. For example, running a batch query using `SELECT * from my_dataset` will not produce any results. + +- Batch Queries are not intended for large query payloads (e.g: inserting thousands of rows), use the [Import API](https://carto.com/docs/carto-engine/import-api/) for this type of data management. + +- There is a limit of 16kb per job. The following error message appears if your job exceeds this size: + + `Your payload is too large. Max size allowed is 16384 (16kb)` diff --git a/doc/handling_geospatial_data.md b/doc/handling_geospatial_data.md index fbfeb9bcf..f0cebf057 100644 --- a/doc/handling_geospatial_data.md +++ b/doc/handling_geospatial_data.md @@ -9,7 +9,7 @@ The first is to use the format=GeoJSON method described above. Others can be han #### Call ```bash -https://{username}.cartodb.com/api/v2/sql?q=SELECT cartodb_id,ST_AsGeoJSON(the_geom) as the_geom FROM {table_name} LIMIT 1 +https://{username}.carto.com/api/v2/sql?q=SELECT cartodb_id,ST_AsGeoJSON(the_geom) as the_geom FROM {table_name} LIMIT 1 ``` #### Result @@ -32,7 +32,7 @@ https://{username}.cartodb.com/api/v2/sql?q=SELECT cartodb_id,ST_AsGeoJSON(the_g #### Call ```bash -https://{username}.cartodb.com/api/v2/sql?q=SELECT cartodb_id,ST_AsText(the_geom) FROM {table_name} LIMIT 1 +https://{username}.carto.com/api/v2/sql?q=SELECT cartodb_id,ST_AsText(the_geom) FROM {table_name} LIMIT 1 ``` #### Result @@ -57,7 +57,7 @@ All data returned from *the_geom* column is in WGS 84 (EPSG:4326). You can chang ### ST_Transform ```bash -https://{username}.cartodb.com/api/v2/sql?q=SELECT ST_Transform(the_geom,4147) FROM {table_name} LIMIT 1 +https://{username}.carto.com/api/v2/sql?q=SELECT ST_Transform(the_geom,4147) FROM {table_name} LIMIT 1 ``` -CartoDB also stores a second geometry column, *the_geom_webmercator*. We use this internally to build your map tiles as fast as we can. In the user-interface it is hidden, but it is visible and available for use. In this column, we store a reprojected version of all your geometries using Web Mercator (EPSG:3857). +CARTO also stores a second geometry column, *the_geom_webmercator*. We use this internally to build your map tiles as fast as we can. In the user-interface it is hidden, but it is visible and available for use. In this column, we store a reprojected version of all your geometries using Web Mercator (EPSG:3857). diff --git a/doc/libraries_support.md b/doc/libraries_support.md index d8fd081be..5709cecc0 100644 --- a/doc/libraries_support.md +++ b/doc/libraries_support.md @@ -3,25 +3,25 @@ To make things easier for developers, we provide client libraries for different programming languages and caching functionalities. - **R** - To help more researchers use CartoDB to drive their geospatial data, we have released the R client library. [Fork it on GitHub!](https://github.com/Vizzuality/cartodb-r) + To help more researchers use CARTO to drive their geospatial data, we have released the R client library. [Fork it on GitHub!](https://github.com/Vizzuality/cartodb-r) - **NODE.js** - This demo app authenticates with your CartoDB and shows how to perform read and write queries using the SQL API. [Fork it on GitHub!](https://github.com/Vizzuality/cartodb-nodejs) + This demo app authenticates with your CARTO and shows how to perform read and write queries using the SQL API. [Fork it on GitHub!](https://github.com/Vizzuality/cartodb-nodejs) - **PHP** - The PHP library provides a wrapper around the SQL API to get PHP objects straight from SQL calls to CartoDB. [Fork it on GitHub!](https://github.com/Vizzuality/cartodbclient-php) + The PHP library provides a wrapper around the SQL API to get PHP objects straight from SQL calls to CARTO. [Fork it on GitHub!](https://github.com/Vizzuality/cartodbclient-php) - **PYTHON** Provides API Key access to SQL API. [Fork it on GitHub!](https://github.com/vizzuality/cartodb-python) - **JAVA** - Very basic example of how to access CartoDB SQL API. [Fork it on GitHub!](https://github.com/cartodb/cartodb-java-client) + Very basic example of how to access CARTO SQL API. [Fork it on GitHub!](https://github.com/cartodb/cartodb-java-client) - **NET** - .NET library for authenticating with CartoDB using an API Key, based on work started by [The Data Republic](http://www.thedatarepublic.com/). [Fork it on GitHub!](https://github.com/thedatarepublic/CartoDBClientDotNET) + .NET library for authenticating with CARTO using an API Key, based on work started by [The Data Republic](http://www.thedatarepublic.com/). [Fork it on GitHub!](https://github.com/thedatarepublic/CartoDBClientDotNET) - **Clojure** - Clojure library for authenticating with CartoDB, maintained by [REDD Metrics](http://www.reddmetrics.com/). [Fork it on GitHub!](https://github.com/reddmetrics/cartodb-clj) + Clojure library for authenticating with CARTO, maintained by [REDD Metrics](http://www.reddmetrics.com/). [Fork it on GitHub!](https://github.com/reddmetrics/cartodb-clj) - **iOS** - Objective-C library for interacting with CartoDB in native iOS applications. [Fork it on GitHub!](https://github.com/jmnavarro/cartodb-objectivec-client) + Objective-C library for interacting with CARTO in native iOS applications. [Fork it on GitHub!](https://github.com/jmnavarro/cartodb-objectivec-client) diff --git a/doc/making_calls.md b/doc/making_calls.md index df0d6fe31..852657ed9 100644 --- a/doc/making_calls.md +++ b/doc/making_calls.md @@ -1,18 +1,18 @@ # Making Calls to the SQL API -CartoDB is based on the rock solid PostgreSQL database. All of your tables reside a single database, which means you can perform complex queries joining tables, or carrying out geospatial operations. The best place to learn about PostgreSQL's SQL language is the [official documentation](http://www.postgresql.org/docs/9.1/static/). +CARTO is based on the rock solid PostgreSQL database. All of your tables reside a single database, which means you can perform complex queries joining tables, or carrying out geospatial operations. The best place to learn about PostgreSQL's SQL language is the [official documentation](http://www.postgresql.org/docs/9.1/static/). -CartoDB is also based on PostGIS, so take a look at the [official PostGIS reference](http://postgis.refractions.net/docs/) to know what functionality we support in terms of geospatial operations. All of our tables include a column called *the_geom,* which is a geometry field that indexes geometries in the EPSG:4326 (WGS 1984) coordinate system. All tables also have an automatically generated and updated column called *the_geom_webmercator*. We use the column internally to quickly create tiles for maps. +CARTO is also based on PostGIS, so take a look at the [official PostGIS reference](http://postgis.refractions.net/docs/) to know what functionality we support in terms of geospatial operations. All of our tables include a column called *the_geom,* which is a geometry field that indexes geometries in the EPSG:4326 (WGS 1984) coordinate system. All tables also have an automatically generated and updated column called *the_geom_webmercator*. We use the column internally to quickly create tiles for maps. ## URL endpoints -All SQL API requests to your CartoDB account should follow this general pattern: +All SQL API requests to your CARTO account should follow this general pattern: #### SQL query example ```bash -https://{username}.cartodb.com/api/v2/sql?q={SQL statement} +https://{username}.carto.com/api/v2/sql?q={SQL statement} ``` If you encounter errors, double-check that you are using the correct account name, and that your SQL statement is valid. A simple example of this pattern is conducting a count of all the records in your table: @@ -20,7 +20,7 @@ If you encounter errors, double-check that you are using the correct account nam #### Count example ```bash -https://{username}.cartodb.com/api/v2/sql?q=SELECT count(*) FROM {table_name} +https://{username}.carto.com/api/v2/sql?q=SELECT count(*) FROM {table_name} ``` #### Result @@ -37,38 +37,38 @@ https://{username}.cartodb.com/api/v2/sql?q=SELECT count(*) FROM {table_name} } ``` -Finally, remember that in order to use the SQL API, either your table must be public, or you must be [authenticated](http://docs.cartodb.com/cartodb-platform/sql-api/authentication/#authentication) using API Keys. +Finally, remember that in order to use the SQL API, either your table must be public, or you must be [authenticated](http://docs.carto.com/carto-engine/sql-api/authentication/#authentication) using API Keys. ## POST and GET -The CartoDB SQL API is setup to handle both GET and POST requests. You can test the GET method directly in your browser. Below is an example of a jQuery SQL API request to CartoDB: +The CARTO SQL API is setup to handle both GET and POST requests. You can test the GET method directly in your browser. Below is an example of a jQuery SQL API request to CARTO: ### jQuery #### Call ```javascript -$.getJSON('https://{username}.cartodb.com/api/v2/sql/?q='+sql_statement, function(data) { +$.getJSON('https://{username}.carto.com/api/v2/sql/?q='+sql_statement, function(data) { $.each(data.rows, function(key, val) { // do something! }); }); ``` -By default, GET requests work from anywhere. In CartoDB, POST requests work from any website as well. We achieve this by hosting a cross-domain policy file at the root of all of our servers. This allows you the greatest level of flexibility when developing your application. +By default, GET requests work from anywhere. In CARTO, POST requests work from any website as well. We achieve this by hosting a cross-domain policy file at the root of all of our servers. This allows you the greatest level of flexibility when developing your application. ## Response formats -The standard response from the CartoDB SQL API is JSON. If you are building a web-application, the lightweight JSON format allows you to quickly integrate data from the SQL API. +The standard response from the CARTO SQL API is JSON. If you are building a web-application, the lightweight JSON format allows you to quickly integrate data from the SQL API. ### JSON #### Call ```bash -https://{username}.cartodb.com/api/v2/sql?q=SELECT * FROM {table_name} LIMIT 1 +https://{username}.carto.com/api/v2/sql?q=SELECT * FROM {table_name} LIMIT 1 ``` #### Result @@ -97,7 +97,7 @@ Alternatively, you can use the [GeoJSON specification](http://www.geojson.org/ge #### Call ```bash -https://{username}.cartodb.com/api/v2/sql?format=GeoJSON&q=SELECT * FROM {table_name} LIMIT 1 +https://{username}.carto.com/api/v2/sql?format=GeoJSON&q=SELECT * FROM {table_name} LIMIT 1 ``` #### Result @@ -136,7 +136,7 @@ To customize the output filename, add the `filename` parameter to your URL: #### Call ```bash -https://{username}.cartodb.com/api/v2/sql?filename={custom_filename}&q=SELECT * FROM {table_name} LIMIT 1 +https://{username}.carto.com/api/v2/sql?filename={custom_filename}&q=SELECT * FROM {table_name} LIMIT 1 ``` @@ -147,13 +147,13 @@ Currently, there is no public method to access your table schemas. The simplest #### Call ```bash -https://{username}.cartodb.com/api/v2/sql?q=SELECT * FROM {table_name} LIMIT 1 +https://{username}.carto.com/api/v2/sql?q=SELECT * FROM {table_name} LIMIT 1 ``` ## Response errors -To help you debug your SQL queries, the CartoDB SQL API returns the full error provided by PostgreSQL, as part of the JSON response. Error responses appear in the following format, +To help you debug your SQL queries, the CARTO SQL API returns the full error provided by PostgreSQL, as part of the JSON response. Error responses appear in the following format, #### Result @@ -165,18 +165,18 @@ To help you debug your SQL queries, the CartoDB SQL API returns the full error p } ``` -You can use these errors to help understand your SQL. If you encounter errors executing SQL, either through the CartoDB Editor, or through the SQL API, it is suggested to Google search the error for independent troubleshooting. +You can use these errors to help understand your SQL. If you encounter errors executing SQL, either through the CARTO Editor, or through the SQL API, it is suggested to Google search the error for independent troubleshooting. -## Write data to your CartoDB account +## Write data to your CARTO account -Performing inserts or updates on your data is simple using your [API Key](#authentication). All you need to do is supply a correct SQL [INSERT](http://www.postgresql.org/docs/9.1/static/sql-insert.html) or [UPDATE](http://www.postgresql.org/docs/9.1/static/sql-update.html) statement for your table along with the api_key parameter for your account. Be sure to keep these requests private, as anyone with your API Key will be able to modify your tables. A correct SQL insert statement means that all the columns you want to insert into already exist in your table, and all the values for those columns are the right type (quoted string, unquoted string for geoms and dates, or numbers). +Performing inserts or updates on your data is simple using your [API Key](https://carto.com/docs/carto-engine/sql-api/authentication/). All you need to do is supply a correct SQL [INSERT](http://www.postgresql.org/docs/9.1/static/sql-insert.html) or [UPDATE](http://www.postgresql.org/docs/9.1/static/sql-update.html) statement for your table along with the api_key parameter for your account. Be sure to keep these requests private, as anyone with your API Key will be able to modify your tables. A correct SQL insert statement means that all the columns you want to insert into already exist in your table, and all the values for those columns are the right type (quoted string, unquoted string for geoms and dates, or numbers). ### Insert #### Call ```bash -https://{username}.cartodb.com/api/v2/sql?q=INSERT INTO test_table (column_name, column_name_2, the_geom) VALUES ('this is a string', 11, ST_SetSRID(ST_Point(-110, 43),4326))&api_key={api_key} +https://{username}.carto.com/api/v2/sql?q=INSERT INTO test_table (column_name, column_name_2, the_geom) VALUES ('this is a string', 11, ST_SetSRID(ST_Point(-110, 43),4326))&api_key={api_key} ``` Updates are just as simple. Here is an example of updating a row based on the value of the cartodb_id column. @@ -186,5 +186,5 @@ Updates are just as simple. Here is an example of updating a row based on the va #### Call ```bash -https://{username}.cartodb.com/api/v2/sql?q=UPDATE test_table SET column_name = 'my new string value' WHERE cartodb_id = 1 &api_key={api_key} +https://{username}.carto.com/api/v2/sql?q=UPDATE test_table SET column_name = 'my new string value' WHERE cartodb_id = 1 &api_key={api_key} ``` diff --git a/doc/query_optimizations.md b/doc/query_optimizations.md index bb644c601..7517d2e7c 100644 --- a/doc/query_optimizations.md +++ b/doc/query_optimizations.md @@ -2,9 +2,25 @@ There are some tricks to consider when using the SQL API that might make your application a little faster. -* Only request the fields you need. Selecting all columns will return a full version of your geometry in *the_geom*, as well as a reprojected version in *the_geom_webmercator*. -* Use PostGIS functions to simplify and filter out unneeded geometries when possible. One very handy function is, [ST_Simplify](http://www.postgis.org/docs/ST_Simplify.html). -* Remember to build indexes that will speed up some of your more common queries. For details, see [Creating Indexes](http://docs.cartodb.com/cartodb-editor/managing-your-data/#creating-indexes) -* Use *cartodb_id* to retrieve specific rows of your data, this is the unique key column added to every CartoDB table. +* Only request the fields you need. Selecting all columns will return a full version of your geometry in *the_geom*, as well as a reprojected version in *the_geom_webmercator* +* Use PostGIS functions to simplify and filter out unneeded geometries when possible. One very handy function is, [ST_Simplify](http://www.postgis.org/docs/ST_Simplify.html) +* Remember to build indexes that will speed up some of your more common queries. For details, see [Creating Indexes](#creating-indexes) +* Use *cartodb_id* to retrieve specific rows of your data, this is the unique key column added to every CARTO table. For a sample use case, view the [_Faster data updates with CARTO](https://carto.com/blog/faster-data-updates-with-cartodb/) blogpost - +## Creating Indexes + +In order to better improve map performance, advanced users can use the SQL API to add custom indexes to their data. Creating indexes is useful if you have a large dataset with filtered data. By indexing select data, you are improving the performance of the map and generating the results faster. The index functionality is useful in the following scenarios: + +- If you are filtering a dataset by values in one or a more columns +- If you are regularly querying data through the SQL API, and filtering by one or a more columns +- If you are creating Torque maps on very large datasets. Since Torque maps are based on time-sensitive data (i.e. a date or numeric column), creating an index on the time data is optimal + +Indexed data is typically a single column representing filtered data. To create a single column index, apply this SQL query to your dataset: + +{% highlight bash %} +CREATE INDEX idx_{DATASET NAME}_{COLUMN_NAME} ON {DATASET_NAME} ({COLUMN_NAME}) +{% endhighlight %} + +**Tip:** You can also apply more advanced, multi-column indexes. Please review the full documentation about [PostgreSQL Indexes](http://www.postgresql.org/docs/9.1/static/sql-createindex.html) before proceeding. + +**Note:** Indexes are allocated towards the amount of data storage associated with your account. Be mindful when creating custom indexes. Note that indexes automatically generated by CARTO are _not_ counted against your quota. For example, `the_geom` and `cartodb_id` columns. These columns are used to index geometries for your dataset and are not associated with storage. diff --git a/doc/sql_batch_api.md b/doc/sql_batch_api.md deleted file mode 100644 index 8b9e75d87..000000000 --- a/doc/sql_batch_api.md +++ /dev/null @@ -1,531 +0,0 @@ -# SQL Batch API - -The SQL Batch API enables you to request queries with long-running CPU processing times. Typically, these kind of requests raise timeout errors when using the SQL API. In order to avoid timeouts, you can use the SQL Batch API to [create](#create-a-job), [read](#read-a-job), [list](#list-jobs), [update](#update-a-job) and [cancel](#cancel-a-job) queries. You can also run [multiple](#multi-query-batch-jobs) SQL queries in one job. The SQL Batch API schedules the incoming jobs and allows you to request the job status for each query. - -_The Batch API is not intended to be used for large query payloads than contain over 4096 characters (4kb). For instance, if you are inserting a large number of rows into your table, you still need to use the [Import API](http://docs.cartodb.com/cartodb-platform/import-api/) or [SQL API](http://docs.cartodb.com/cartodb-platform/sql-api/) for this type of data management. The Batch API is specific to queries and CPU usage._ - -**Note:** In order to use the SQL Batch API, your table must be public, or you must be [authenticated](http://docs.cartodb.com/cartodb-platform/sql-api/authentication/#authentication) using API keys. For details about how to manipulate private datasets with the SQL Batch API, see [Private Datasets](#private-datasets). - -## SQL Batch API Job Schema - -The SQL Batch API request to your CartoDB account includes the following job schema elements. _Only the `query` element can be modified._ All other elements of the job schema are defined by the SQL Batch API and are read-only. - -Name | Description ---- | --- -`job_id` | a universally unique identifier (uuid). -`user` | user identifier, as displayed by the username. -`status` | displays the result of the long-running query. The possible status results are: ---- | --- -|_ `pending` | job waiting to be executed. -|_ `running` | indicates that the job is currently running. -|_ `done` | job executed successfully. -|_ `failed` | job executed but failed, with errors. -|_ `canceled` | job canceled by user request. -|_ `unknown` | appears when it is not possible to determine what exactly happened with the job. -`query` | the SQL statement to be executed in a database. _You can modify the select SQL statement to be used in the job schema._

**Tip:** In some scenarios, you may need to retrieve the query results from a finished job. See [Fetching Job Results](#fetching-job-results) for details. -`created_at` | the date and time when the job schema was created. -`updated_at` | the date and time of when the job schema was last updated, or modified. -`failed_reason` | displays the database error message, if something went wrong. - -#### Example - -```bash -HEADERS: 201 CREATED; application/json -BODY: { - "job_id": "de305d54-75b4-431b-adb2-eb6b9e546014", - "user": "cartofante", - "query": "UPDATE airports SET type = 'international'", - "status": "pending", - "created_at": "2015-12-15T07:36:25Z", - "updated_at": "2015-12-15T07:36:25Z" -} -``` - -### Create a Job - -To create an SQL Batch API job, make a POST request with the following parameters. - -Creates an SQL Batch API job request. - -```bash -HEADERS: POST /api/v2/sql/job -BODY: { - "query": "UPDATE airports SET type = 'international'" -} -``` - -#### Response - -```bash -HEADERS: 201 CREATED; application/json -BODY: { - "job_id": "de305d54-75b4-431b-adb2-eb6b9e546014", - "user": "cartofante" - "query": "UPDATE airports SET type = 'international'", - "status": "pending", - "created_at": "2015-12-15T07:36:25Z", - "updated_at": "2015-12-15T07:36:25Z" -} -``` - -##### POST Examples - -If you are using the Batch API create operation for cURL POST request, use the following code: - -```bash -curl -X POST -H "Content-Type: application/json" -d '{ - "query": "CREATE TABLE world_airports AS SELECT a.cartodb_id, a.the_geom, a.the_geom_webmercator, a.name airport, b.name country FROM world_borders b JOIN airports a ON ST_Contains(b.the_geom, a.the_geom)" -}' "http://{username}.cartodb.com/api/v2/sql/job" -``` - -If you are using the Batch API create operation for a Node.js client POST request, use the following code: - -```bash -var request = require("request"); - -var options = { - method: "POST", - url: "http://{username}.cartodb.com/api/v2/sql/job", - headers: { "content-type": "application/json" }, - body: { - query: "CREATE TABLE world_airports AS SELECT a.cartodb_id, a.the_geom, a.the_geom_webmercator, a.name airport, b.name country FROM world_borders b JOIN airports a ON ST_Contains(b.the_geom, a.the_geom)" - }, - json: true -}; - -request(options, function (error, response, body) { - if (error) throw new Error(error); - - console.log(body); -}); -``` - -### Read a Job - -To read an SQL Batch API job, make a GET request with the following parameters. - -```bash -HEADERS: GET /api/v2/sql/job/de305d54-75b4-431b-adb2-eb6b9e546014 -BODY: {} -``` - -#### Response - -```bash -HEADERS: 200 OK; application/json -BODY: { - "job_id": "de305d54-75b4-431b-adb2-eb6b9e546014", - "user": "cartofante" - "query": "UPDATE airports SET type = 'international'", - "status": "pending", - "created_at": "2015-12-15T07:36:25Z", - "updated_at": "2015-12-15T07:36:25Z" -} -``` - -##### GET Examples - -If you are using the Batch API read operation for cURL GET request, use the following code: - -```bash -curl -X GET "http://{username}.cartodb.com/api/v2/sql/job/{job_id}" -``` - -If you are using the Batch API read operation for a Node.js client GET request, use the following code: - -```bash -var request = require("request"); - -var options = { - method: "GET", - url: "http://{username}.cartodb.com/api/v2/sql/job/{job_id}" -}; - -request(options, function (error, response, body) { - if (error) throw new Error(error); - - console.log(body); -}); -``` - -### List Jobs - -To list SQL Batch API jobs, make a GET request with the following parameters. - -```bash -HEADERS: GET /api/v2/sql/job -BODY: {} -``` - -#### Response - -```bash -HEADERS: 200 OK; application/json -BODY: [{ - "job_id": "de305d54-75b4-431b-adb2-eb6b9e546014", - "user": "cartofante" - "query": "UPDATE airports SET type = 'international'", - "status": "pending", - "created_at": "2015-12-15T07:36:25Z", - "updated_at": "2015-12-15T07:36:25Z" -}, { - "job_id": "ba25ed54-75b4-431b-af27-eb6b9e5428ff", - "user": "cartofante" - "query": "CREATE TABLE world_airports AS SELECT a.cartodb_id, a.the_geom, a.the_geom_webmercator, a.name airport, b.name country FROM world_borders b JOIN airports a ON ST_Contains(b.the_geom, a.the_geom)", - "status": "pending", - "created_at": "2015-12-15T07:43:12Z", - "updated_at": "2015-12-15T07:43:12Z" -}] -``` - -##### GET Examples - -If you are using the Batch API list operation for cURL GET request, use the following code: - -```bash -curl -X GET "http://{username}.cartodb.com/api/v2/sql/job" -``` - -If you are using the Batch API list operation for a Node.js client GET request, use the following code: - -```bash -var request = require("request"); - -var options = { - method: "GET", - url: "http://{username}.cartodb.com/api/v2/sql/job" -}; - -request(options, function (error, response, body) { - if (error) throw new Error(error); - - console.log(body); -}); -``` - -### Update a Job - -To update an SQL Batch API job, make a PUT request with the following parameters. - -```bash -HEADERS: PUT /api/v2/sql/job/de305d54-75b4-431b-adb2-eb6b9e546014 -BODY: { - "query": "UPDATE airports SET type = 'military'" -} -``` - -#### Response - -```bash -HEADERS: 200 OK; application/json -BODY: { - "job_id": "de305d54-75b4-431b-adb2-eb6b9e546014", - "user": "cartofante" - "query": "UPDATE airports SET type = 'military'", - "status": "pending", - "created_at": "2015-12-15T07:36:25Z", - "updated_at": "2015-12-17T15:45:56Z" -} -``` - -**Note:** Jobs can only be updated while the `status: "pending"`, otherwise the SQL Batch API Update operation is not allowed. You will receive an error if the job status is anything but "pending". - -```bash -errors: [ - "The job status is not pending, it cannot be updated" -] -``` - -##### PUT Examples - -If you are using the Batch API update operation for cURL PUT request, use the following code: - -```bash -curl -X PUT -H "Content-Type: application/json" -d '{ - "query": "UPDATE airports SET type = 'military'" -}' "http://{username}.cartodb.com/api/v2/sql/job/{job_id}" -``` - -If you are using the Batch API update operation for a Node.js client PUT request, use the following code: - -```bash -var request = require("request"); - -var options = { - method: "PUT", - url: "http://{username}.cartodb.com/api/v2/sql/job/{job_id}", - headers: { - "content-type": "application/json" - }, - body: { query: "UPDATE airports SET type = 'military'" }, - json: true -}; - -request(options, function (error, response, body) { - if (error) throw new Error(error); - - console.log(body); -}); -``` - -### Cancel a Job - -To cancel an SQL Batch API job, make a DELETE request with the following parameters. - -```bash -HEADERS: DELETE /api/v2/sql/job/de305d54-75b4-431b-adb2-eb6b9e546014 -BODY: {} -``` - -**Note:** Be mindful when cancelling a job when the status: `pending` or `running`. - -- If the job is `pending`, the job will never be executed -- If the job is `running`, the job will be terminated immediately - -#### Response - -```bash -HEADERS: 200 OK; application/json -BODY: { - "job_id": "de305d54-75b4-431b-adb2-eb6b9e546014", - "user": "cartofante" - "query": "UPDATE airports SET type = 'international'", - "status": "cancelled", - "created_at": "2015-12-15T07:36:25Z", - "updated_at": "2015-12-17T06:22:42Z" -} -``` - -**Note:** Jobs can only be cancelled while the `status: "running"` or `status: "pending"`, otherwise the SQL Batch API Cancel operation is not allowed. You will receive an error if the job status is anything but "running" or "pending". - -```bash -errors: [ - "The job status is done, cancel is not allowed" -] -``` - -##### DELETE Examples - -If you are using the Batch API cancel operation for cURL DELETE request, use the following code: - -```bash -curl -X DELETE "http://{username}.cartodb.com/api/v2/sql/job/{job_id}" -``` - -If you are using the Batch API cancel operation for a Node.js client DELETE request, use the following code: - -```bash -var request = require("request"); - -var options = { - method: "DELETE", - url: "http://{username}.cartodb.com/api/v2/sql/job/{job_id}", -}; - -request(options, function (error, response, body) { - if (error) throw new Error(error); - - console.log(body); -}); -``` - -### Multi Query Batch Jobs - -In some cases, you may need to run multiple SQL queries in one job. The Multi Query batch option enables you run an array of SQL statements, and define the order in which the queries are executed. You can use any of the operations (create, read, list, update, cancel) for the queries in a Multi Query batch job. - -```bash -HEADERS: POST /api/v2/sql/job -BODY: { - query: [ - "CREATE TABLE world_airports AS SELECT a.cartodb_id, a.the_geom, a.the_geom_webmercator, a.name airport, b.name country FROM world_borders b JOIN airports a ON ST_Contains(b.the_geom, a.the_geom)", - "DROP TABLE airports", - "ALTER TABLE world_airports RENAME TO airport" - ] -} -``` - -#### Response - -```bash -HEADERS: 201 CREATED; application/json -BODY: { - "job_id": "de305d54-75b4-431b-adb2-eb6b9e546014", - "user": "cartofante" - "query": [{ - "query": "CREATE TABLE world_airports AS SELECT a.cartodb_id, a.the_geom, a.the_geom_webmercator, a.name airport, b.name country FROM world_borders b JOIN airports a ON ST_Contains(b.the_geom, a.the_geom)", - "status": "pending" - }, { - "query": "DROP TABLE airports", - "status": "pending" - }, { - "query": "ALTER TABLE world_airports RENAME TO airport", - "status": "pending" - }], - "status": "pending", - "created_at": "2015-12-15T07:36:25Z", - "updated_at": "2015-12-15T07:36:25Z" -} -``` - -**Note:** The SQL Batch API returns a job status for both the parent Multi Query request, and for each child query within the request. The order in which each query is executed is guaranteed. Here are the possible status results for Multi Query batch jobs: - -- If one query within the Multi Query batch fails, the `"status": "failed"` is returned for both the job and the query, and any "pending" queries will not be processed - -- If you cancel the Multi Query batch job, the job status changes to `"status": "cancelled"`. Any running queries within the job will be stopped and changed to `"status": "pending"`, and will not be processed - -- Suppose the first query job status is `"status": "done"`, the second query is `"status": "running"`, and the third query `"status": "pending"`. If the second query fails for some reason, the job status changes to `"status": "failed"` and the last query will not be processed. It is indicated which query failed in the Multi Query batch job - -##### POST Examples - -If you are using the Batch API Multi Query operation for cURL POST request, use the following code: - -```bash -curl -X POST -H "Content-Type: application/json" -d '{ - "query": [ - "CREATE TABLE world_airports AS SELECT a.cartodb_id, a.the_geom, a.the_geom_webmercator, a.name airport, b.name country FROM world_borders b JOIN airports a ON ST_Contains(b.the_geom, a.the_geom)", - "DROP TABLE airports", - "ALTER TABLE world_airports RENAME TO airport" - ] -}' "http://{username}.cartodb.com/api/v2/sql/job" -``` - -If you are using the Batch API Multi Query operation for a Node.js client POST request, use the following code: - -```bash -var request = require("request"); - -var options = { - method: "POST", - url: "http://{username}.cartodb.com/api/v2/sql/job", - headers: { "content-type": "application/json" }, - body: { - "query": [ - "CREATE TABLE world_airports AS SELECT a.cartodb_id, a.the_geom, a.the_geom_webmercator, a.name airport, b.name country FROM world_borders b JOIN airports a ON ST_Contains(b.the_geom, a.the_geom)", - "DROP TABLE airports", - "ALTER TABLE world_airports RENAME TO airport" - ] - }, - json: true -}; - -request(options, function (error, response, body) { - if (error) throw new Error(error); - - console.log(body); -}); -``` - -##### PUT Examples - -If you are using the Batch API Multi Query operation for cURL PUT request, use the following code: - -```bash -curl -X PUT -H "Content-Type: application/json" -d '{ - "query": [ - "CREATE TABLE world_airports AS SELECT a.cartodb_id, a.the_geom, a.the_geom_webmercator, a.name airport, b.name country FROM world_borders b JOIN airports a ON ST_Contains(b.the_geom, a.the_geom)", - "DROP TABLE airports", - "ALTER TABLE world_airports RENAME TO airport", - "UPDATE airports SET airport = upper(airport)" - ] -}' "http://{username}.cartodb.com/api/v2/sql/job/{job_id}" -``` - -If you are using the Batch API Multi Query operation for a Node.js client PUT request, use the following code: - -```bash -var request = require("request"); - -var options = { - method: "PUT", - url: "http://{username}.cartodb.com/api/v2/sql/job/{job_id}", - headers: { "content-type": "application/json" }, - body: { - query: [ - "CREATE TABLE world_airports AS SELECT a.cartodb_id, a.the_geom, a.the_geom_webmercator, a.name airport, b.name country FROM world_borders b JOIN airports a ON ST_Contains(b.the_geom, a.the_geom)", - "DROP TABLE airports", - "ALTER TABLE world_airports RENAME TO airport", - "UPDATE airports SET airport = upper(airport)" - ] - }, - json: true -}; - -request(options, function (error, response, body) { - if (error) throw new Error(error); - - console.log(body); -}); -``` - -## Fetching Job Results - -In some scenarios, you may need to fetch the output of a job. If that is the case, wrap the query with `SELECT * INTO`, or `CREATE TABLE AS`. The output is stored in a new table in your database. For example, if using the query `SELECT * FROM airports`: - -1. Wrap the query `SELECT * INTO job_result FROM (SELECT * FROM airports) AS job` - -2. [Create a job](#create-a-job), as described previously - -3. Once the job is done, fetch the results through the [CartoDB SQL API](http://docs.cartodb.com/cartodb-platform/sql-api/), `SELECT * FROM job_result` - -**Note:** If you need to create a map or analysis with the new table, use the [CDB_CartodbfyTable function](https://github.com/CartoDB/cartodb-postgresql/blob/master/doc/cartodbfy-requirements.rst). - -## Private Datasets - -For access to all private tables, and for write access to public tables, an API Key is required to [authenticate](http://docs.cartodb.com/cartodb-platform/sql-api/authentication/#authentication) your queries with the Batch API. The following error message appears if you are using private tables and are not authenticated: - -```bash -{ - "error": [ - "permission denied" - ] -} -``` - -In order to get full access, you must use your API Key. - -Using cURL tool: - -```bash -curl -X POST -H "Content-Type: application/json" -d '{ - "query": "{query}" -}' "http://{username}.cartodb.com/api/v2/sql/job?api_key={api_key}" -``` - -Using Node.js request client: - -```bash -var request = require("request"); - -var options = { - method: "POST", - url: "http://{username}.cartodb.com/api/v2/sql/job", - qs: { - "api_key": "{api_key}" - }, - headers: { - "content-type": "application/json" - }, - body: { - query: "{query}" - }, - json: true -}; - -request(options, function (error, response, body) { - if (error) throw new Error(error); - - console.log(body); -}); -``` - -## Best Practices - -For best practices, ensure that you are following these recommended usage notes when using the SQL Batch API: - -- The Batch API is not intended for large query payloads (e.g: inserting thousands of rows), use the [Import API](http://docs.cartodb.com/cartodb-platform/import-api/) for this type of data management - -- There is a limit of 4kb per job. The following error message appears if your job exceeds this size: - - `Your payload is too large. Max size allowed is 4096 (4kb)` - -- Only the `query` element of the job scheme can be modified. All other elements of the job schema are defined by the SQL Batch API and are read-only diff --git a/doc/tips_and_tricks.md b/doc/tips_and_tricks.md index d19fab978..ad9e54c51 100644 --- a/doc/tips_and_tricks.md +++ b/doc/tips_and_tricks.md @@ -1,39 +1,39 @@ # Other Tips and Questions -## What does CartoDB do to prevent SQL injection? +## What does CARTO do to prevent SQL injection? -CartoDB uses the database access mechanism for security. Every writable connection is verified by an API Key. If you have the correct API Key, you can write-access to the database. If you do not have the correct API Key, your client is "logged in" as a low privilege user, and you have read-only access to the database (if the database allows you to read). +CARTO uses the database access mechanism for security. Every writable connection is verified by an API Key. If you have the correct API Key, you can write-access to the database. If you do not have the correct API Key, your client is "logged in" as a low privilege user, and you have read-only access to the database (if the database allows you to read). SQL injection works by tricking a database user, so that running a query retrieves database wide results, even though the database is protected. -Because CartoDB enforces roles and access at the database level, the idea of a "SQL injection attack" is not possible with CartoDB. Injection is possible, but clients will still run into our security wall at the database level. The SQL API already lets you _attempt_ to run any query you want. The database will reject your SQL API request if it finds your user/role does not have the requisite permissions. In other words, you can ask any question of the database you like; the CartoDB database does not guarantee it will be answered. +Because CARTO enforces roles and access at the database level, the idea of a "SQL injection attack" is not possible with CARTO. Injection is possible, but clients will still run into our security wall at the database level. The SQL API already lets you _attempt_ to run any query you want. The database will reject your SQL API request if it finds your user/role does not have the requisite permissions. In other words, you can ask any question of the database you like; the CARTO Engine does not guarantee it will be answered. -If a user's API Key found its way out into the wild, that could be a problem, but it is not something CartoDB can prevent. _This is why it is very important for all CartoDB users to secure their API Keys_. In the event a user's API Key is compromised, the user (or the CartoDB Enterprise administrator), can regenerate the API Key in their account settings. +If a user's API Key found its way out into the wild, that could be a problem, but it is not something CARTO can prevent. _This is why it is very important for all CARTO users to secure their API Keys_. In the event a user's API Key is compromised, the user (or the CARTO Enterprise administrator), can regenerate the API Key in their account settings. **Note:** While the SQL API is SQL injection secure, if you build additional layers to allow another person to run queries (i.e., building a proxy so that others can indirectly perform authenticated queries through the SQL API), the security of those newly added layers are the responsibility of the creator. ## What levels of database access can roles/users have? -There are three levels of access with CartoDB: +There are three levels of access with CARTO: 1. __API Key level:__ Do whatever you want in your account on the tables you own (or have been shared with you in Enterprise/multi-user accounts). 2. __"publicuser" level:__ Do whatever has been granted to you. The publicuser level is normally read-only, but you could GRANT INSERT/UPDATE/DELETE permissions to publicuser if needed for some reason - for API Key-less write operations. Use with caution. -3. __postgres superadmin level:__ This third access level, the actual PostgreSQL system user, is only accessible from a direct database connection via the command line, which is only available currently via [CartoDB On-Premises](https://cartodb.com/on-premises/). +3. __postgres superadmin level:__ This third access level, the actual PostgreSQL system user, is only accessible from a direct database connection via the command line, which is only available currently via [CARTO On-Premises](https://carto.com/on-premises/). ## If a user has write access and makes a `DROP TABLE` query, is that data gone? -Yes. Grant write access with caution and keep backups of your data elsewhere / as duplicate CartoDB tables. +Yes. Grant write access with caution and keep backups of your data elsewhere / as duplicate CARTO tables. ## Is there an in between where a user can write but not `DROP` or `DELETE`? Yes. Create the table, and GRANT INSERT/UPDATE to the user. -## Is there an actual PostgreSQL account for each CartoDB login/username? +## Is there an actual PostgreSQL account for each CARTO login/username? -Yes, there is. Unfortunately, the names are different - though there is a way to determine the name of the PostgreSQL user account. Every CartoDB user gets their own PostgreSQL database. But there is a system database too, with the name mappings in `username` and `database_name` columns. `database_name` is the name of the database that user belongs to. It will be `cartodb_user_ID`. `id` holds long hashkey. The `database_name` is derived from this ID hash too, but in case of an Enterprise/multi-user account it will come from the user ID of the owner of the organization - and `database_name` will hold the same value for every user in an Enterprise/multi-user account. +Yes, there is. Unfortunately, the names are different - though there is a way to determine the name of the PostgreSQL user account. Every CARTO user gets their own PostgreSQL database. But there is a system database too, with the name mappings in `username` and `database_name` columns. `database_name` is the name of the database that user belongs to. It will be `cartodb_user_ID`. `id` holds long hashkey. The `database_name` is derived from this ID hash too, but in case of an Enterprise/multi-user account it will come from the user ID of the owner of the organization - and `database_name` will hold the same value for every user in an Enterprise/multi-user account. -You can also just do `select user` using the SQL API (without an API Key to get the publicuser name and with an API Key to get the CartoDB user's PostgreSQL user name), to determine the name of the corresponding PostgreSQL user. +You can also just do `select user` using the SQL API (without an API Key to get the publicuser name and with an API Key to get the CARTO user's PostgreSQL user name), to determine the name of the corresponding PostgreSQL user. -## Can I configure my CartoDB database permissions exactly the same way I do on my own PostgreSQL instance? +## Can I configure my CARTO database permissions exactly the same way I do on my own PostgreSQL instance? Yes, through using GRANT statements to the SQL API. There are a few caveats to be aware of, including the aforementioned naming differences. Also, you will be limited to permissions a user has with their own tables. Users do not have PostgreSQL superuser privileges. So they cannot be creating languages, or C functions, or anything that requires superuser or CREATEUSER privileges. diff --git a/doc/version.md b/doc/version.md index 31460e066..5c281b887 100644 --- a/doc/version.md +++ b/doc/version.md @@ -1,3 +1,3 @@ # API Version Number -All CartoDB applications use **Version 2** of our APIs. All other APIs are deprecated and will not be maintained or supported. You can check that you are using **Version 2** of our APIs by looking at your request URLS. They should all begin containing **/v2/** in the URLs as follows, `https://{username}.cartodb.com/api/v2/` +All CARTO applications use **Version 2** of our APIs. All other APIs are deprecated and will not be maintained or supported. You can check that you are using **Version 2** of our APIs by looking at your request URLS. They should all begin containing **/v2/** in the URLs as follows, `https://{username}.carto.com/api/v2/` diff --git a/npm-shrinkwrap.json b/npm-shrinkwrap.json index 3afd24459..5f6afd6ba 100644 --- a/npm-shrinkwrap.json +++ b/npm-shrinkwrap.json @@ -1,25 +1,155 @@ { "name": "cartodb_sql_api", - "version": "1.29.1", + "version": "1.42.1", "dependencies": { + "bintrees": { + "version": "1.0.1", + "from": "https://registry.npmjs.org/bintrees/-/bintrees-1.0.1.tgz", + "resolved": "https://registry.npmjs.org/bintrees/-/bintrees-1.0.1.tgz" + }, + "bunyan": { + "version": "1.8.1", + "from": "https://registry.npmjs.org/bunyan/-/bunyan-1.8.1.tgz", + "resolved": "https://registry.npmjs.org/bunyan/-/bunyan-1.8.1.tgz", + "dependencies": { + "dtrace-provider": { + "version": "0.6.0", + "from": "https://registry.npmjs.org/dtrace-provider/-/dtrace-provider-0.6.0.tgz", + "resolved": "https://registry.npmjs.org/dtrace-provider/-/dtrace-provider-0.6.0.tgz", + "dependencies": { + "nan": { + "version": "2.4.0", + "from": "https://registry.npmjs.org/nan/-/nan-2.4.0.tgz", + "resolved": "https://registry.npmjs.org/nan/-/nan-2.4.0.tgz" + } + } + }, + "mv": { + "version": "2.1.1", + "from": "https://registry.npmjs.org/mv/-/mv-2.1.1.tgz", + "resolved": "https://registry.npmjs.org/mv/-/mv-2.1.1.tgz", + "dependencies": { + "mkdirp": { + "version": "0.5.1", + "from": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.1.tgz", + "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.1.tgz", + "dependencies": { + "minimist": { + "version": "0.0.8", + "from": "https://registry.npmjs.org/minimist/-/minimist-0.0.8.tgz", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-0.0.8.tgz" + } + } + }, + "ncp": { + "version": "2.0.0", + "from": "https://registry.npmjs.org/ncp/-/ncp-2.0.0.tgz", + "resolved": "https://registry.npmjs.org/ncp/-/ncp-2.0.0.tgz" + }, + "rimraf": { + "version": "2.4.5", + "from": "https://registry.npmjs.org/rimraf/-/rimraf-2.4.5.tgz", + "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.4.5.tgz", + "dependencies": { + "glob": { + "version": "6.0.4", + "from": "https://registry.npmjs.org/glob/-/glob-6.0.4.tgz", + "resolved": "https://registry.npmjs.org/glob/-/glob-6.0.4.tgz", + "dependencies": { + "inflight": { + "version": "1.0.6", + "from": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", + "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", + "dependencies": { + "wrappy": { + "version": "1.0.2", + "from": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", + "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz" + } + } + }, + "inherits": { + "version": "2.0.3", + "from": "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz" + }, + "minimatch": { + "version": "3.0.3", + "from": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.3.tgz", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.3.tgz", + "dependencies": { + "brace-expansion": { + "version": "1.1.6", + "from": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.6.tgz", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.6.tgz", + "dependencies": { + "balanced-match": { + "version": "0.4.2", + "from": "https://registry.npmjs.org/balanced-match/-/balanced-match-0.4.2.tgz", + "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-0.4.2.tgz" + }, + "concat-map": { + "version": "0.0.1", + "from": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", + "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz" + } + } + } + } + }, + "once": { + "version": "1.4.0", + "from": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", + "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", + "dependencies": { + "wrappy": { + "version": "1.0.2", + "from": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", + "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz" + } + } + }, + "path-is-absolute": { + "version": "1.0.1", + "from": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", + "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz" + } + } + } + } + } + } + }, + "safe-json-stringify": { + "version": "1.0.3", + "from": "https://registry.npmjs.org/safe-json-stringify/-/safe-json-stringify-1.0.3.tgz", + "resolved": "https://registry.npmjs.org/safe-json-stringify/-/safe-json-stringify-1.0.3.tgz" + }, + "moment": { + "version": "2.15.1", + "from": "https://registry.npmjs.org/moment/-/moment-2.15.1.tgz", + "resolved": "https://registry.npmjs.org/moment/-/moment-2.15.1.tgz" + } + } + }, "cartodb-psql": { "version": "0.6.1", - "from": "cartodb-psql@>=0.6.0 <0.7.0", + "from": "https://registry.npmjs.org/cartodb-psql/-/cartodb-psql-0.6.1.tgz", "resolved": "https://registry.npmjs.org/cartodb-psql/-/cartodb-psql-0.6.1.tgz", "dependencies": { "pg": { "version": "2.6.2-cdb3", - "from": "git://github.com/CartoDB/node-postgres.git#2.6.2-cdb3", + "from": "git://github.com/CartoDB/node-postgres.git#069c5296d1a093077feff21719641bb9e71fc50e", "resolved": "git://github.com/CartoDB/node-postgres.git#069c5296d1a093077feff21719641bb9e71fc50e", "dependencies": { "generic-pool": { "version": "2.0.3", - "from": "generic-pool@2.0.3", + "from": "https://registry.npmjs.org/generic-pool/-/generic-pool-2.0.3.tgz", "resolved": "https://registry.npmjs.org/generic-pool/-/generic-pool-2.0.3.tgz" }, "buffer-writer": { "version": "1.0.0", - "from": "buffer-writer@1.0.0", + "from": "https://registry.npmjs.org/buffer-writer/-/buffer-writer-1.0.0.tgz", "resolved": "https://registry.npmjs.org/buffer-writer/-/buffer-writer-1.0.0.tgz" } } @@ -27,250 +157,991 @@ } }, "cartodb-query-tables": { - "version": "0.1.0", - "from": "cartodb-query-tables@0.1.0", - "resolved": "https://registry.npmjs.org/cartodb-query-tables/-/cartodb-query-tables-0.1.0.tgz" + "version": "0.2.0", + "from": "https://registry.npmjs.org/cartodb-query-tables/-/cartodb-query-tables-0.2.0.tgz", + "resolved": "https://registry.npmjs.org/cartodb-query-tables/-/cartodb-query-tables-0.2.0.tgz" }, "cartodb-redis": { - "version": "0.11.0", - "from": "cartodb-redis@>=0.11.0 <0.12.0", - "resolved": "https://registry.npmjs.org/cartodb-redis/-/cartodb-redis-0.11.0.tgz", + "version": "0.13.1", + "from": "https://registry.npmjs.org/cartodb-redis/-/cartodb-redis-0.13.1.tgz", + "resolved": "https://registry.npmjs.org/cartodb-redis/-/cartodb-redis-0.13.1.tgz", "dependencies": { - "strftime": { - "version": "0.8.4", - "from": "strftime@>=0.8.2 <0.9.0", - "resolved": "https://registry.npmjs.org/strftime/-/strftime-0.8.4.tgz" - }, "dot": { "version": "1.0.3", - "from": "dot@>=1.0.2 <1.1.0", + "from": "https://registry.npmjs.org/dot/-/dot-1.0.3.tgz", "resolved": "https://registry.npmjs.org/dot/-/dot-1.0.3.tgz" - }, - "redis-mpool": { - "version": "0.1.0", - "from": "git://github.com/CartoDB/node-redis-mpool.git#0.1.0", - "resolved": "git://github.com/CartoDB/node-redis-mpool.git#47510b8d4525ee24aa2e5328976372274a1d144e", - "dependencies": { - "generic-pool": { - "version": "2.1.1", - "from": "generic-pool@>=2.1.1 <2.2.0", - "resolved": "https://registry.npmjs.org/generic-pool/-/generic-pool-2.1.1.tgz" - }, - "redis": { - "version": "0.12.1", - "from": "redis@>=0.12.1 <0.13.0", - "resolved": "https://registry.npmjs.org/redis/-/redis-0.12.1.tgz" - }, - "hiredis": { - "version": "0.1.17", - "from": "hiredis@>=0.1.17 <0.2.0", - "resolved": "https://registry.npmjs.org/hiredis/-/hiredis-0.1.17.tgz", - "dependencies": { - "bindings": { - "version": "1.2.1", - "from": "bindings@*", - "resolved": "https://registry.npmjs.org/bindings/-/bindings-1.2.1.tgz" - }, - "nan": { - "version": "1.1.2", - "from": "nan@>=1.1.0 <1.2.0", - "resolved": "https://registry.npmjs.org/nan/-/nan-1.1.2.tgz" - } - } - } - } } } }, "debug": { "version": "2.2.0", - "from": "debug@2.2.0", + "from": "https://registry.npmjs.org/debug/-/debug-2.2.0.tgz", "resolved": "https://registry.npmjs.org/debug/-/debug-2.2.0.tgz", "dependencies": { "ms": { "version": "0.7.1", - "from": "ms@0.7.1", + "from": "https://registry.npmjs.org/ms/-/ms-0.7.1.tgz", "resolved": "https://registry.npmjs.org/ms/-/ms-0.7.1.tgz" } } }, "express": { - "version": "2.5.11", - "from": "express@>=2.5.11 <2.6.0", - "resolved": "https://registry.npmjs.org/express/-/express-2.5.11.tgz", + "version": "4.13.4", + "from": "https://registry.npmjs.org/express/-/express-4.13.4.tgz", + "resolved": "https://registry.npmjs.org/express/-/express-4.13.4.tgz", "dependencies": { - "connect": { - "version": "1.9.2", - "from": "connect@>=1.0.0 <2.0.0", - "resolved": "https://registry.npmjs.org/connect/-/connect-1.9.2.tgz", + "accepts": { + "version": "1.2.13", + "from": "https://registry.npmjs.org/accepts/-/accepts-1.2.13.tgz", + "resolved": "https://registry.npmjs.org/accepts/-/accepts-1.2.13.tgz", + "dependencies": { + "mime-types": { + "version": "2.1.12", + "from": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.12.tgz", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.12.tgz", + "dependencies": { + "mime-db": { + "version": "1.24.0", + "from": "https://registry.npmjs.org/mime-db/-/mime-db-1.24.0.tgz", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.24.0.tgz" + } + } + }, + "negotiator": { + "version": "0.5.3", + "from": "https://registry.npmjs.org/negotiator/-/negotiator-0.5.3.tgz", + "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.5.3.tgz" + } + } + }, + "array-flatten": { + "version": "1.1.1", + "from": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz", + "resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz" + }, + "content-disposition": { + "version": "0.5.1", + "from": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.1.tgz", + "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.1.tgz" + }, + "content-type": { + "version": "1.0.2", + "from": "https://registry.npmjs.org/content-type/-/content-type-1.0.2.tgz", + "resolved": "https://registry.npmjs.org/content-type/-/content-type-1.0.2.tgz" + }, + "cookie": { + "version": "0.1.5", + "from": "https://registry.npmjs.org/cookie/-/cookie-0.1.5.tgz", + "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.1.5.tgz" + }, + "cookie-signature": { + "version": "1.0.6", + "from": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.0.6.tgz", + "resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.0.6.tgz" + }, + "depd": { + "version": "1.1.0", + "from": "https://registry.npmjs.org/depd/-/depd-1.1.0.tgz", + "resolved": "https://registry.npmjs.org/depd/-/depd-1.1.0.tgz" + }, + "escape-html": { + "version": "1.0.3", + "from": "https://registry.npmjs.org/escape-html/-/escape-html-1.0.3.tgz", + "resolved": "https://registry.npmjs.org/escape-html/-/escape-html-1.0.3.tgz" + }, + "etag": { + "version": "1.7.0", + "from": "https://registry.npmjs.org/etag/-/etag-1.7.0.tgz", + "resolved": "https://registry.npmjs.org/etag/-/etag-1.7.0.tgz" + }, + "finalhandler": { + "version": "0.4.1", + "from": "https://registry.npmjs.org/finalhandler/-/finalhandler-0.4.1.tgz", + "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-0.4.1.tgz", "dependencies": { - "formidable": { - "version": "1.0.17", - "from": "formidable@>=1.0.0 <1.1.0", - "resolved": "https://registry.npmjs.org/formidable/-/formidable-1.0.17.tgz" + "unpipe": { + "version": "1.0.0", + "from": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz", + "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz" } } }, - "mime": { - "version": "1.2.4", - "from": "mime@1.2.4", - "resolved": "https://registry.npmjs.org/mime/-/mime-1.2.4.tgz" + "fresh": { + "version": "0.3.0", + "from": "https://registry.npmjs.org/fresh/-/fresh-0.3.0.tgz", + "resolved": "https://registry.npmjs.org/fresh/-/fresh-0.3.0.tgz" + }, + "merge-descriptors": { + "version": "1.0.1", + "from": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-1.0.1.tgz", + "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-1.0.1.tgz" + }, + "methods": { + "version": "1.1.2", + "from": "https://registry.npmjs.org/methods/-/methods-1.1.2.tgz", + "resolved": "https://registry.npmjs.org/methods/-/methods-1.1.2.tgz" + }, + "on-finished": { + "version": "2.3.0", + "from": "https://registry.npmjs.org/on-finished/-/on-finished-2.3.0.tgz", + "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.3.0.tgz", + "dependencies": { + "ee-first": { + "version": "1.1.1", + "from": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz", + "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz" + } + } + }, + "parseurl": { + "version": "1.3.1", + "from": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.1.tgz", + "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.1.tgz" + }, + "path-to-regexp": { + "version": "0.1.7", + "from": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.7.tgz", + "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.7.tgz" + }, + "proxy-addr": { + "version": "1.0.10", + "from": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-1.0.10.tgz", + "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-1.0.10.tgz", + "dependencies": { + "forwarded": { + "version": "0.1.0", + "from": "https://registry.npmjs.org/forwarded/-/forwarded-0.1.0.tgz", + "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.1.0.tgz" + }, + "ipaddr.js": { + "version": "1.0.5", + "from": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.0.5.tgz", + "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.0.5.tgz" + } + } }, "qs": { - "version": "0.4.2", - "from": "qs@>=0.4.0 <0.5.0", - "resolved": "https://registry.npmjs.org/qs/-/qs-0.4.2.tgz" + "version": "4.0.0", + "from": "https://registry.npmjs.org/qs/-/qs-4.0.0.tgz", + "resolved": "https://registry.npmjs.org/qs/-/qs-4.0.0.tgz" }, - "mkdirp": { - "version": "0.3.0", - "from": "mkdirp@0.3.0", - "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.3.0.tgz" + "range-parser": { + "version": "1.0.3", + "from": "https://registry.npmjs.org/range-parser/-/range-parser-1.0.3.tgz", + "resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.0.3.tgz" + }, + "send": { + "version": "0.13.1", + "from": "https://registry.npmjs.org/send/-/send-0.13.1.tgz", + "resolved": "https://registry.npmjs.org/send/-/send-0.13.1.tgz", + "dependencies": { + "destroy": { + "version": "1.0.4", + "from": "https://registry.npmjs.org/destroy/-/destroy-1.0.4.tgz", + "resolved": "https://registry.npmjs.org/destroy/-/destroy-1.0.4.tgz" + }, + "http-errors": { + "version": "1.3.1", + "from": "https://registry.npmjs.org/http-errors/-/http-errors-1.3.1.tgz", + "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-1.3.1.tgz", + "dependencies": { + "inherits": { + "version": "2.0.3", + "from": "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz" + } + } + }, + "mime": { + "version": "1.3.4", + "from": "https://registry.npmjs.org/mime/-/mime-1.3.4.tgz", + "resolved": "https://registry.npmjs.org/mime/-/mime-1.3.4.tgz" + }, + "ms": { + "version": "0.7.1", + "from": "https://registry.npmjs.org/ms/-/ms-0.7.1.tgz", + "resolved": "https://registry.npmjs.org/ms/-/ms-0.7.1.tgz" + }, + "statuses": { + "version": "1.2.1", + "from": "https://registry.npmjs.org/statuses/-/statuses-1.2.1.tgz", + "resolved": "https://registry.npmjs.org/statuses/-/statuses-1.2.1.tgz" + } + } + }, + "serve-static": { + "version": "1.10.3", + "from": "https://registry.npmjs.org/serve-static/-/serve-static-1.10.3.tgz", + "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-1.10.3.tgz", + "dependencies": { + "send": { + "version": "0.13.2", + "from": "https://registry.npmjs.org/send/-/send-0.13.2.tgz", + "resolved": "https://registry.npmjs.org/send/-/send-0.13.2.tgz", + "dependencies": { + "destroy": { + "version": "1.0.4", + "from": "https://registry.npmjs.org/destroy/-/destroy-1.0.4.tgz", + "resolved": "https://registry.npmjs.org/destroy/-/destroy-1.0.4.tgz" + }, + "http-errors": { + "version": "1.3.1", + "from": "https://registry.npmjs.org/http-errors/-/http-errors-1.3.1.tgz", + "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-1.3.1.tgz", + "dependencies": { + "inherits": { + "version": "2.0.3", + "from": "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz" + } + } + }, + "mime": { + "version": "1.3.4", + "from": "https://registry.npmjs.org/mime/-/mime-1.3.4.tgz", + "resolved": "https://registry.npmjs.org/mime/-/mime-1.3.4.tgz" + }, + "ms": { + "version": "0.7.1", + "from": "https://registry.npmjs.org/ms/-/ms-0.7.1.tgz", + "resolved": "https://registry.npmjs.org/ms/-/ms-0.7.1.tgz" + }, + "statuses": { + "version": "1.2.1", + "from": "https://registry.npmjs.org/statuses/-/statuses-1.2.1.tgz", + "resolved": "https://registry.npmjs.org/statuses/-/statuses-1.2.1.tgz" + } + } + } + } + }, + "type-is": { + "version": "1.6.13", + "from": "https://registry.npmjs.org/type-is/-/type-is-1.6.13.tgz", + "resolved": "https://registry.npmjs.org/type-is/-/type-is-1.6.13.tgz", + "dependencies": { + "media-typer": { + "version": "0.3.0", + "from": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz", + "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz" + }, + "mime-types": { + "version": "2.1.12", + "from": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.12.tgz", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.12.tgz", + "dependencies": { + "mime-db": { + "version": "1.24.0", + "from": "https://registry.npmjs.org/mime-db/-/mime-db-1.24.0.tgz", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.24.0.tgz" + } + } + } + } + }, + "utils-merge": { + "version": "1.0.0", + "from": "https://registry.npmjs.org/utils-merge/-/utils-merge-1.0.0.tgz", + "resolved": "https://registry.npmjs.org/utils-merge/-/utils-merge-1.0.0.tgz" + }, + "vary": { + "version": "1.0.1", + "from": "https://registry.npmjs.org/vary/-/vary-1.0.1.tgz", + "resolved": "https://registry.npmjs.org/vary/-/vary-1.0.1.tgz" } } }, "log4js": { "version": "0.6.25", - "from": "https://github.com/CartoDB/log4js-node/tarball/cdb", - "resolved": "https://github.com/CartoDB/log4js-node/tarball/cdb", + "from": "git://github.com/cartodb/log4js-node.git#145d5f91e35e7fb14a6278cbf7a711ced6603727", + "resolved": "git://github.com/cartodb/log4js-node.git#145d5f91e35e7fb14a6278cbf7a711ced6603727", "dependencies": { "async": { "version": "0.2.10", - "from": "async@>=0.2.0 <0.3.0", + "from": "https://registry.npmjs.org/async/-/async-0.2.10.tgz", "resolved": "https://registry.npmjs.org/async/-/async-0.2.10.tgz" }, "readable-stream": { "version": "1.0.34", - "from": "readable-stream@>=1.0.2 <1.1.0", + "from": "https://registry.npmjs.org/readable-stream/-/readable-stream-1.0.34.tgz", "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-1.0.34.tgz", "dependencies": { "core-util-is": { "version": "1.0.2", - "from": "core-util-is@>=1.0.0 <1.1.0", + "from": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz", "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz" }, "isarray": { "version": "0.0.1", - "from": "isarray@0.0.1", + "from": "https://registry.npmjs.org/isarray/-/isarray-0.0.1.tgz", "resolved": "https://registry.npmjs.org/isarray/-/isarray-0.0.1.tgz" }, "string_decoder": { "version": "0.10.31", - "from": "string_decoder@>=0.10.0 <0.11.0", + "from": "https://registry.npmjs.org/string_decoder/-/string_decoder-0.10.31.tgz", "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-0.10.31.tgz" }, "inherits": { - "version": "2.0.1", - "from": "inherits@>=2.0.1 <2.1.0", - "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.1.tgz" + "version": "2.0.3", + "from": "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz" } } }, "semver": { "version": "4.3.6", - "from": "semver@>=4.3.3 <4.4.0", + "from": "https://registry.npmjs.org/semver/-/semver-4.3.6.tgz", "resolved": "https://registry.npmjs.org/semver/-/semver-4.3.6.tgz" }, "underscore": { "version": "1.8.2", - "from": "underscore@1.8.2", + "from": "https://registry.npmjs.org/underscore/-/underscore-1.8.2.tgz", "resolved": "https://registry.npmjs.org/underscore/-/underscore-1.8.2.tgz" } } }, "lru-cache": { "version": "2.5.2", - "from": "lru-cache@>=2.5.0 <2.6.0", + "from": "https://registry.npmjs.org/lru-cache/-/lru-cache-2.5.2.tgz", "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-2.5.2.tgz" }, + "multer": { + "version": "1.2.0", + "from": "https://registry.npmjs.org/multer/-/multer-1.2.0.tgz", + "resolved": "https://registry.npmjs.org/multer/-/multer-1.2.0.tgz", + "dependencies": { + "append-field": { + "version": "0.1.0", + "from": "https://registry.npmjs.org/append-field/-/append-field-0.1.0.tgz", + "resolved": "https://registry.npmjs.org/append-field/-/append-field-0.1.0.tgz" + }, + "busboy": { + "version": "0.2.13", + "from": "https://registry.npmjs.org/busboy/-/busboy-0.2.13.tgz", + "resolved": "https://registry.npmjs.org/busboy/-/busboy-0.2.13.tgz", + "dependencies": { + "dicer": { + "version": "0.2.5", + "from": "https://registry.npmjs.org/dicer/-/dicer-0.2.5.tgz", + "resolved": "https://registry.npmjs.org/dicer/-/dicer-0.2.5.tgz", + "dependencies": { + "streamsearch": { + "version": "0.1.2", + "from": "https://registry.npmjs.org/streamsearch/-/streamsearch-0.1.2.tgz", + "resolved": "https://registry.npmjs.org/streamsearch/-/streamsearch-0.1.2.tgz" + } + } + }, + "readable-stream": { + "version": "1.1.14", + "from": "https://registry.npmjs.org/readable-stream/-/readable-stream-1.1.14.tgz", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-1.1.14.tgz", + "dependencies": { + "core-util-is": { + "version": "1.0.2", + "from": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz", + "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz" + }, + "isarray": { + "version": "0.0.1", + "from": "https://registry.npmjs.org/isarray/-/isarray-0.0.1.tgz", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-0.0.1.tgz" + }, + "string_decoder": { + "version": "0.10.31", + "from": "https://registry.npmjs.org/string_decoder/-/string_decoder-0.10.31.tgz", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-0.10.31.tgz" + }, + "inherits": { + "version": "2.0.3", + "from": "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz" + } + } + } + } + }, + "concat-stream": { + "version": "1.5.2", + "from": "https://registry.npmjs.org/concat-stream/-/concat-stream-1.5.2.tgz", + "resolved": "https://registry.npmjs.org/concat-stream/-/concat-stream-1.5.2.tgz", + "dependencies": { + "inherits": { + "version": "2.0.3", + "from": "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz" + }, + "typedarray": { + "version": "0.0.6", + "from": "https://registry.npmjs.org/typedarray/-/typedarray-0.0.6.tgz", + "resolved": "https://registry.npmjs.org/typedarray/-/typedarray-0.0.6.tgz" + }, + "readable-stream": { + "version": "2.0.6", + "from": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.0.6.tgz", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.0.6.tgz", + "dependencies": { + "core-util-is": { + "version": "1.0.2", + "from": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz", + "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz" + }, + "isarray": { + "version": "1.0.0", + "from": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz" + }, + "process-nextick-args": { + "version": "1.0.7", + "from": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-1.0.7.tgz", + "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-1.0.7.tgz" + }, + "string_decoder": { + "version": "0.10.31", + "from": "https://registry.npmjs.org/string_decoder/-/string_decoder-0.10.31.tgz", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-0.10.31.tgz" + }, + "util-deprecate": { + "version": "1.0.2", + "from": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", + "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz" + } + } + } + } + }, + "mkdirp": { + "version": "0.5.1", + "from": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.1.tgz", + "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.1.tgz", + "dependencies": { + "minimist": { + "version": "0.0.8", + "from": "https://registry.npmjs.org/minimist/-/minimist-0.0.8.tgz", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-0.0.8.tgz" + } + } + }, + "object-assign": { + "version": "3.0.0", + "from": "https://registry.npmjs.org/object-assign/-/object-assign-3.0.0.tgz", + "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-3.0.0.tgz" + }, + "on-finished": { + "version": "2.3.0", + "from": "https://registry.npmjs.org/on-finished/-/on-finished-2.3.0.tgz", + "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.3.0.tgz", + "dependencies": { + "ee-first": { + "version": "1.1.1", + "from": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz", + "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz" + } + } + }, + "type-is": { + "version": "1.6.13", + "from": "https://registry.npmjs.org/type-is/-/type-is-1.6.13.tgz", + "resolved": "https://registry.npmjs.org/type-is/-/type-is-1.6.13.tgz", + "dependencies": { + "media-typer": { + "version": "0.3.0", + "from": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz", + "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz" + }, + "mime-types": { + "version": "2.1.12", + "from": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.12.tgz", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.12.tgz", + "dependencies": { + "mime-db": { + "version": "1.24.0", + "from": "https://registry.npmjs.org/mime-db/-/mime-db-1.24.0.tgz", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.24.0.tgz" + } + } + } + } + }, + "xtend": { + "version": "4.0.1", + "from": "https://registry.npmjs.org/xtend/-/xtend-4.0.1.tgz", + "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.1.tgz" + } + } + }, "node-statsd": { "version": "0.0.7", - "from": "node-statsd@>=0.0.7 <0.1.0", + "from": "https://registry.npmjs.org/node-statsd/-/node-statsd-0.0.7.tgz", "resolved": "https://registry.npmjs.org/node-statsd/-/node-statsd-0.0.7.tgz" }, "node-uuid": { "version": "1.4.7", - "from": "node-uuid@>=1.4.7 <2.0.0", + "from": "https://registry.npmjs.org/node-uuid/-/node-uuid-1.4.7.tgz", "resolved": "https://registry.npmjs.org/node-uuid/-/node-uuid-1.4.7.tgz" }, "oauth-client": { "version": "0.3.0", - "from": "oauth-client@0.3.0", + "from": "https://registry.npmjs.org/oauth-client/-/oauth-client-0.3.0.tgz", "resolved": "https://registry.npmjs.org/oauth-client/-/oauth-client-0.3.0.tgz", "dependencies": { "node-uuid": { "version": "1.1.0", - "from": "node-uuid@1.1.0", + "from": "https://registry.npmjs.org/node-uuid/-/node-uuid-1.1.0.tgz", "resolved": "https://registry.npmjs.org/node-uuid/-/node-uuid-1.1.0.tgz" } } }, + "qs": { + "version": "6.2.1", + "from": "https://registry.npmjs.org/qs/-/qs-6.2.1.tgz", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.2.1.tgz" + }, "queue-async": { "version": "1.0.7", - "from": "queue-async@>=1.0.7 <1.1.0", + "from": "https://registry.npmjs.org/queue-async/-/queue-async-1.0.7.tgz", "resolved": "https://registry.npmjs.org/queue-async/-/queue-async-1.0.7.tgz" }, - "redis": { - "version": "2.5.3", - "from": "redis@>=2.4.2 <3.0.0", - "resolved": "https://registry.npmjs.org/redis/-/redis-2.5.3.tgz", + "redis-mpool": { + "version": "0.4.0", + "from": "https://registry.npmjs.org/redis-mpool/-/redis-mpool-0.4.0.tgz", + "resolved": "https://registry.npmjs.org/redis-mpool/-/redis-mpool-0.4.0.tgz", "dependencies": { - "double-ended-queue": { - "version": "2.1.0-0", - "from": "double-ended-queue@>=2.1.0-0 <3.0.0", - "resolved": "https://registry.npmjs.org/double-ended-queue/-/double-ended-queue-2.1.0-0.tgz" + "generic-pool": { + "version": "2.1.1", + "from": "https://registry.npmjs.org/generic-pool/-/generic-pool-2.1.1.tgz", + "resolved": "https://registry.npmjs.org/generic-pool/-/generic-pool-2.1.1.tgz" }, - "redis-commands": { - "version": "1.2.0", - "from": "redis-commands@>=1.0.1 <2.0.0", - "resolved": "https://registry.npmjs.org/redis-commands/-/redis-commands-1.2.0.tgz" + "redis": { + "version": "0.12.1", + "from": "https://registry.npmjs.org/redis/-/redis-0.12.1.tgz", + "resolved": "https://registry.npmjs.org/redis/-/redis-0.12.1.tgz" }, - "redis-parser": { - "version": "1.3.0", - "from": "redis-parser@>=1.1.0 <2.0.0", - "resolved": "https://registry.npmjs.org/redis-parser/-/redis-parser-1.3.0.tgz" + "hiredis": { + "version": "0.1.17", + "from": "https://registry.npmjs.org/hiredis/-/hiredis-0.1.17.tgz", + "resolved": "https://registry.npmjs.org/hiredis/-/hiredis-0.1.17.tgz", + "dependencies": { + "bindings": { + "version": "1.2.1", + "from": "https://registry.npmjs.org/bindings/-/bindings-1.2.1.tgz", + "resolved": "https://registry.npmjs.org/bindings/-/bindings-1.2.1.tgz" + }, + "nan": { + "version": "1.1.2", + "from": "https://registry.npmjs.org/nan/-/nan-1.1.2.tgz", + "resolved": "https://registry.npmjs.org/nan/-/nan-1.1.2.tgz" + } + } } } }, - "rollbar": { - "version": "0.3.13", - "from": "rollbar@>=0.3.2 <0.4.0", - "resolved": "https://registry.npmjs.org/rollbar/-/rollbar-0.3.13.tgz", + "redlock": { + "version": "2.0.1", + "from": "https://registry.npmjs.org/redlock/-/redlock-2.0.1.tgz", + "resolved": "https://registry.npmjs.org/redlock/-/redlock-2.0.1.tgz", "dependencies": { - "lru-cache": { - "version": "2.2.4", - "from": "lru-cache@>=2.2.1 <2.3.0", - "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-2.2.4.tgz" + "bluebird": { + "version": "3.4.6", + "from": "https://registry.npmjs.org/bluebird/-/bluebird-3.4.6.tgz", + "resolved": "https://registry.npmjs.org/bluebird/-/bluebird-3.4.6.tgz" + } + } + }, + "request": { + "version": "2.75.0", + "from": "https://registry.npmjs.org/request/-/request-2.75.0.tgz", + "resolved": "https://registry.npmjs.org/request/-/request-2.75.0.tgz", + "dependencies": { + "aws-sign2": { + "version": "0.6.0", + "from": "https://registry.npmjs.org/aws-sign2/-/aws-sign2-0.6.0.tgz", + "resolved": "https://registry.npmjs.org/aws-sign2/-/aws-sign2-0.6.0.tgz" + }, + "aws4": { + "version": "1.5.0", + "from": "https://registry.npmjs.org/aws4/-/aws4-1.5.0.tgz", + "resolved": "https://registry.npmjs.org/aws4/-/aws4-1.5.0.tgz" + }, + "bl": { + "version": "1.1.2", + "from": "https://registry.npmjs.org/bl/-/bl-1.1.2.tgz", + "resolved": "https://registry.npmjs.org/bl/-/bl-1.1.2.tgz", + "dependencies": { + "readable-stream": { + "version": "2.0.6", + "from": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.0.6.tgz", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.0.6.tgz", + "dependencies": { + "core-util-is": { + "version": "1.0.2", + "from": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz", + "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz" + }, + "inherits": { + "version": "2.0.3", + "from": "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz" + }, + "isarray": { + "version": "1.0.0", + "from": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz" + }, + "process-nextick-args": { + "version": "1.0.7", + "from": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-1.0.7.tgz", + "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-1.0.7.tgz" + }, + "string_decoder": { + "version": "0.10.31", + "from": "https://registry.npmjs.org/string_decoder/-/string_decoder-0.10.31.tgz", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-0.10.31.tgz" + }, + "util-deprecate": { + "version": "1.0.2", + "from": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", + "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz" + } + } + } + } + }, + "caseless": { + "version": "0.11.0", + "from": "https://registry.npmjs.org/caseless/-/caseless-0.11.0.tgz", + "resolved": "https://registry.npmjs.org/caseless/-/caseless-0.11.0.tgz" + }, + "combined-stream": { + "version": "1.0.5", + "from": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.5.tgz", + "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.5.tgz", + "dependencies": { + "delayed-stream": { + "version": "1.0.0", + "from": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", + "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz" + } + } + }, + "extend": { + "version": "3.0.0", + "from": "https://registry.npmjs.org/extend/-/extend-3.0.0.tgz", + "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.0.tgz" + }, + "forever-agent": { + "version": "0.6.1", + "from": "https://registry.npmjs.org/forever-agent/-/forever-agent-0.6.1.tgz", + "resolved": "https://registry.npmjs.org/forever-agent/-/forever-agent-0.6.1.tgz" + }, + "form-data": { + "version": "2.0.0", + "from": "https://registry.npmjs.org/form-data/-/form-data-2.0.0.tgz", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-2.0.0.tgz", + "dependencies": { + "asynckit": { + "version": "0.4.0", + "from": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", + "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz" + } + } + }, + "har-validator": { + "version": "2.0.6", + "from": "https://registry.npmjs.org/har-validator/-/har-validator-2.0.6.tgz", + "resolved": "https://registry.npmjs.org/har-validator/-/har-validator-2.0.6.tgz", + "dependencies": { + "chalk": { + "version": "1.1.3", + "from": "https://registry.npmjs.org/chalk/-/chalk-1.1.3.tgz", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-1.1.3.tgz", + "dependencies": { + "ansi-styles": { + "version": "2.2.1", + "from": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-2.2.1.tgz", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-2.2.1.tgz" + }, + "escape-string-regexp": { + "version": "1.0.5", + "from": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz", + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz" + }, + "has-ansi": { + "version": "2.0.0", + "from": "https://registry.npmjs.org/has-ansi/-/has-ansi-2.0.0.tgz", + "resolved": "https://registry.npmjs.org/has-ansi/-/has-ansi-2.0.0.tgz", + "dependencies": { + "ansi-regex": { + "version": "2.0.0", + "from": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-2.0.0.tgz", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-2.0.0.tgz" + } + } + }, + "strip-ansi": { + "version": "3.0.1", + "from": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-3.0.1.tgz", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-3.0.1.tgz", + "dependencies": { + "ansi-regex": { + "version": "2.0.0", + "from": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-2.0.0.tgz", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-2.0.0.tgz" + } + } + }, + "supports-color": { + "version": "2.0.0", + "from": "https://registry.npmjs.org/supports-color/-/supports-color-2.0.0.tgz", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-2.0.0.tgz" + } + } + }, + "commander": { + "version": "2.9.0", + "from": "https://registry.npmjs.org/commander/-/commander-2.9.0.tgz", + "resolved": "https://registry.npmjs.org/commander/-/commander-2.9.0.tgz", + "dependencies": { + "graceful-readlink": { + "version": "1.0.1", + "from": "https://registry.npmjs.org/graceful-readlink/-/graceful-readlink-1.0.1.tgz", + "resolved": "https://registry.npmjs.org/graceful-readlink/-/graceful-readlink-1.0.1.tgz" + } + } + }, + "is-my-json-valid": { + "version": "2.15.0", + "from": "https://registry.npmjs.org/is-my-json-valid/-/is-my-json-valid-2.15.0.tgz", + "resolved": "https://registry.npmjs.org/is-my-json-valid/-/is-my-json-valid-2.15.0.tgz", + "dependencies": { + "generate-function": { + "version": "2.0.0", + "from": "https://registry.npmjs.org/generate-function/-/generate-function-2.0.0.tgz", + "resolved": "https://registry.npmjs.org/generate-function/-/generate-function-2.0.0.tgz" + }, + "generate-object-property": { + "version": "1.2.0", + "from": "https://registry.npmjs.org/generate-object-property/-/generate-object-property-1.2.0.tgz", + "resolved": "https://registry.npmjs.org/generate-object-property/-/generate-object-property-1.2.0.tgz", + "dependencies": { + "is-property": { + "version": "1.0.2", + "from": "https://registry.npmjs.org/is-property/-/is-property-1.0.2.tgz", + "resolved": "https://registry.npmjs.org/is-property/-/is-property-1.0.2.tgz" + } + } + }, + "jsonpointer": { + "version": "4.0.0", + "from": "https://registry.npmjs.org/jsonpointer/-/jsonpointer-4.0.0.tgz", + "resolved": "https://registry.npmjs.org/jsonpointer/-/jsonpointer-4.0.0.tgz" + }, + "xtend": { + "version": "4.0.1", + "from": "https://registry.npmjs.org/xtend/-/xtend-4.0.1.tgz", + "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.1.tgz" + } + } + }, + "pinkie-promise": { + "version": "2.0.1", + "from": "https://registry.npmjs.org/pinkie-promise/-/pinkie-promise-2.0.1.tgz", + "resolved": "https://registry.npmjs.org/pinkie-promise/-/pinkie-promise-2.0.1.tgz", + "dependencies": { + "pinkie": { + "version": "2.0.4", + "from": "https://registry.npmjs.org/pinkie/-/pinkie-2.0.4.tgz", + "resolved": "https://registry.npmjs.org/pinkie/-/pinkie-2.0.4.tgz" + } + } + } + } + }, + "hawk": { + "version": "3.1.3", + "from": "https://registry.npmjs.org/hawk/-/hawk-3.1.3.tgz", + "resolved": "https://registry.npmjs.org/hawk/-/hawk-3.1.3.tgz", + "dependencies": { + "hoek": { + "version": "2.16.3", + "from": "https://registry.npmjs.org/hoek/-/hoek-2.16.3.tgz", + "resolved": "https://registry.npmjs.org/hoek/-/hoek-2.16.3.tgz" + }, + "boom": { + "version": "2.10.1", + "from": "https://registry.npmjs.org/boom/-/boom-2.10.1.tgz", + "resolved": "https://registry.npmjs.org/boom/-/boom-2.10.1.tgz" + }, + "cryptiles": { + "version": "2.0.5", + "from": "https://registry.npmjs.org/cryptiles/-/cryptiles-2.0.5.tgz", + "resolved": "https://registry.npmjs.org/cryptiles/-/cryptiles-2.0.5.tgz" + }, + "sntp": { + "version": "1.0.9", + "from": "https://registry.npmjs.org/sntp/-/sntp-1.0.9.tgz", + "resolved": "https://registry.npmjs.org/sntp/-/sntp-1.0.9.tgz" + } + } + }, + "http-signature": { + "version": "1.1.1", + "from": "https://registry.npmjs.org/http-signature/-/http-signature-1.1.1.tgz", + "resolved": "https://registry.npmjs.org/http-signature/-/http-signature-1.1.1.tgz", + "dependencies": { + "assert-plus": { + "version": "0.2.0", + "from": "https://registry.npmjs.org/assert-plus/-/assert-plus-0.2.0.tgz", + "resolved": "https://registry.npmjs.org/assert-plus/-/assert-plus-0.2.0.tgz" + }, + "jsprim": { + "version": "1.3.1", + "from": "https://registry.npmjs.org/jsprim/-/jsprim-1.3.1.tgz", + "resolved": "https://registry.npmjs.org/jsprim/-/jsprim-1.3.1.tgz", + "dependencies": { + "extsprintf": { + "version": "1.0.2", + "from": "https://registry.npmjs.org/extsprintf/-/extsprintf-1.0.2.tgz", + "resolved": "https://registry.npmjs.org/extsprintf/-/extsprintf-1.0.2.tgz" + }, + "json-schema": { + "version": "0.2.3", + "from": "https://registry.npmjs.org/json-schema/-/json-schema-0.2.3.tgz", + "resolved": "https://registry.npmjs.org/json-schema/-/json-schema-0.2.3.tgz" + }, + "verror": { + "version": "1.3.6", + "from": "https://registry.npmjs.org/verror/-/verror-1.3.6.tgz", + "resolved": "https://registry.npmjs.org/verror/-/verror-1.3.6.tgz" + } + } + }, + "sshpk": { + "version": "1.10.1", + "from": "https://registry.npmjs.org/sshpk/-/sshpk-1.10.1.tgz", + "resolved": "https://registry.npmjs.org/sshpk/-/sshpk-1.10.1.tgz", + "dependencies": { + "asn1": { + "version": "0.2.3", + "from": "https://registry.npmjs.org/asn1/-/asn1-0.2.3.tgz", + "resolved": "https://registry.npmjs.org/asn1/-/asn1-0.2.3.tgz" + }, + "assert-plus": { + "version": "1.0.0", + "from": "https://registry.npmjs.org/assert-plus/-/assert-plus-1.0.0.tgz", + "resolved": "https://registry.npmjs.org/assert-plus/-/assert-plus-1.0.0.tgz" + }, + "dashdash": { + "version": "1.14.0", + "from": "https://registry.npmjs.org/dashdash/-/dashdash-1.14.0.tgz", + "resolved": "https://registry.npmjs.org/dashdash/-/dashdash-1.14.0.tgz" + }, + "getpass": { + "version": "0.1.6", + "from": "https://registry.npmjs.org/getpass/-/getpass-0.1.6.tgz", + "resolved": "https://registry.npmjs.org/getpass/-/getpass-0.1.6.tgz" + }, + "jsbn": { + "version": "0.1.0", + "from": "https://registry.npmjs.org/jsbn/-/jsbn-0.1.0.tgz", + "resolved": "https://registry.npmjs.org/jsbn/-/jsbn-0.1.0.tgz" + }, + "tweetnacl": { + "version": "0.14.3", + "from": "https://registry.npmjs.org/tweetnacl/-/tweetnacl-0.14.3.tgz", + "resolved": "https://registry.npmjs.org/tweetnacl/-/tweetnacl-0.14.3.tgz" + }, + "jodid25519": { + "version": "1.0.2", + "from": "https://registry.npmjs.org/jodid25519/-/jodid25519-1.0.2.tgz", + "resolved": "https://registry.npmjs.org/jodid25519/-/jodid25519-1.0.2.tgz" + }, + "ecc-jsbn": { + "version": "0.1.1", + "from": "https://registry.npmjs.org/ecc-jsbn/-/ecc-jsbn-0.1.1.tgz", + "resolved": "https://registry.npmjs.org/ecc-jsbn/-/ecc-jsbn-0.1.1.tgz" + }, + "bcrypt-pbkdf": { + "version": "1.0.0", + "from": "https://registry.npmjs.org/bcrypt-pbkdf/-/bcrypt-pbkdf-1.0.0.tgz", + "resolved": "https://registry.npmjs.org/bcrypt-pbkdf/-/bcrypt-pbkdf-1.0.0.tgz" + } + } + } + } + }, + "is-typedarray": { + "version": "1.0.0", + "from": "https://registry.npmjs.org/is-typedarray/-/is-typedarray-1.0.0.tgz", + "resolved": "https://registry.npmjs.org/is-typedarray/-/is-typedarray-1.0.0.tgz" + }, + "isstream": { + "version": "0.1.2", + "from": "https://registry.npmjs.org/isstream/-/isstream-0.1.2.tgz", + "resolved": "https://registry.npmjs.org/isstream/-/isstream-0.1.2.tgz" }, "json-stringify-safe": { "version": "5.0.1", - "from": "json-stringify-safe@>=5.0.0 <5.1.0", + "from": "https://registry.npmjs.org/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz", "resolved": "https://registry.npmjs.org/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz" + }, + "mime-types": { + "version": "2.1.12", + "from": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.12.tgz", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.12.tgz", + "dependencies": { + "mime-db": { + "version": "1.24.0", + "from": "https://registry.npmjs.org/mime-db/-/mime-db-1.24.0.tgz", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.24.0.tgz" + } + } + }, + "oauth-sign": { + "version": "0.8.2", + "from": "https://registry.npmjs.org/oauth-sign/-/oauth-sign-0.8.2.tgz", + "resolved": "https://registry.npmjs.org/oauth-sign/-/oauth-sign-0.8.2.tgz" + }, + "stringstream": { + "version": "0.0.5", + "from": "https://registry.npmjs.org/stringstream/-/stringstream-0.0.5.tgz", + "resolved": "https://registry.npmjs.org/stringstream/-/stringstream-0.0.5.tgz" + }, + "tough-cookie": { + "version": "2.3.1", + "from": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-2.3.1.tgz", + "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-2.3.1.tgz" + }, + "tunnel-agent": { + "version": "0.4.3", + "from": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.4.3.tgz", + "resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.4.3.tgz" } } }, "step": { "version": "0.0.6", - "from": "step@>=0.0.5 <0.1.0", + "from": "https://registry.npmjs.org/step/-/step-0.0.6.tgz", "resolved": "https://registry.npmjs.org/step/-/step-0.0.6.tgz" }, "step-profiler": { "version": "0.3.0", - "from": "step-profiler@>=0.3.0 <0.4.0", + "from": "https://registry.npmjs.org/step-profiler/-/step-profiler-0.3.0.tgz", "resolved": "https://registry.npmjs.org/step-profiler/-/step-profiler-0.3.0.tgz" }, "topojson": { "version": "0.0.8", - "from": "topojson@0.0.8", + "from": "https://registry.npmjs.org/topojson/-/topojson-0.0.8.tgz", "resolved": "https://registry.npmjs.org/topojson/-/topojson-0.0.8.tgz", "dependencies": { "optimist": { "version": "0.3.5", - "from": "optimist@0.3.5", + "from": "https://registry.npmjs.org/optimist/-/optimist-0.3.5.tgz", "resolved": "https://registry.npmjs.org/optimist/-/optimist-0.3.5.tgz", "dependencies": { "wordwrap": { "version": "0.0.3", - "from": "wordwrap@>=0.0.2 <0.1.0", + "from": "https://registry.npmjs.org/wordwrap/-/wordwrap-0.0.3.tgz", "resolved": "https://registry.npmjs.org/wordwrap/-/wordwrap-0.0.3.tgz" } } @@ -279,8 +1150,338 @@ }, "underscore": { "version": "1.6.0", - "from": "underscore@>=1.6.0 <1.7.0", + "from": "https://registry.npmjs.org/underscore/-/underscore-1.6.0.tgz", "resolved": "https://registry.npmjs.org/underscore/-/underscore-1.6.0.tgz" + }, + "yargs": { + "version": "5.0.0", + "from": "https://registry.npmjs.org/yargs/-/yargs-5.0.0.tgz", + "resolved": "https://registry.npmjs.org/yargs/-/yargs-5.0.0.tgz", + "dependencies": { + "cliui": { + "version": "3.2.0", + "from": "https://registry.npmjs.org/cliui/-/cliui-3.2.0.tgz", + "resolved": "https://registry.npmjs.org/cliui/-/cliui-3.2.0.tgz", + "dependencies": { + "strip-ansi": { + "version": "3.0.1", + "from": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-3.0.1.tgz", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-3.0.1.tgz", + "dependencies": { + "ansi-regex": { + "version": "2.0.0", + "from": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-2.0.0.tgz", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-2.0.0.tgz" + } + } + }, + "wrap-ansi": { + "version": "2.0.0", + "from": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-2.0.0.tgz", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-2.0.0.tgz" + } + } + }, + "decamelize": { + "version": "1.2.0", + "from": "https://registry.npmjs.org/decamelize/-/decamelize-1.2.0.tgz", + "resolved": "https://registry.npmjs.org/decamelize/-/decamelize-1.2.0.tgz" + }, + "get-caller-file": { + "version": "1.0.2", + "from": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-1.0.2.tgz", + "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-1.0.2.tgz" + }, + "lodash.assign": { + "version": "4.2.0", + "from": "https://registry.npmjs.org/lodash.assign/-/lodash.assign-4.2.0.tgz", + "resolved": "https://registry.npmjs.org/lodash.assign/-/lodash.assign-4.2.0.tgz" + }, + "os-locale": { + "version": "1.4.0", + "from": "https://registry.npmjs.org/os-locale/-/os-locale-1.4.0.tgz", + "resolved": "https://registry.npmjs.org/os-locale/-/os-locale-1.4.0.tgz", + "dependencies": { + "lcid": { + "version": "1.0.0", + "from": "https://registry.npmjs.org/lcid/-/lcid-1.0.0.tgz", + "resolved": "https://registry.npmjs.org/lcid/-/lcid-1.0.0.tgz", + "dependencies": { + "invert-kv": { + "version": "1.0.0", + "from": "https://registry.npmjs.org/invert-kv/-/invert-kv-1.0.0.tgz", + "resolved": "https://registry.npmjs.org/invert-kv/-/invert-kv-1.0.0.tgz" + } + } + } + } + }, + "read-pkg-up": { + "version": "1.0.1", + "from": "https://registry.npmjs.org/read-pkg-up/-/read-pkg-up-1.0.1.tgz", + "resolved": "https://registry.npmjs.org/read-pkg-up/-/read-pkg-up-1.0.1.tgz", + "dependencies": { + "find-up": { + "version": "1.1.2", + "from": "https://registry.npmjs.org/find-up/-/find-up-1.1.2.tgz", + "resolved": "https://registry.npmjs.org/find-up/-/find-up-1.1.2.tgz", + "dependencies": { + "path-exists": { + "version": "2.1.0", + "from": "https://registry.npmjs.org/path-exists/-/path-exists-2.1.0.tgz", + "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-2.1.0.tgz" + }, + "pinkie-promise": { + "version": "2.0.1", + "from": "https://registry.npmjs.org/pinkie-promise/-/pinkie-promise-2.0.1.tgz", + "resolved": "https://registry.npmjs.org/pinkie-promise/-/pinkie-promise-2.0.1.tgz", + "dependencies": { + "pinkie": { + "version": "2.0.4", + "from": "https://registry.npmjs.org/pinkie/-/pinkie-2.0.4.tgz", + "resolved": "https://registry.npmjs.org/pinkie/-/pinkie-2.0.4.tgz" + } + } + } + } + }, + "read-pkg": { + "version": "1.1.0", + "from": "https://registry.npmjs.org/read-pkg/-/read-pkg-1.1.0.tgz", + "resolved": "https://registry.npmjs.org/read-pkg/-/read-pkg-1.1.0.tgz", + "dependencies": { + "load-json-file": { + "version": "1.1.0", + "from": "https://registry.npmjs.org/load-json-file/-/load-json-file-1.1.0.tgz", + "resolved": "https://registry.npmjs.org/load-json-file/-/load-json-file-1.1.0.tgz", + "dependencies": { + "graceful-fs": { + "version": "4.1.9", + "from": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.1.9.tgz", + "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.1.9.tgz" + }, + "parse-json": { + "version": "2.2.0", + "from": "https://registry.npmjs.org/parse-json/-/parse-json-2.2.0.tgz", + "resolved": "https://registry.npmjs.org/parse-json/-/parse-json-2.2.0.tgz", + "dependencies": { + "error-ex": { + "version": "1.3.0", + "from": "https://registry.npmjs.org/error-ex/-/error-ex-1.3.0.tgz", + "resolved": "https://registry.npmjs.org/error-ex/-/error-ex-1.3.0.tgz", + "dependencies": { + "is-arrayish": { + "version": "0.2.1", + "from": "https://registry.npmjs.org/is-arrayish/-/is-arrayish-0.2.1.tgz", + "resolved": "https://registry.npmjs.org/is-arrayish/-/is-arrayish-0.2.1.tgz" + } + } + } + } + }, + "pify": { + "version": "2.3.0", + "from": "https://registry.npmjs.org/pify/-/pify-2.3.0.tgz", + "resolved": "https://registry.npmjs.org/pify/-/pify-2.3.0.tgz" + }, + "pinkie-promise": { + "version": "2.0.1", + "from": "https://registry.npmjs.org/pinkie-promise/-/pinkie-promise-2.0.1.tgz", + "resolved": "https://registry.npmjs.org/pinkie-promise/-/pinkie-promise-2.0.1.tgz", + "dependencies": { + "pinkie": { + "version": "2.0.4", + "from": "https://registry.npmjs.org/pinkie/-/pinkie-2.0.4.tgz", + "resolved": "https://registry.npmjs.org/pinkie/-/pinkie-2.0.4.tgz" + } + } + }, + "strip-bom": { + "version": "2.0.0", + "from": "https://registry.npmjs.org/strip-bom/-/strip-bom-2.0.0.tgz", + "resolved": "https://registry.npmjs.org/strip-bom/-/strip-bom-2.0.0.tgz", + "dependencies": { + "is-utf8": { + "version": "0.2.1", + "from": "https://registry.npmjs.org/is-utf8/-/is-utf8-0.2.1.tgz", + "resolved": "https://registry.npmjs.org/is-utf8/-/is-utf8-0.2.1.tgz" + } + } + } + } + }, + "normalize-package-data": { + "version": "2.3.5", + "from": "https://registry.npmjs.org/normalize-package-data/-/normalize-package-data-2.3.5.tgz", + "resolved": "https://registry.npmjs.org/normalize-package-data/-/normalize-package-data-2.3.5.tgz", + "dependencies": { + "hosted-git-info": { + "version": "2.1.5", + "from": "https://registry.npmjs.org/hosted-git-info/-/hosted-git-info-2.1.5.tgz", + "resolved": "https://registry.npmjs.org/hosted-git-info/-/hosted-git-info-2.1.5.tgz" + }, + "is-builtin-module": { + "version": "1.0.0", + "from": "https://registry.npmjs.org/is-builtin-module/-/is-builtin-module-1.0.0.tgz", + "resolved": "https://registry.npmjs.org/is-builtin-module/-/is-builtin-module-1.0.0.tgz", + "dependencies": { + "builtin-modules": { + "version": "1.1.1", + "from": "https://registry.npmjs.org/builtin-modules/-/builtin-modules-1.1.1.tgz", + "resolved": "https://registry.npmjs.org/builtin-modules/-/builtin-modules-1.1.1.tgz" + } + } + }, + "semver": { + "version": "5.3.0", + "from": "https://registry.npmjs.org/semver/-/semver-5.3.0.tgz", + "resolved": "https://registry.npmjs.org/semver/-/semver-5.3.0.tgz" + }, + "validate-npm-package-license": { + "version": "3.0.1", + "from": "https://registry.npmjs.org/validate-npm-package-license/-/validate-npm-package-license-3.0.1.tgz", + "resolved": "https://registry.npmjs.org/validate-npm-package-license/-/validate-npm-package-license-3.0.1.tgz", + "dependencies": { + "spdx-correct": { + "version": "1.0.2", + "from": "https://registry.npmjs.org/spdx-correct/-/spdx-correct-1.0.2.tgz", + "resolved": "https://registry.npmjs.org/spdx-correct/-/spdx-correct-1.0.2.tgz", + "dependencies": { + "spdx-license-ids": { + "version": "1.2.2", + "from": "https://registry.npmjs.org/spdx-license-ids/-/spdx-license-ids-1.2.2.tgz", + "resolved": "https://registry.npmjs.org/spdx-license-ids/-/spdx-license-ids-1.2.2.tgz" + } + } + }, + "spdx-expression-parse": { + "version": "1.0.4", + "from": "https://registry.npmjs.org/spdx-expression-parse/-/spdx-expression-parse-1.0.4.tgz", + "resolved": "https://registry.npmjs.org/spdx-expression-parse/-/spdx-expression-parse-1.0.4.tgz" + } + } + } + } + }, + "path-type": { + "version": "1.1.0", + "from": "https://registry.npmjs.org/path-type/-/path-type-1.1.0.tgz", + "resolved": "https://registry.npmjs.org/path-type/-/path-type-1.1.0.tgz", + "dependencies": { + "graceful-fs": { + "version": "4.1.9", + "from": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.1.9.tgz", + "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.1.9.tgz" + }, + "pify": { + "version": "2.3.0", + "from": "https://registry.npmjs.org/pify/-/pify-2.3.0.tgz", + "resolved": "https://registry.npmjs.org/pify/-/pify-2.3.0.tgz" + }, + "pinkie-promise": { + "version": "2.0.1", + "from": "https://registry.npmjs.org/pinkie-promise/-/pinkie-promise-2.0.1.tgz", + "resolved": "https://registry.npmjs.org/pinkie-promise/-/pinkie-promise-2.0.1.tgz", + "dependencies": { + "pinkie": { + "version": "2.0.4", + "from": "https://registry.npmjs.org/pinkie/-/pinkie-2.0.4.tgz", + "resolved": "https://registry.npmjs.org/pinkie/-/pinkie-2.0.4.tgz" + } + } + } + } + } + } + } + } + }, + "require-directory": { + "version": "2.1.1", + "from": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", + "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz" + }, + "require-main-filename": { + "version": "1.0.1", + "from": "https://registry.npmjs.org/require-main-filename/-/require-main-filename-1.0.1.tgz", + "resolved": "https://registry.npmjs.org/require-main-filename/-/require-main-filename-1.0.1.tgz" + }, + "set-blocking": { + "version": "2.0.0", + "from": "https://registry.npmjs.org/set-blocking/-/set-blocking-2.0.0.tgz", + "resolved": "https://registry.npmjs.org/set-blocking/-/set-blocking-2.0.0.tgz" + }, + "string-width": { + "version": "1.0.2", + "from": "https://registry.npmjs.org/string-width/-/string-width-1.0.2.tgz", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-1.0.2.tgz", + "dependencies": { + "code-point-at": { + "version": "1.0.1", + "from": "https://registry.npmjs.org/code-point-at/-/code-point-at-1.0.1.tgz", + "resolved": "https://registry.npmjs.org/code-point-at/-/code-point-at-1.0.1.tgz", + "dependencies": { + "number-is-nan": { + "version": "1.0.1", + "from": "https://registry.npmjs.org/number-is-nan/-/number-is-nan-1.0.1.tgz", + "resolved": "https://registry.npmjs.org/number-is-nan/-/number-is-nan-1.0.1.tgz" + } + } + }, + "is-fullwidth-code-point": { + "version": "1.0.0", + "from": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-1.0.0.tgz", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-1.0.0.tgz", + "dependencies": { + "number-is-nan": { + "version": "1.0.1", + "from": "https://registry.npmjs.org/number-is-nan/-/number-is-nan-1.0.1.tgz", + "resolved": "https://registry.npmjs.org/number-is-nan/-/number-is-nan-1.0.1.tgz" + } + } + }, + "strip-ansi": { + "version": "3.0.1", + "from": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-3.0.1.tgz", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-3.0.1.tgz", + "dependencies": { + "ansi-regex": { + "version": "2.0.0", + "from": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-2.0.0.tgz", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-2.0.0.tgz" + } + } + } + } + }, + "which-module": { + "version": "1.0.0", + "from": "https://registry.npmjs.org/which-module/-/which-module-1.0.0.tgz", + "resolved": "https://registry.npmjs.org/which-module/-/which-module-1.0.0.tgz" + }, + "window-size": { + "version": "0.2.0", + "from": "https://registry.npmjs.org/window-size/-/window-size-0.2.0.tgz", + "resolved": "https://registry.npmjs.org/window-size/-/window-size-0.2.0.tgz" + }, + "y18n": { + "version": "3.2.1", + "from": "https://registry.npmjs.org/y18n/-/y18n-3.2.1.tgz", + "resolved": "https://registry.npmjs.org/y18n/-/y18n-3.2.1.tgz" + }, + "yargs-parser": { + "version": "3.2.0", + "from": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-3.2.0.tgz", + "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-3.2.0.tgz", + "dependencies": { + "camelcase": { + "version": "3.0.0", + "from": "https://registry.npmjs.org/camelcase/-/camelcase-3.0.0.tgz", + "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-3.0.0.tgz" + } + } + } + } } } } diff --git a/package.json b/package.json index fc9eda7b0..dc7ebdf69 100644 --- a/package.json +++ b/package.json @@ -1,58 +1,62 @@ { - "private": true, - "name": "cartodb_sql_api", - "description": "high speed SQL api for cartodb", - "keywords": [ - "cartodb" - ], - "version": "1.29.1", - "repository": { - "type": "git", - "url": "git://github.com/CartoDB/CartoDB-SQL-API.git" - }, - "license": "BSD-3-Clause", - "author": "Vizzuality (http://vizzuality.com)", - "contributors": [ - "Simon Tokumine ", - "Sandro Santilli " - ], - "dependencies": { - "cartodb-psql": "~0.6.0", - "cartodb-redis": "~0.11.0", - "debug": "2.2.0", - "express": "~2.5.11", - "log4js": "https://github.com/CartoDB/log4js-node/tarball/cdb", - "lru-cache": "~2.5.0", - "node-statsd": "~0.0.7", - "node-uuid": "^1.4.7", - "oauth-client": "0.3.0", - "redis": "^2.4.2", - "rollbar": "~0.3.2", - "step": "~0.0.5", - "step-profiler": "~0.3.0", - "topojson": "0.0.8", - "underscore": "~1.6.0", - "queue-async": "~1.0.7", - "cartodb-query-tables": "0.1.0" - }, - "devDependencies": { - "istanbul": "~0.4.2", - "request": "~2.60.0", - "shapefile": "0.3.0", - "mocha": "~1.21.4", - "jshint": "~2.6.0", - "zipfile": "~0.5.0", - "libxmljs": "~0.8.1", - "qs": "6.2.0", - "sqlite3": "~3.0.8" - }, - "scripts": { - "test": "make test-all" , - "test:unit": "mocha test/unit/**/*.js", - "test:unit:watch": "npm run test:unit -- -w" - }, - "engines": { - "node": ">=0.8 <0.11", - "npm": ">=1.2.1" - } + "private": true, + "name": "cartodb_sql_api", + "description": "high speed SQL api for cartodb", + "keywords": [ + "cartodb" + ], + "version": "1.42.3", + "repository": { + "type": "git", + "url": "git://github.com/CartoDB/CartoDB-SQL-API.git" + }, + "license": "BSD-3-Clause", + "author": "Vizzuality (http://vizzuality.com)", + "contributors": [ + "Simon Tokumine ", + "Sandro Santilli " + ], + "dependencies": { + "bintrees": "1.0.1", + "bunyan": "1.8.1", + "cartodb-psql": "~0.6.0", + "cartodb-query-tables": "0.2.0", + "cartodb-redis": "0.13.1", + "debug": "2.2.0", + "express": "~4.13.3", + "log4js": "cartodb/log4js-node#cdb", + "lru-cache": "~2.5.0", + "multer": "~1.2.0", + "node-statsd": "~0.0.7", + "node-uuid": "^1.4.7", + "oauth-client": "0.3.0", + "qs": "~6.2.1", + "queue-async": "~1.0.7", + "redis-mpool": "0.4.0", + "redlock": "2.0.1", + "request": "~2.75.0", + "step": "~0.0.5", + "step-profiler": "~0.3.0", + "topojson": "0.0.8", + "underscore": "~1.6.0", + "yargs": "~5.0.0" + }, + "devDependencies": { + "istanbul": "~0.4.2", + "shapefile": "0.3.0", + "mocha": "~1.21.4", + "jshint": "~2.6.0", + "zipfile": "~0.5.0", + "libxmljs": "~0.8.1", + "sqlite3": "~3.0.8" + }, + "scripts": { + "test": "make test-all", + "test:unit": "mocha test/unit/**/*.js", + "test:unit:watch": "npm run test:unit -- -w" + }, + "engines": { + "node": ">=0.8 <0.11", + "npm": ">=1.2.1" + } } diff --git a/test/acceptance/app.auth.test.js b/test/acceptance/app.auth.test.js index 8418ffa99..1f9902759 100644 --- a/test/acceptance/app.auth.test.js +++ b/test/acceptance/app.auth.test.js @@ -1,6 +1,6 @@ require('../helper'); -var app = require(global.settings.app_root + '/app/app')(); +var server = require('../../app/server')(); var assert = require('../support/assert'); describe('app.auth', function() { @@ -40,7 +40,7 @@ describe('app.auth', function() { scenarios.forEach(function(scenario) { it(scenario.desc, function(done) { - assert.response(app, { + assert.response(server, { // view prepare_db.sh to find public table name and structure url: scenario.url, headers: { @@ -49,7 +49,7 @@ describe('app.auth', function() { method: 'GET' }, {}, - function(res) { + function(err, res) { assert.equal(res.statusCode, scenario.statusCode, res.statusCode + ': ' + res.body); done(); } diff --git a/test/acceptance/app.test.js b/test/acceptance/app.test.js index 85a2723c3..4d82e5952 100644 --- a/test/acceptance/app.test.js +++ b/test/acceptance/app.test.js @@ -14,7 +14,7 @@ */ require('../helper'); -var app = require(global.settings.app_root + '/app/app')(); +var server = require('../../app/server')(); var assert = require('../support/assert'); var querystring = require('querystring'); var _ = require('underscore'); @@ -23,15 +23,19 @@ var step = require('step'); describe('app.test', function() { + var RESPONSE_OK = { + statusCode: 200 + }; + var expected_cache_control = 'no-cache,max-age=31536000,must-revalidate,public'; var expected_rw_cache_control = 'no-cache,max-age=0,must-revalidate,public'; var expected_cache_control_persist = 'public,max-age=31536000'; it('GET /api/v1/version', function(done){ - assert.response(app, { + assert.response(server, { url: '/api/v1/version', method: 'GET' - },{}, function(res) { + },{}, function(err, res) { assert.equal(res.statusCode, 200); var parsed = JSON.parse(res.body); var sqlapi_version = require(__dirname + '/../../package.json').version; @@ -42,12 +46,12 @@ it('GET /api/v1/version', function(done){ }); it('GET /api/v1/sql', function(done){ - assert.response(app, { + assert.response(server, { url: '/api/v1/sql', method: 'GET' },{ status: 400 - }, function(res) { + }, function(err, res) { assert.deepEqual(res.headers['content-type'], 'application/json; charset=utf-8'); assert.deepEqual(res.headers['content-disposition'], 'inline'); assert.deepEqual(JSON.parse(res.body), {"error":["You must indicate a sql query"]}); @@ -57,12 +61,12 @@ it('GET /api/v1/sql', function(done){ // Test base_url setting it('GET /api/whatever/sql', function(done){ - assert.response(app, { + assert.response(server, { url: '/api/whatever/sql?q=SELECT%201', headers: {host: 'vizzuality.cartodb.com'}, method: 'GET' },{ - }, function(res) { + }, function(err, res) { assert.equal(res.statusCode, 200, res.body); done(); }); @@ -70,12 +74,12 @@ it('GET /api/whatever/sql', function(done){ // Test CORS headers with GET it('GET /api/whatever/sql', function(done){ - assert.response(app, { + assert.response(server, { url: '/api/whatever/sql?q=SELECT%201', headers: {host: 'vizzuality.cartodb.com'}, method: 'GET' },{ - }, function(res) { + }, function(err, res) { assert.equal(res.statusCode, 200, res.body); assert.equal( res.headers['access-control-allow-headers'], 'X-Requested-With, X-Prototype-Version, X-CSRF-Token' @@ -87,11 +91,11 @@ it('GET /api/whatever/sql', function(done){ // Test that OPTIONS does not run queries it('OPTIONS /api/x/sql', function(done){ - assert.response(app, { + assert.response(server, { url: '/api/x/sql?q=syntax%20error', headers: {host: 'vizzuality.cartodb.com'}, method: 'OPTIONS' - },{}, function(res) { + },{}, function(err, res) { assert.equal(res.statusCode, 200, res.body); assert.equal(res.body, ''); assert.equal( @@ -105,11 +109,11 @@ it('OPTIONS /api/x/sql', function(done){ it('GET /api/v1/sql with SQL parameter on SELECT only. No oAuth included ', function(done){ - assert.response(app, { + assert.response(server, { url: '/api/v1/sql?q=SELECT%20*%20FROM%20untitle_table_4&database=cartodb_test_user_1_db', headers: {host: 'vizzuality.cartodb.com'}, method: 'GET' - },{ }, function(res) { + },{ }, function(err, res) { assert.equal(res.statusCode, 200, res.body); // Check cache headers assert.equal(res.headers['x-cache-channel'], 'cartodb_test_user_1_db:public.untitle_table_4'); @@ -119,11 +123,11 @@ it('GET /api/v1/sql with SQL parameter on SELECT only. No oAuth included ', func }); it('cache_policy=persist', function(done){ - assert.response(app, { + assert.response(server, { url: '/api/v1/sql?q=SELECT%20*%20FROM%20untitle_table_4&database=cartodb_test_user_1_db&cache_policy=persist', headers: {host: 'vizzuality.cartodb.com'}, method: 'GET' - },{ }, function(res) { + },{ }, function(err, res) { assert.equal(res.statusCode, 200, res.body); // Check cache headers assert.ok(res.headers.hasOwnProperty('x-cache-channel')); @@ -135,21 +139,21 @@ it('cache_policy=persist', function(done){ }); it('GET /api/v1/sql with SQL parameter on SELECT only. no database param, just id using headers', function(done){ - assert.response(app, { + assert.response(server, { url: '/api/v1/sql?q=SELECT%20*%20FROM%20untitle_table_4', headers: {host: 'vizzuality.cartodb.com'}, method: 'GET' - },{ }, function(res) { + },{ }, function(err, res) { assert.equal(res.statusCode, 200, res.body); done(); }); }); it('GET /user/vizzuality/api/v1/sql with SQL parameter on SELECT only', function(done){ - assert.response(app, { + assert.response(server, { url: '/user/vizzuality/api/v1/sql?q=SELECT%20*%20FROM%20untitle_table_4', method: 'GET' - },{ }, function(res) { + },{ }, function(err, res) { assert.equal(res.statusCode, 200, res.body); done(); }); @@ -160,22 +164,20 @@ it('GET /user/vizzuality/api/v1/sql with SQL parameter on SELECT only', function it('SELECT from user-specific database', function(done){ var backupDBHost = global.settings.db_host; global.settings.db_host = '6.6.6.6'; - assert.response(app, { + assert.response(server, { url: '/api/v1/sql?q=SELECT+2+as+n', headers: {host: 'cartodb250user.cartodb.com'}, method: 'GET' - },{}, function(res) { + }, RESPONSE_OK, function(err, res) { global.settings.db_host = backupDBHost; - var err = null; try { - assert.equal(res.statusCode, 200, res.statusCode + ": " + res.body); - var parsed = JSON.parse(res.body); - assert.equal(parsed.rows.length, 1); - assert.equal(parsed.rows[0].n, 2); + var parsed = JSON.parse(res.body); + assert.equal(parsed.rows.length, 1); + assert.equal(parsed.rows[0].n, 2); } catch (e) { - err = e; + return done(e); } - done(err); + done(); }); }); @@ -183,32 +185,31 @@ it('SELECT from user-specific database', function(done){ it('SELECT with user-specific password', function(done){ var backupDBUserPass = global.settings.db_user_pass; global.settings.db_user_pass = '<%= user_password %>'; - assert.response(app, { + assert.response(server, { url: '/api/v1/sql?q=SELECT+2+as+n&api_key=1234', headers: {host: 'cartodb250user.cartodb.com'}, method: 'GET' - },{}, function(res) { + }, RESPONSE_OK, function(err, res) { global.settings.db_user_pass = backupDBUserPass; - var err = null; try { assert.equal(res.statusCode, 200, res.statusCode + ": " + res.body); var parsed = JSON.parse(res.body); assert.equal(parsed.rows.length, 1); assert.equal(parsed.rows[0].n, 2); } catch (e) { - err = e; + return done(e); } - done(err); + return done(); }); }); it('GET /api/v1/sql with SQL parameter on SELECT only. no database param, just id using headers. Authenticated.', function(done){ - assert.response(app, { + assert.response(server, { url: '/api/v1/sql?q=SELECT%20cartodb_id*2%20FROM%20untitle_table_4&api_key=1234', headers: {host: 'vizzuality.cartodb.com'}, method: 'GET' - },{ }, function(res) { + },{ }, function(err, res) { assert.equal(res.statusCode, 200, res.body); // Check cache headers assert.equal(res.headers['x-cache-channel'], 'cartodb_test_user_1_db:public.untitle_table_4'); @@ -220,7 +221,7 @@ function(done){ // Test for https://github.com/Vizzuality/CartoDB-SQL-API/issues/85 it("paging doesn't break x-cache-channel", function(done){ - assert.response(app, { + assert.response(server, { url: '/api/v1/sql?' + querystring.stringify({ // note: select casing intentionally mixed q: 'selECT cartodb_id*3 FROM untitle_table_4', @@ -230,7 +231,7 @@ function(done){ }), headers: {host: 'vizzuality.cartodb.com'}, method: 'GET' - },{ }, function(res) { + },{ }, function(err, res) { assert.equal(res.statusCode, 200, res.body); assert.equal(res.headers['x-cache-channel'], 'cartodb_test_user_1_db:public.untitle_table_4'); var parsed = JSON.parse(res.body); @@ -290,7 +291,7 @@ it("paging", function(done){ req.headers['Content-Type'] = 'application/x-www-form-urlencoded'; req.data = data; } - assert.response(app, req, {}, function(res) { + assert.response(server, req, {}, function(err, res) { assert.equal(res.statusCode, 200, res.body); var parsed = JSON.parse(res.body); assert.equal(parsed.rows.length, nrows); @@ -311,7 +312,7 @@ it("paging starting with comment", function(done){ "SELECT * FROM (VALUES(1),(2),(3),(4),(5),(6),(7),(8),(9)) t(v)"; var nrows = 3; var page = 2; - assert.response(app, { + assert.response(server, { url: '/api/v1/sql?' + querystring.stringify({ q: sql, rows_per_page: nrows, @@ -319,7 +320,7 @@ it("paging starting with comment", function(done){ }), headers: {host: 'vizzuality.cartodb.com'}, method: 'GET' - }, {}, function(res) { + }, {}, function(err, res) { assert.equal(res.statusCode, 200, res.body); var parsed = JSON.parse(res.body); assert.equal(parsed.rows.length, 3); @@ -333,25 +334,25 @@ it("paging starting with comment", function(done){ }); it('POST /api/v1/sql with SQL parameter on SELECT only. no database param, just id using headers', function(done){ - assert.response(app, { + assert.response(server, { url: '/api/v1/sql', data: querystring.stringify({q: "SELECT * FROM untitle_table_4"}), headers: {host: 'vizzuality.cartodb.com', 'Content-Type': 'application/x-www-form-urlencoded' }, method: 'POST' - },{ }, function(res) { + },{ }, function(err, res) { assert.equal(res.statusCode, 200, res.body); done(); }); }); it('GET /api/v1/sql with INSERT. oAuth not used, so public user - should fail', function(done){ - assert.response(app, { + assert.response(server, { url: "/api/v1/sql?q=INSERT%20INTO%20untitle_table_4%20(cartodb_id)%20VALUES%20(1e4)" + "&database=cartodb_test_user_1_db", headers: {host: 'vizzuality.cartodb.com'}, method: 'GET' },{ - }, function(res) { + }, function(err, res) { assert.equal(res.statusCode, 401, res.statusCode + ': ' + res.body); assert.deepEqual(res.headers['content-type'], 'application/json; charset=utf-8'); assert.deepEqual(res.headers['content-disposition'], 'inline'); @@ -363,12 +364,12 @@ it('GET /api/v1/sql with INSERT. oAuth not used, so public user - should fail', }); it('GET /api/v1/sql with DROP TABLE. oAuth not used, so public user - should fail', function(done){ - assert.response(app, { + assert.response(server, { url: "/api/v1/sql?q=DROP%20TABLE%20untitle_table_4&database=cartodb_test_user_1_db", headers: {host: 'vizzuality.cartodb.com'}, method: 'GET' },{ - }, function(res) { + }, function(err, res) { assert.equal(res.statusCode, 400, res.statusCode + ': ' + res.body); assert.deepEqual(res.headers['content-type'], 'application/json; charset=utf-8'); assert.deepEqual(res.headers['content-disposition'], 'inline'); @@ -380,29 +381,27 @@ it('GET /api/v1/sql with DROP TABLE. oAuth not used, so public user - should fai }); it('GET /api/v1/sql with INSERT. header based db - should fail', function (done) { - assert.response(app, { + assert.response(server, { url: "/api/v1/sql?q=INSERT%20INTO%20untitle_table_4%20(id)%20VALUES%20(1)", headers: {host: 'vizzuality.cartodb.com'}, method: 'GET' }, { status: 400 - }, function (res, err) { - done(err); - }); + }, done); }); // Check results from INSERT // // See https://github.com/Vizzuality/CartoDB-SQL-API/issues/13 it('INSERT returns affected rows', function(done){ - assert.response(app, { + assert.response(server, { // view prepare_db.sh to see where to set api_key url: "/api/v1/sql?api_key=1234&" + querystring.stringify({q: "INSERT INTO private_table(name) VALUES('noret1') UNION VALUES('noret2')" }), headers: {host: 'vizzuality.localhost.lan:8080' }, method: 'GET' - },{}, function(res) { + },{}, function(err, res) { assert.equal(res.statusCode, 200, res.statusCode + ': ' + res.body); var out = JSON.parse(res.body); assert.ok(out.hasOwnProperty('time')); @@ -420,14 +419,14 @@ it('INSERT returns affected rows', function(done){ // // See https://github.com/Vizzuality/CartoDB-SQL-API/issues/13 it('UPDATE returns affected rows', function(done){ - assert.response(app, { + assert.response(server, { // view prepare_db.sh to see where to set api_key url: "/api/v1/sql?api_key=1234&" + querystring.stringify({q: "UPDATE private_table SET name = upper(name) WHERE name in ('noret1', 'noret2')" }), headers: {host: 'vizzuality.localhost.lan:8080' }, method: 'GET' - },{}, function(res) { + },{}, function(err, res) { assert.equal(res.statusCode, 200, res.statusCode + ': ' + res.body); var out = JSON.parse(res.body); assert.ok(out.hasOwnProperty('time')); @@ -445,14 +444,14 @@ it('UPDATE returns affected rows', function(done){ // // See https://github.com/Vizzuality/CartoDB-SQL-API/issues/13 it('DELETE returns affected rows', function(done){ - assert.response(app, { + assert.response(server, { // view prepare_db.sh to see where to set api_key url: "/api/v1/sql?api_key=1234&" + querystring.stringify({q: "DELETE FROM private_table WHERE name in ('NORET1', 'NORET2')" }), headers: {host: 'vizzuality.localhost.lan:8080' }, method: 'GET' - },{}, function(res) { + },{}, function(err, res) { assert.equal(res.statusCode, 200, res.statusCode + ': ' + res.body); var out = JSON.parse(res.body); assert.ok(out.hasOwnProperty('time')); @@ -470,14 +469,14 @@ it('DELETE returns affected rows', function(done){ // // See https://github.com/Vizzuality/CartoDB-SQL-API/issues/50 it('INSERT with RETURNING returns all results', function(done){ - assert.response(app, { + assert.response(server, { // view prepare_db.sh to see where to set api_key url: "/api/v1/sql?api_key=1234&" + querystring.stringify({q: "INSERT INTO private_table(name) VALUES('test') RETURNING upper(name), reverse(name)" }), headers: {host: 'vizzuality.localhost.lan:8080' }, method: 'GET' - },{}, function(res) { + },{}, function(err, res) { assert.equal(res.statusCode, 200, res.statusCode + ': ' + res.body); var out = JSON.parse(res.body); assert.ok(out.hasOwnProperty('time')); @@ -494,14 +493,14 @@ it('INSERT with RETURNING returns all results', function(done){ // // See https://github.com/Vizzuality/CartoDB-SQL-API/issues/50 it('UPDATE with RETURNING returns all results', function(done){ - assert.response(app, { + assert.response(server, { // view prepare_db.sh to see where to set api_key url: "/api/v1/sql?api_key=1234&" + querystring.stringify({q: "UPDATE private_table SET name = 'tost' WHERE name = 'test' RETURNING upper(name), reverse(name)" }), headers: {host: 'vizzuality.localhost.lan:8080' }, method: 'GET' - },{}, function(res) { + },{}, function(err, res) { assert.equal(res.statusCode, 200, res.statusCode + ': ' + res.body); var out = JSON.parse(res.body); assert.ok(out.hasOwnProperty('time')); @@ -518,14 +517,14 @@ it('UPDATE with RETURNING returns all results', function(done){ // // See https://github.com/Vizzuality/CartoDB-SQL-API/issues/50 it('DELETE with RETURNING returns all results', function(done){ - assert.response(app, { + assert.response(server, { // view prepare_db.sh to see where to set api_key url: "/api/v1/sql?api_key=1234&" + querystring.stringify({q: "DELETE FROM private_table WHERE name = 'tost' RETURNING name" }), headers: {host: 'vizzuality.localhost.lan:8080' }, method: 'GET' - },{}, function(res) { + },{}, function(err, res) { assert.equal(res.statusCode, 200, res.statusCode + ': ' + res.body); var out = JSON.parse(res.body); assert.ok(out.hasOwnProperty('time')); @@ -538,11 +537,11 @@ it('DELETE with RETURNING returns all results', function(done){ }); it('GET /api/v1/sql with SQL parameter on DROP TABLE. should fail', function(done){ - assert.response(app, { + assert.response(server, { url: "/api/v1/sql?q=DROP%20TABLE%20untitle_table_4", headers: {host: 'vizzuality.cartodb.com'}, method: 'GET' - },{}, function(res) { + },{}, function(err, res) { assert.equal(res.statusCode, 400, res.statusCode + ': ' + res.body); assert.deepEqual(res.headers['content-type'], 'application/json; charset=utf-8'); assert.deepEqual(res.headers['content-disposition'], 'inline'); @@ -557,14 +556,14 @@ it('GET /api/v1/sql with SQL parameter on DROP TABLE. should fail', function(don // // See https://github.com/Vizzuality/CartoDB-SQL-API/issues/99 it('Field name is not confused with UPDATE operation', function(done){ - assert.response(app, { + assert.response(server, { // view prepare_db.sh to see where to set api_key url: "/api/v1/sql?api_key=1234&" + querystring.stringify({q: "SELECT min(updated_at) FROM private_table" }), headers: {host: 'vizzuality.localhost.lan:8080' }, method: 'GET' - },{}, function(res) { + },{}, function(err, res) { assert.equal(res.statusCode, 200, res.statusCode + ': ' + res.body); assert.equal(res.headers['x-cache-channel'], 'cartodb_test_user_1_db:public.private_table'); done(); @@ -572,14 +571,14 @@ it('Field name is not confused with UPDATE operation', function(done){ }); it('CREATE TABLE with GET and auth', function(done){ - assert.response(app, { + assert.response(server, { url: "/api/v1/sql?" + querystring.stringify({ q: 'CREATE TABLE test_table(a int)', api_key: 1234 }), headers: {host: 'vizzuality.cartodb.com'}, method: 'GET' - },{}, function(res) { + },{}, function(err, res) { assert.equal(res.statusCode, 200, res.statusCode + ': ' + res.body); // Check cache headers // See https://github.com/Vizzuality/CartoDB-SQL-API/issues/43 @@ -595,7 +594,7 @@ it('SELECT INTO with paging ', function(done){ step( function select_into() { var next = this; - assert.response(app, { + assert.response(server, { url: "/api/v1/sql?" + querystring.stringify({ q: 'SELECT generate_series(1,10) InTO "' + esc_tabname + '"', rows_per_page: 1, page: 1, @@ -603,13 +602,13 @@ it('SELECT INTO with paging ', function(done){ }), headers: {host: 'vizzuality.cartodb.com'}, method: 'GET' - },{}, function(res) { next(null, res); }); + },{}, function(err, res) { next(null, res); }); }, function check_res_test_fake_into_1(err, res) { assert.ifError(err); assert.equal(res.statusCode, 200, res.statusCode + ': ' + res.body); var next = this; - assert.response(app, { + assert.response(server, { url: "/api/v1/sql?" + querystring.stringify({ q: 'SELECT \' INTO "c"\' FROM "' + esc_tabname + '"', rows_per_page: 1, page: 1, @@ -617,7 +616,7 @@ it('SELECT INTO with paging ', function(done){ }), headers: {host: 'vizzuality.cartodb.com'}, method: 'GET' - },{}, function(res) { next(null, res); }); + },{}, function(err, res) { next(null, res); }); }, function check_res_drop_table(err, res) { assert.ifError(err); @@ -625,14 +624,14 @@ it('SELECT INTO with paging ', function(done){ var out = JSON.parse(res.body); assert.equal(out.total_rows, 1); // windowing works var next = this; - assert.response(app, { + assert.response(server, { url: "/api/v1/sql?" + querystring.stringify({ q: 'DROP TABLE "' + esc_tabname + '"', api_key: 1234 }), headers: {host: 'vizzuality.cartodb.com'}, method: 'GET' - },{}, function(res) { next(null, res); }); + },{}, function(err, res) { next(null, res); }); }, function check_drop(err, res) { assert.ifError(err); @@ -648,14 +647,14 @@ it('SELECT INTO with paging ', function(done){ // Test effects of COPY // See https://github.com/Vizzuality/cartodb-management/issues/1502 it('COPY TABLE with GET and auth', function(done){ - assert.response(app, { + assert.response(server, { url: "/api/v1/sql?" + querystring.stringify({ q: 'COPY test_table FROM stdin;', api_key: 1234 }), headers: {host: 'vizzuality.cartodb.com'}, method: 'GET' - },{}, function(res) { + },{}, function(err, res) { // We expect a problem, actually assert.equal(res.statusCode, 400, res.statusCode + ': ' + res.body); assert.deepEqual(res.headers['content-type'], 'application/json; charset=utf-8'); @@ -666,14 +665,14 @@ it('COPY TABLE with GET and auth', function(done){ }); it('COPY TABLE with GET and auth', function(done){ - assert.response(app, { + assert.response(server, { url: "/api/v1/sql?" + querystring.stringify({ q: "COPY test_table to '/tmp/x';", api_key: 1234 }), headers: {host: 'vizzuality.cartodb.com'}, method: 'GET' - },{}, function(res) { + },{}, function(err, res) { // We expect a problem, actually assert.equal(res.statusCode, 400, res.statusCode + ': ' + res.body); assert.deepEqual(res.headers['content-type'], 'application/json; charset=utf-8'); @@ -687,14 +686,14 @@ it('COPY TABLE with GET and auth', function(done){ }); it('ALTER TABLE with GET and auth', function(done){ - assert.response(app, { + assert.response(server, { url: "/api/v1/sql?" + querystring.stringify({ q: 'ALTER TABLE test_table ADD b int', api_key: 1234 }), headers: {host: 'vizzuality.cartodb.com'}, method: 'GET' - },{}, function(res) { + },{}, function(err, res) { assert.equal(res.statusCode, 200, res.statusCode + ': ' + res.body); // Check cache headers // See https://github.com/Vizzuality/CartoDB-SQL-API/issues/43 @@ -705,7 +704,7 @@ it('ALTER TABLE with GET and auth', function(done){ }); it('multistatement insert, alter, select, begin, commit', function(done){ - assert.response(app, { + assert.response(server, { url: "/api/v1/sql?" + querystring.stringify({ q: 'BEGIN; DELETE FROM test_table; COMMIT; BEGIN; INSERT INTO test_table(b) values (5); COMMIT; ' + 'ALTER TABLE test_table ALTER b TYPE float USING b::float/2; SELECT b FROM test_table;', @@ -713,7 +712,7 @@ it('multistatement insert, alter, select, begin, commit', function(done){ }), headers: {host: 'vizzuality.cartodb.com'}, method: 'GET' - },{}, function(res) { + },{}, function(err, res) { assert.equal(res.statusCode, 200, res.statusCode + ': ' + res.body); var parsedBody = JSON.parse(res.body); assert.equal(parsedBody.total_rows, 1); @@ -723,29 +722,30 @@ it('multistatement insert, alter, select, begin, commit', function(done){ }); it('TRUNCATE TABLE with GET and auth', function(done){ - assert.response(app, { + assert.response(server, { url: "/api/v1/sql?" + querystring.stringify({ q: 'TRUNCATE TABLE test_table', api_key: 1234 }), headers: {host: 'vizzuality.cartodb.com'}, method: 'GET' - },{}, function(res) { + },{}, function(err, res) { assert.equal(res.statusCode, 200, res.statusCode + ': ' + res.body); assert.ok(!res.hasOwnProperty('x-cache-channel')); assert.equal(res.headers['cache-control'], expected_rw_cache_control); var pbody = JSON.parse(res.body); assert.equal(pbody.rows.length, 0); - assert.response(app, { + assert.response(server, { url: "/api/v1/sql?" + querystring.stringify({ q: 'SELECT count(*) FROM test_table', api_key: 1234 }), headers: {host: 'vizzuality.cartodb.com'}, method: 'GET' - },{}, function(res) { + },{}, function(err, res) { assert.equal(res.statusCode, 200, res.statusCode + ': ' + res.body); - assert.equal(res.headers['x-cache-channel'], 'cartodb_test_user_1_db:public.test_table'); + // table should not get a cache channel as it won't get invalidated + assert.ok(!res.headers.hasOwnProperty('x-cache-channel')); assert.equal(res.headers['cache-control'], expected_cache_control); var pbody = JSON.parse(res.body); assert.equal(pbody.total_rows, 1); @@ -756,14 +756,14 @@ it('TRUNCATE TABLE with GET and auth', function(done){ }); it('REINDEX TABLE with GET and auth', function(done){ - assert.response(app, { + assert.response(server, { url: "/api/v1/sql?" + querystring.stringify({ q: ' ReINdEX TABLE test_table', api_key: 1234 }), headers: {host: 'vizzuality.cartodb.com'}, method: 'GET' - },{}, function(res) { + },{}, function(err, res) { assert.equal(res.statusCode, 200, res.statusCode + ': ' + res.body); assert.ok(!res.hasOwnProperty('x-cache-channel')); assert.equal(res.headers['cache-control'], expected_rw_cache_control); @@ -774,14 +774,14 @@ it('REINDEX TABLE with GET and auth', function(done){ }); it('DROP TABLE with GET and auth', function(done){ - assert.response(app, { + assert.response(server, { url: "/api/v1/sql?" + querystring.stringify({ q: 'DROP TABLE test_table', api_key: 1234 }), headers: {host: 'vizzuality.cartodb.com'}, method: 'GET' - },{}, function(res) { + },{}, function(err, res) { assert.equal(res.statusCode, 200, res.statusCode + ': ' + res.body); // Check cache headers // See https://github.com/Vizzuality/CartoDB-SQL-API/issues/43 @@ -792,14 +792,14 @@ it('DROP TABLE with GET and auth', function(done){ }); it('CREATE FUNCTION with GET and auth', function(done){ - assert.response(app, { + assert.response(server, { url: "/api/v1/sql?" + querystring.stringify({ q: 'CREATE FUNCTION create_func_test(a int) RETURNS INT AS \'SELECT 1\' LANGUAGE \'sql\'', api_key: 1234 }), headers: {host: 'vizzuality.cartodb.com'}, method: 'GET' - },{}, function(res) { + },{}, function(err, res) { assert.equal(res.statusCode, 200, res.statusCode + ': ' + res.body); // Check cache headers // See https://github.com/Vizzuality/CartoDB-SQL-API/issues/43 @@ -810,14 +810,14 @@ it('CREATE FUNCTION with GET and auth', function(done){ }); it('DROP FUNCTION with GET and auth', function(done){ - assert.response(app, { + assert.response(server, { url: "/api/v1/sql?" + querystring.stringify({ q: 'DROP FUNCTION create_func_test(a int)', api_key: 1234 }), headers: {host: 'vizzuality.cartodb.com'}, method: 'GET' - },{}, function(res) { + },{}, function(err, res) { assert.equal(res.statusCode, 200, res.statusCode + ': ' + res.body); // Check cache headers // See https://github.com/Vizzuality/CartoDB-SQL-API/issues/43 @@ -828,11 +828,11 @@ it('DROP FUNCTION with GET and auth', function(done){ }); it('sends a 400 when an unsupported format is requested', function(done){ - assert.response(app, { + assert.response(server, { url: '/api/v1/sql?q=SELECT%20*%20FROM%20untitle_table_4&format=unknown', headers: {host: 'vizzuality.cartodb.com'}, method: 'GET' - },{ }, function(res){ + },{ }, function(err, res){ assert.equal(res.statusCode, 400, res.body); assert.deepEqual(res.headers['content-type'], 'application/json; charset=utf-8'); assert.deepEqual(res.headers['content-disposition'], 'inline'); @@ -842,15 +842,15 @@ it('sends a 400 when an unsupported format is requested', function(done){ }); it('GET /api/v1/sql with SQL parameter and no format, ensuring content-disposition set to json', function(done){ - assert.response(app, { + assert.response(server, { url: '/api/v1/sql?q=SELECT%20*%20FROM%20untitle_table_4', headers: {host: 'vizzuality.cartodb.com'}, method: 'GET' - },{ }, function(res){ + },{ }, function(err, res){ assert.equal(res.statusCode, 200, res.body); - var ct = res.header('Content-Type'); + var ct = res.headers['content-type']; assert.ok(/json/.test(ct), 'Default format is not JSON: ' + ct); - var cd = res.header('Content-Disposition'); + var cd = res.headers['content-disposition']; assert.equal(true, /^inline/.test(cd), 'Default format is not disposed inline: ' + cd); assert.equal(true, /filename=cartodb-query.json/gi.test(cd), 'Unexpected JSON filename: ' + cd); done(); @@ -858,16 +858,16 @@ it('GET /api/v1/sql with SQL parameter and no format, ensuring content-dispositi }); it('POST /api/v1/sql with SQL parameter and no format, ensuring content-disposition set to json', function(done){ - assert.response(app, { + assert.response(server, { url: '/api/v1/sql', data: querystring.stringify({q: "SELECT * FROM untitle_table_4" }), headers: {host: 'vizzuality.cartodb.com', 'Content-Type': 'application/x-www-form-urlencoded' }, method: 'POST' - },{ }, function(res){ + },{ }, function(err, res){ assert.equal(res.statusCode, 200, res.body); - var ct = res.header('Content-Type'); + var ct = res.headers['content-type']; assert.ok(/json/.test(ct), 'Default format is not JSON: ' + ct); - var cd = res.header('Content-Disposition'); + var cd = res.headers['content-disposition']; assert.equal(true, /^inline/.test(cd), 'Default format is not disposed inline: ' + cd); assert.equal(true, /filename=cartodb-query.json/gi.test(cd), 'Unexpected JSON filename: ' + cd); done(); @@ -875,15 +875,15 @@ it('POST /api/v1/sql with SQL parameter and no format, ensuring content-disposit }); it('GET /api/v1/sql with SQL parameter and no format, but a filename', function(done){ - assert.response(app, { + assert.response(server, { url: '/api/v1/sql?q=SELECT%20*%20FROM%20untitle_table_4&filename=x', headers: {host: 'vizzuality.cartodb.com'}, method: 'GET' - },{ }, function(res){ + },{ }, function(err, res){ assert.equal(res.statusCode, 200, res.body); - var ct = res.header('Content-Type'); + var ct = res.headers['content-type']; assert.ok(/json/.test(ct), 'Default format is not JSON: ' + ct); - var cd = res.header('Content-Disposition'); + var cd = res.headers['content-disposition']; assert.equal(true, /^attachment/.test(cd), 'Format with filename is not disposed as attachment: ' + cd); assert.equal(true, /filename=x.json/gi.test(cd), 'Unexpected JSON filename: ' + cd); done(); @@ -891,11 +891,11 @@ it('GET /api/v1/sql with SQL parameter and no format, but a filename', function( }); it('field named "the_geom_webmercator" is not skipped by default', function(done){ - assert.response(app, { + assert.response(server, { url: '/api/v1/sql?q=SELECT%20*%20FROM%20untitle_table_4', headers: {host: 'vizzuality.cartodb.com'}, method: 'GET' - },{ }, function(res){ + },{ }, function(err, res){ assert.equal(res.statusCode, 200, res.body); var row0 = JSON.parse(res.body).rows[0]; var checkfields = {'name':1, 'cartodb_id':1, 'the_geom':1, 'the_geom_webmercator':1}; @@ -911,11 +911,11 @@ it('field named "the_geom_webmercator" is not skipped by default', function(done }); it('skipfields controls included fields', function(done){ - assert.response(app, { + assert.response(server, { url: '/api/v1/sql?q=SELECT%20*%20FROM%20untitle_table_4&skipfields=the_geom_webmercator,cartodb_id,unexistant', headers: {host: 'vizzuality.cartodb.com'}, method: 'GET' - },{ }, function(res){ + },{ }, function(err, res){ assert.equal(res.statusCode, 200, res.body); var row0 = JSON.parse(res.body).rows[0]; var checkfields = {'name':1, 'cartodb_id':0, 'the_geom':1, 'the_geom_webmercator':0}; @@ -931,12 +931,12 @@ it('skipfields controls included fields', function(done){ }); it('multiple skipfields parameter do not kill the backend', function(done){ - assert.response(app, { + assert.response(server, { url: '/api/v1/sql?q=SELECT%20*%20FROM%20untitle_table_4&skipfields=unexistent,the_geom_webmercator' + '&skipfields=cartodb_id,unexistant', headers: {host: 'vizzuality.cartodb.com'}, method: 'GET' - },{ }, function(res){ + },{ }, function(err, res){ assert.equal(res.statusCode, 200, res.body); var row0 = JSON.parse(res.body).rows[0]; var checkfields = {'name':1, 'cartodb_id':0, 'the_geom':1, 'the_geom_webmercator':0}; @@ -952,14 +952,14 @@ it('multiple skipfields parameter do not kill the backend', function(done){ }); it('GET /api/v1/sql ensure cross domain set on errors', function(done){ - assert.response(app, { + assert.response(server, { url: '/api/v1/sql?q=SELECT%20*gadfgadfg%20FROM%20untitle_table_4', headers: {host: 'vizzuality.cartodb.com'}, method: 'GET' },{ status: 400 - }, function(res){ - var cd = res.header('Access-Control-Allow-Origin'); + }, function(err, res){ + var cd = res.headers['access-control-allow-origin']; assert.deepEqual(res.headers['content-type'], 'application/json; charset=utf-8'); assert.deepEqual(res.headers['content-disposition'], 'inline'); assert.equal(cd, '*'); @@ -1021,7 +1021,7 @@ function testSystemQueries(description, queries, statusErrorCode, apiKey) { method: 'GET', url: '/api/v1/sql?' + querystring.stringify(queryStringParams) }; - assert.response(app, request, function(response) { + assert.response(server, request, function(err, response) { assert.equal(response.statusCode, statusErrorCode); done(); }); @@ -1030,11 +1030,11 @@ function testSystemQueries(description, queries, statusErrorCode, apiKey) { } it('GET decent error if domain is incorrect', function(done){ - assert.response(app, { + assert.response(server, { url: '/api/v1/sql?q=SELECT%20*%20FROM%20untitle_table_4&format=geojson', headers: {host: 'vizzualinot.cartodb.com'}, method: 'GET' - }, {}, function(res){ + }, {}, function(err, res){ assert.equal(res.statusCode, 404, res.statusCode + ( res.statusCode !== 200 ? ( ': ' + res.body ) : '')); assert.deepEqual(res.headers['content-type'], 'application/json; charset=utf-8'); assert.deepEqual(res.headers['content-disposition'], 'inline'); @@ -1049,13 +1049,13 @@ it('GET decent error if domain is incorrect', function(done){ // this test does not make sense with the current CDB_QueryTables implementation it('GET decent error if SQL is broken', function(done){ - assert.response(app, { + assert.response(server, { url: '/api/v1/sql?' + querystring.stringify({q: 'SELECT star FROM this and that' }), headers: {host: 'vizzuality.cartodb.com'}, method: 'GET' - },{}, function(res){ + },{}, function(err, res){ assert.equal(res.statusCode, 400, res.statusCode + ': ' + res.body); assert.deepEqual(res.headers['content-type'], 'application/json; charset=utf-8'); assert.deepEqual(res.headers['content-disposition'], 'inline'); @@ -1068,13 +1068,13 @@ it('GET decent error if SQL is broken', function(done){ // See https://github.com/Vizzuality/CartoDB-SQL-API/issues/88 it('numeric arrays are rendered as such', function(done){ - assert.response(app, { + assert.response(server, { url: "/api/v1/sql?" + querystring.stringify({q: "SELECT ARRAY[8.7,4.3]::numeric[] as x" }), headers: {host: 'vizzuality.localhost.lan:8080' }, method: 'GET' - },{}, function(res) { + },{}, function(err, res) { assert.equal(res.statusCode, 200, res.statusCode + ': ' + res.body); var out = JSON.parse(res.body); assert.ok(out.hasOwnProperty('time')); @@ -1091,7 +1091,7 @@ it('numeric arrays are rendered as such', function(done){ // See https://github.com/Vizzuality/CartoDB-SQL-API/issues/97 it('field names and types are exposed', function(done){ - assert.response(app, { + assert.response(server, { url: '/api/v1/sql?' + querystring.stringify({ q: "SELECT 1::int as a, 2::float8 as b, 3::varchar as c, " + "4::char as d, now() as e, 'a'::text as f" + @@ -1104,7 +1104,7 @@ it('field names and types are exposed', function(done){ }), headers: {host: 'vizzuality.cartodb.com'}, method: 'GET' - },{ }, function(res) { + },{ }, function(err, res) { assert.equal(res.statusCode, 200, res.body); var parsedBody = JSON.parse(res.body); assert.equal(_.keys(parsedBody.fields).length, 10); @@ -1124,14 +1124,14 @@ it('field names and types are exposed', function(done){ // See https://github.com/CartoDB/CartoDB-SQL-API/issues/109 it('schema response takes skipfields into account', function(done){ - assert.response(app, { + assert.response(server, { url: '/api/v1/sql?' + querystring.stringify({ q: "SELECT 1 as a, 2 as b, 3 as c ", skipfields: 'b' }), headers: {host: 'vizzuality.cartodb.com'}, method: 'GET' - },{ }, function(res) { + },{ }, function(err, res) { assert.equal(res.statusCode, 200, res.body); var parsedBody = JSON.parse(res.body); assert.equal(_.keys(parsedBody.fields).length, 2); @@ -1144,7 +1144,7 @@ it('schema response takes skipfields into account', function(done){ // See https://github.com/Vizzuality/CartoDB-SQL-API/issues/100 it('numeric fields are rendered as numbers in JSON', function(done){ - assert.response(app, { + assert.response(server, { url: '/api/v1/sql?' + querystring.stringify({ q: "WITH inp AS ( SELECT 1::int2 as a, 2::int4 as b, " + "3::int8 as c, 4::float4 as d, " + @@ -1160,7 +1160,7 @@ it('numeric fields are rendered as numbers in JSON', function(done){ }), headers: {host: 'vizzuality.cartodb.com'}, method: 'GET' - },{ }, function(res) { + },{ }, function(err, res) { assert.equal(res.statusCode, 200, res.body); var parsedBody = JSON.parse(res.body); var row = parsedBody.rows[0]; @@ -1195,13 +1195,13 @@ it('timezone info in JSON output', function(done){ step( function testEuropeRomeExplicit() { var next = this; - assert.response(app, { + assert.response(server, { url: '/api/v1/sql?' + querystring.stringify({ q: "SET timezone TO 'Europe/Rome'; SELECT '2000-01-01T00:00:00+01'::timestamptz as d" }), headers: {host: 'vizzuality.cartodb.com'}, method: 'GET' - },{ }, function(res) { + },{ }, function(err, res) { try { assert.equal(res.statusCode, 200, res.body); var parsedBody = JSON.parse(res.body); @@ -1215,13 +1215,13 @@ it('timezone info in JSON output', function(done){ function testEuropeRomeImplicit(err) { assert.ifError(err); var next = this; - assert.response(app, { + assert.response(server, { url: '/api/v1/sql?' + querystring.stringify({ q: "SET timezone TO 'Europe/Rome'; SELECT '2000-01-01T00:00:00'::timestamp as d" }), headers: {host: 'vizzuality.cartodb.com'}, method: 'GET' - },{ }, function(res) { + },{ }, function(err, res) { try { assert.equal(res.statusCode, 200, res.body); var parsedBody = JSON.parse(res.body); @@ -1235,13 +1235,13 @@ it('timezone info in JSON output', function(done){ function testUTCExplicit(err) { assert.ifError(err); var next = this; - assert.response(app, { + assert.response(server, { url: '/api/v1/sql?' + querystring.stringify({ q: "SET timezone TO 'UTC'; SELECT '2000-01-01T00:00:00+00'::timestamptz as d" }), headers: {host: 'vizzuality.cartodb.com'}, method: 'GET' - },{ }, function(res) { + },{ }, function(err, res) { try { assert.equal(res.statusCode, 200, res.body); var parsedBody = JSON.parse(res.body); @@ -1255,13 +1255,13 @@ it('timezone info in JSON output', function(done){ function testUTCImplicit(err) { assert.ifError(err); var next = this; - assert.response(app, { + assert.response(server, { url: '/api/v1/sql?' + querystring.stringify({ q: "SET timezone TO 'UTC'; SELECT '2000-01-01T00:00:00'::timestamp as d" }), headers: {host: 'vizzuality.cartodb.com'}, method: 'GET' - },{ }, function(res) { + },{ }, function(err, res) { try { assert.equal(res.statusCode, 200, res.body); var parsedBody = JSON.parse(res.body); @@ -1283,8 +1283,7 @@ it('timezone info in JSON output', function(done){ it('notice and warning info in JSON output', function(done){ step( function addRaiseFunction() { - var next = this; - assert.response(app, { + assert.response(server, { url: '/api/v1/sql?' + querystring.stringify({ q: "create or replace function raise(lvl text, msg text) returns void as $$ begin if lvl = 'notice' " + "then raise notice '%', msg; elsif lvl = 'warning' then raise warning '%', msg; " + @@ -1293,70 +1292,62 @@ it('notice and warning info in JSON output', function(done){ }), headers: {host: 'vizzuality.cartodb.com'}, method: 'GET' - },{ }, function(res) { - var err = null; - try { - assert.equal(res.statusCode, 200, res.statusCode + ': ' + res.body); - } catch (e) { err = e; } - next(err); - }); + }, RESPONSE_OK, this); }, function raiseNotice(err) { assert.ifError(err); var next = this; - assert.response(app, { + assert.response(server, { url: '/api/v1/sql?' + querystring.stringify({ q: "SET client_min_messages TO 'notice'; select raise('notice', 'hello notice')" }), headers: {host: 'vizzuality.cartodb.com'}, method: 'GET' - },{}, function(res) { - var err = null; + }, RESPONSE_OK, function(err, res) { try { - assert.equal(res.statusCode, 200, res.body); var parsedBody = JSON.parse(res.body); assert.ok(parsedBody.hasOwnProperty('notices'), 'Missing notices from result'); assert.equal(parsedBody.notices.length, 1); assert.equal(parsedBody.notices[0], 'hello notice'); - } catch (e) { err = e; } + } catch (e) { + return next(e); + } next(err); }); }, function raiseWarning(err) { assert.ifError(err); var next = this; - assert.response(app, { + assert.response(server, { url: '/api/v1/sql?' + querystring.stringify({ q: "SET client_min_messages TO 'notice'; select raise('warning', 'hello warning')" }), headers: {host: 'vizzuality.cartodb.com'}, method: 'GET' - },{}, function(res) { - var err = null; + }, RESPONSE_OK, function(err, res) { try { - assert.equal(res.statusCode, 200, res.body); var parsedBody = JSON.parse(res.body); assert.ok(parsedBody.hasOwnProperty('warnings'), 'Missing warnings from result'); assert.equal(parsedBody.warnings.length, 1); assert.equal(parsedBody.warnings[0], 'hello warning'); - } catch (e) { err = e; } + } catch (e) { + return next(e); + } next(err); }); }, function raiseBothWarningAndNotice(err) { assert.ifError(err); var next = this; - assert.response(app, { + assert.response(server, { url: '/api/v1/sql?' + querystring.stringify({ q: "SET client_min_messages TO 'notice'; select raise('warning', 'hello again warning'), " + "raise('notice', 'hello again notice');" }), headers: {host: 'vizzuality.cartodb.com'}, method: 'GET' - },{}, function(res) { - var err = null; + }, RESPONSE_OK, function(err, res) { try { - assert.equal(res.statusCode, 200, res.body); var parsedBody = JSON.parse(res.body); assert.ok(parsedBody.hasOwnProperty('warnings'), 'Missing warnings from result'); assert.equal(parsedBody.warnings.length, 1); @@ -1364,29 +1355,29 @@ it('notice and warning info in JSON output', function(done){ assert.ok(parsedBody.hasOwnProperty('notices'), 'Missing notices from result'); assert.equal(parsedBody.notices.length, 1); assert.equal(parsedBody.notices[0], 'hello again notice'); - } catch (e) { err = e; } + } catch (e) { + return next(e); + } next(err); }); }, - function delRaiseFunction(err) { - var next = this; - assert.response(app, { + function delRaiseFunction() { + assert.response(server, { url: '/api/v1/sql?' + querystring.stringify({ q: "DROP function raise(text, text)", api_key: '1234' }), headers: {host: 'vizzuality.cartodb.com'}, method: 'GET' - },{ }, function(res) { + }, RESPONSE_OK, function(err, res) { try { assert.equal(res.statusCode, 200, res.body); JSON.parse(res.body); - } catch (e) { err = new Error(err + ',' + e); } - next(err); + } catch (e) { + err = new Error(err + ',' + e); + } + done(err); }); - }, - function finish(err) { - done(err); } ); }); @@ -1395,11 +1386,11 @@ it('notice and warning info in JSON output', function(done){ * CORS */ it('GET /api/v1/sql with SQL parameter on SELECT only should return CORS headers ', function(done){ - assert.response(app, { + assert.response(server, { url: '/api/v1/sql?q=SELECT%20*%20FROM%20untitle_table_4&database=cartodb_test_user_1_db', headers: {host: 'vizzuality.cartodb.com'}, method: 'GET' - },{ }, function(res) { + },{ }, function(err, res) { assert.equal(res.statusCode, 200, res.body); // Check cache headers assert.equal(res.headers['x-cache-channel'], 'cartodb_test_user_1_db:public.untitle_table_4'); @@ -1414,11 +1405,11 @@ it('GET /api/v1/sql with SQL parameter on SELECT only should return CORS headers }); it('GET with callback param returns wrapped result set with callback as jsonp', function(done) { - assert.response(app, { + assert.response(server, { url: '/api/v1/sql?q=SELECT%20*%20FROM%20untitle_table_4&callback=foo_jsonp', headers: {host: 'vizzuality.cartodb.com'}, method: 'GET' - },{ }, function(res) { + },{ }, function(err, res) { assert.equal(res.statusCode, 200, res.body); assert.ok(res.body.match(/foo\_jsonp\(.*\)/)); done(); @@ -1426,11 +1417,11 @@ it('GET with callback param returns wrapped result set with callback as jsonp', }); it('GET with callback must return 200 status error even if it is an error', function(done){ - assert.response(app, { + assert.response(server, { url: "/api/v1/sql?q=DROP%20TABLE%20untitle_table_4&callback=foo_jsonp", headers: {host: 'vizzuality.cartodb.com'}, method: 'GET' - },{}, function(res) { + },{}, function(err, res) { assert.equal(res.statusCode, 200, res.statusCode + ': ' + res.body); var didRunJsonCallback = false; // jshint ignore:start @@ -1446,7 +1437,7 @@ it('GET with callback must return 200 status error even if it is an error', func }); it('GET with slow query exceeding statement timeout returns proper error message', function(done){ - assert.response(app, { + assert.response(server, { url: "/api/v1/sql?q=select%20pg_sleep(2.1)%20as%20sleep", headers: {host: 'vizzuality.cartodb.com'}, method: 'GET' @@ -1454,7 +1445,7 @@ it('GET with callback must return 200 status error even if it is an error', func { status: 400 }, - function(res) { + function(err, res) { assert.ok(res.body.match(/was not able to finish.*try again/i)); done(); }); @@ -1473,7 +1464,7 @@ it('GET with callback must return 200 status error even if it is an error', func consoleError = what; }; assert.response( - app, + server, { url: "/api/v1/sql?" + querystring.stringify({ q: "SELECT * FROM untitle_table_4" diff --git a/test/acceptance/backend_crash.js b/test/acceptance/backend_crash.js index 592621090..d6d338a1a 100644 --- a/test/acceptance/backend_crash.js +++ b/test/acceptance/backend_crash.js @@ -27,17 +27,14 @@ it('does not hang server', function(done){ var db_port_backup = global.settings.db_port; global.settings.db_host = 'localhost'; global.settings.db_port = sql_server_port; - var app = require(global.settings.app_root + '/app/app')(); + var server = require('../../app/server')(); step( function sendQuery() { - var next = this; - assert.response(app, { + assert.response(server, { url: '/api/v1/sql?q=SELECT+1', method: 'GET', headers: {host: 'vizzuality.localhost' } - },{}, function(res, err) { - next(err, res); - }); + },{}, this); }, function checkResponse(err, res) { assert.ifError(err); @@ -49,14 +46,11 @@ it('does not hang server', function(done){ return null; }, function sendAnotherQuery() { - var next = this; - assert.response(app, { + assert.response(server, { url: '/api/v1/sql?q=SELECT+2', method: 'GET', headers: {host: 'vizzuality.localhost' } - },{}, function(res, err) { - next(err, res); - }); + },{}, this); }, function checkResponse(err, res) { assert.ifError(err); diff --git a/test/acceptance/batch.test.js b/test/acceptance/batch.test.js deleted file mode 100644 index 5035c0119..000000000 --- a/test/acceptance/batch.test.js +++ /dev/null @@ -1,267 +0,0 @@ -var assert = require('../support/assert'); -var _ = require('underscore'); -var redis = require('redis'); -var queue = require('queue-async'); -var batchFactory = require('../../batch'); - -var JobPublisher = require('../../batch/job_publisher'); -var JobQueue = require('../../batch/job_queue'); -var UserIndexer = require('../../batch/user_indexer'); -var JobBackend = require('../../batch/job_backend'); -var JobService = require('../../batch/job_service'); -var UserDatabaseMetadataService = require('../../batch/user_database_metadata_service'); -var JobCanceller = require('../../batch/job_canceller'); -var metadataBackend = require('cartodb-redis')({ - host: global.settings.redis_host, - port: global.settings.redis_port, - max: global.settings.redisPool, - idleTimeoutMillis: global.settings.redisIdleTimeoutMillis, - reapIntervalMillis: global.settings.redisReapIntervalMillis -}); - -describe('batch module', function() { - var dbInstance = 'localhost'; - var username = 'vizzuality'; - var jobQueue = new JobQueue(metadataBackend); - var jobPublisher = new JobPublisher(redis); - var userIndexer = new UserIndexer(metadataBackend); - var jobBackend = new JobBackend(metadataBackend, jobQueue, jobPublisher, userIndexer); - var userDatabaseMetadataService = new UserDatabaseMetadataService(metadataBackend); - var jobCanceller = new JobCanceller(userDatabaseMetadataService); - var jobService = new JobService(jobBackend, jobCanceller); - - var batch = batchFactory(metadataBackend); - - before(function () { - batch.start(); - }); - - after(function (done) { - batch.stop(); - batch.drain(function () { - metadataBackend.redisCmd(5, 'DEL', [ 'batch:queues:localhost' ], done); - }); - }); - - function createJob(sql, done) { - var data = { - user: username, - query: sql, - host: dbInstance - }; - - jobService.create(data, function (err, job) { - if (err) { - return done(err); - } - - done(null, job.serialize()); - }); - } - - it('should perform job with select', function (done) { - createJob('select * from private_table', function (err, job) { - if (err) { - return done(err); - } - - batch.on('job:done', function (job_id) { - if (job_id === job.job_id) { - done(); - } - }); - }); - }); - - it('should perform job with select into', function (done) { - createJob('select * into batch_test_table from (select * from private_table) as job', function (err, job) { - if (err) { - return done(err); - } - - batch.on('job:done', function (job_id) { - if (job_id === job.job_id) { - done(); - } - }); - }); - }); - - it('should perform job swith select from result table', function (done) { - createJob('select * from batch_test_table', function (err, job) { - if (err) { - return done(err); - } - - batch.on('job:done', function (job_id) { - if (job_id === job.job_id) { - done(); - } - }); - }); - }); - - it('should perform all enqueued jobs', function (done) { - var jobs = [ - 'select * from private_table', - 'select * from private_table', - 'select * from private_table', - 'select * from private_table', - 'select * from private_table', - 'select * from private_table', - 'select * from private_table', - 'select * from private_table', - 'select * from private_table', - 'select * from private_table' - ]; - - var jobsQueue = queue(jobs.length); - - jobs.forEach(function(job) { - jobsQueue.defer(createJob, job); - }); - - jobsQueue.awaitAll(function (err, jobsCreated) { - if (err) { - return done(err); - } - - var jobsDone = 0; - - batch.on('job:done', function (job_id) { - _.find(jobsCreated, function(job) { - if (job_id === job.job_id) { - jobsDone += 1; - if (jobsDone === jobs.length) { - done(); - } - } - }); - }); - }); - }); - - it('should set all job as failed', function (done) { - var jobs = [ - 'select * from unexistent_table', - 'select * from unexistent_table', - 'select * from unexistent_table', - 'select * from unexistent_table', - 'select * from unexistent_table', - 'select * from unexistent_table', - 'select * from unexistent_table', - 'select * from unexistent_table', - 'select * from unexistent_table', - 'select * from unexistent_table' - ]; - - var jobsQueue = queue(jobs.length); - - jobs.forEach(function(job) { - jobsQueue.defer(createJob, job); - }); - - jobsQueue.awaitAll(function (err, jobsCreated) { - if (err) { - return done(err); - } - - var jobsFailed = 0; - - batch.on('job:failed', function (job_id) { - _.find(jobsCreated, function(job) { - if (job_id === job.job_id) { - jobsFailed += 1; - if (jobsFailed === jobs.length) { - done(); - } - } - }); - }); - }); - }); - - it('should drain the current job', function (done) { - createJob('select pg_sleep(3)', function (err, job) { - if (err) { - return done(err); - } - setTimeout(function () { - jobBackend.get(job.job_id, function (err, job) { - if (err) { - done(err); - } - - assert.equal(job.status, 'running'); - - batch.drain(function () { - jobBackend.get(job.job_id, function (err, job) { - if (err) { - done(err); - } - assert.equal(job.status, 'pending'); - done(); - }); - }); - }); - }, 50); - }); - }); - - it('should perform job with array of select', function (done) { - var queries = ['select * from private_table limit 1', 'select * from private_table']; - - createJob(queries, function (err, job) { - if (err) { - return done(err); - } - - var queriesDone = 0; - - var checkJobDone = function (job_id) { - if (job_id === job.job_id) { - queriesDone += 1; - if (queriesDone === queries.length) { - done(); - } - } - }; - - batch.on('job:done', checkJobDone); - batch.on('job:pending', checkJobDone); - }); - }); - - it('should set job as failed if last query fails', function (done) { - var queries = ['select * from private_table', 'select * from undefined_table']; - - createJob(queries, function (err, job) { - if (err) { - return done(err); - } - - batch.on('job:failed', function (job_id) { - if (job_id === job.job_id) { - done(); - } - }); - }); - }); - - it('should set job as failed if first query fails', function (done) { - var queries = ['select * from undefined_table', 'select * from private_table']; - - createJob(queries, function (err, job) { - if (err) { - return done(err); - } - - batch.on('job:failed', function (job_id) { - if (job_id === job.job_id) { - done(); - } - }); - }); - }); - -}); diff --git a/test/acceptance/batch/batch-drain.test.js b/test/acceptance/batch/batch-drain.test.js new file mode 100644 index 000000000..9a1cf0d22 --- /dev/null +++ b/test/acceptance/batch/batch-drain.test.js @@ -0,0 +1,80 @@ +require('../../helper'); +var assert = require('../../support/assert'); +var redisUtils = require('../../support/redis_utils'); +var batchFactory = require('../../../batch/index'); + +var JobPublisher = require('../../../batch/pubsub/job-publisher'); +var JobQueue = require('../../../batch/job_queue'); +var JobBackend = require('../../../batch/job_backend'); +var JobService = require('../../../batch/job_service'); +var UserDatabaseMetadataService = require('../../../batch/user_database_metadata_service'); +var JobCanceller = require('../../../batch/job_canceller'); +var metadataBackend = require('cartodb-redis')({ pool: redisUtils.getPool() }); + +describe('batch module', function() { + var dbInstance = 'localhost'; + var username = 'vizzuality'; + var pool = redisUtils.getPool(); + var jobPublisher = new JobPublisher(pool); + var jobQueue = new JobQueue(metadataBackend, jobPublisher); + var jobBackend = new JobBackend(metadataBackend, jobQueue); + var userDatabaseMetadataService = new UserDatabaseMetadataService(metadataBackend); + var jobCanceller = new JobCanceller(userDatabaseMetadataService); + var jobService = new JobService(jobBackend, jobCanceller); + + before(function (done) { + this.batch = batchFactory(metadataBackend, pool); + this.batch.start(); + this.batch.on('ready', done); + }); + + after(function (done) { + this.batch.stop(); + redisUtils.clean('batch:*', done); + }); + + function createJob(sql, done) { + var data = { + user: username, + query: sql, + host: dbInstance + }; + + jobService.create(data, function (err, job) { + if (err) { + return done(err); + } + + done(null, job.serialize()); + }); + } + + it('should drain the current job', function (done) { + var self = this; + createJob('select pg_sleep(3)', function (err, job) { + if (err) { + return done(err); + } + setTimeout(function () { + jobBackend.get(job.job_id, function (err, job) { + if (err) { + done(err); + } + + assert.equal(job.status, 'running'); + + self.batch.drain(function () { + jobBackend.get(job.job_id, function (err, job) { + if (err) { + done(err); + } + assert.equal(job.status, 'pending'); + done(); + }); + }); + }); + }, 50); + }); + }); + +}); diff --git a/test/acceptance/batch/batch-limits.test.js b/test/acceptance/batch/batch-limits.test.js new file mode 100644 index 000000000..f015787eb --- /dev/null +++ b/test/acceptance/batch/batch-limits.test.js @@ -0,0 +1,48 @@ +require('../../helper'); + +var assert = require('../../support/assert'); +var BatchTestClient = require('../../support/batch-test-client'); +var JobStatus = require('../../../batch/job_status'); +var redisUtils = require('../../support/redis_utils'); +var metadataBackend = require('cartodb-redis')({ pool: redisUtils.getPool() }); + +describe('batch query statement_timeout limit', function() { + + before(function(done) { + this.batchTestClient = new BatchTestClient(); + this.batchQueryTimeout = global.settings.batch_query_timeout; + global.settings.batch_query_timeout = 15000; + metadataBackend.redisCmd(5, 'HMSET', ['limits:batch:vizzuality', 'timeout', 100], done); + }); + + after(function(done) { + global.settings.batch_query_timeout = this.batchQueryTimeout; + redisUtils.clean('limits:batch:*', function() { + this.batchTestClient.drain(done); + }.bind(this)); + }); + + function jobPayload(query) { + return { + query: query + }; + } + + it('should cancel with user statement_timeout limit', function (done) { + var payload = jobPayload('select pg_sleep(10)'); + this.batchTestClient.createJob(payload, function(err, jobResult) { + if (err) { + return done(err); + } + jobResult.getStatus(function (err, job) { + if (err) { + return done(err); + } + assert.equal(job.status, JobStatus.FAILED); + assert.ok(job.failed_reason.match(/statement.*timeout/)); + return done(); + }); + }); + }); + +}); diff --git a/test/acceptance/batch/batch.multiquery.test.js b/test/acceptance/batch/batch.multiquery.test.js new file mode 100644 index 000000000..b512ed122 --- /dev/null +++ b/test/acceptance/batch/batch.multiquery.test.js @@ -0,0 +1,236 @@ +'use strict'; + +require('../../helper'); + +var BatchTestClient = require('../../support/batch-test-client'); +var JobStatus = require('../../../batch/job_status'); + +var assert = require('../../support/assert'); +var queue = require('queue-async'); + +describe('batch multiquery', function() { + function jobPayload(query) { + return { + query: query + }; + } + + before(function() { + this.batchTestClient = new BatchTestClient(); + }); + + after(function (done) { + this.batchTestClient.drain(done); + }); + + it('should perform one multiquery job with two queries', function (done) { + var queries = [ + 'select pg_sleep(0)', + 'select pg_sleep(0)' + ]; + + var payload = jobPayload(queries); + this.batchTestClient.createJob(payload, function(err, jobResult) { + if (err) { + return done(err); + } + jobResult.getStatus(function (err, job) { + if (err) { + return done(err); + } + assert.equal(job.status, JobStatus.DONE); + return done(); + }); + }); + }); + + it('should perform one multiquery job with two queries and fail on last one', function (done) { + var queries = [ + 'select pg_sleep(0)', + 'select shouldFail()' + ]; + + var payload = jobPayload(queries); + this.batchTestClient.createJob(payload, function(err, jobResult) { + if (err) { + return done(err); + } + jobResult.getStatus(function (err, job) { + if (err) { + return done(err); + } + assert.equal(job.status, JobStatus.FAILED); + return done(); + }); + }); + }); + + it('should perform one multiquery job with three queries and fail on last one', function (done) { + var queries = [ + 'select pg_sleep(0)', + 'select pg_sleep(0)', + 'select shouldFail()' + ]; + + var payload = jobPayload(queries); + this.batchTestClient.createJob(payload, function(err, jobResult) { + if (err) { + return done(err); + } + jobResult.getStatus(function (err, job) { + if (err) { + return done(err); + } + assert.equal(job.status, JobStatus.FAILED); + return done(); + }); + }); + }); + + + it('should perform one multiquery job with three queries and fail on second one', function (done) { + var queries = [ + 'select pg_sleep(0)', + 'select shouldFail()', + 'select pg_sleep(0)' + ]; + + var payload = jobPayload(queries); + this.batchTestClient.createJob(payload, function(err, jobResult) { + if (err) { + return done(err); + } + jobResult.getStatus(function (err, job) { + if (err) { + return done(err); + } + assert.equal(job.status, JobStatus.FAILED); + return done(); + }); + }); + }); + + it('should perform two multiquery job with two queries for each one', function (done) { + var self = this; + + var jobs = [ + [ + 'select pg_sleep(0)', + 'select pg_sleep(0)' + ], + [ + 'select pg_sleep(0)', + 'select pg_sleep(0)' + ] + ]; + + var jobsQueue = queue(2); + + jobs.forEach(function(job) { + jobsQueue.defer(function(payload, done) { + self.batchTestClient.createJob(payload, function(err, jobResult) { + if (err) { + return done(err); + } + jobResult.getStatus(done); + }); + }, jobPayload(job)); + }); + + jobsQueue.awaitAll(function (err, jobsCreated) { + if (err) { + return done(err); + } + + jobsCreated.forEach(function(job) { + assert.equal(job.status, JobStatus.DONE); + }); + + return done(); + }); + }); + + it('should perform two multiquery job with two queries for each one and fail the first one', function (done) { + var self = this; + + var jobs = [ + [ + 'select pg_sleep(0)', + 'select shouldFail()' + ], + [ + 'select pg_sleep(0)', + 'select pg_sleep(0)' + ] + ]; + + var expectedStatus = [JobStatus.FAILED, JobStatus.DONE]; + var jobsQueue = queue(2); + + jobs.forEach(function(job) { + jobsQueue.defer(function(payload, done) { + self.batchTestClient.createJob(payload, function(err, jobResult) { + if (err) { + return done(err); + } + jobResult.getStatus(done); + }); + }, jobPayload(job)); + }); + + jobsQueue.awaitAll(function (err, jobsCreated) { + if (err) { + return done(err); + } + + var statuses = jobsCreated.map(function(job) { + return job.status; + }); + assert.deepEqual(statuses, expectedStatus); + + return done(); + }); + }); + + it('should perform two multiquery job with two queries for each one and fail the second one', function (done) { + var self = this; + + var jobs = [ + [ + 'select pg_sleep(0)', + 'select pg_sleep(0)' + ], + [ + 'select pg_sleep(0)', + 'select shouldFail()' + ] + ]; + + var expectedStatus = [JobStatus.DONE, JobStatus.FAILED]; + var jobsQueue = queue(2); + + jobs.forEach(function(job) { + jobsQueue.defer(function(payload, done) { + self.batchTestClient.createJob(payload, function(err, jobResult) { + if (err) { + return done(err); + } + jobResult.getStatus(done); + }); + }, jobPayload(job)); + }); + + jobsQueue.awaitAll(function (err, jobsCreated) { + if (err) { + return done(err); + } + + var statuses = jobsCreated.map(function(job) { + return job.status; + }); + assert.deepEqual(statuses, expectedStatus); + + return done(); + }); + }); +}); diff --git a/test/acceptance/batch/batch.test.js b/test/acceptance/batch/batch.test.js new file mode 100644 index 000000000..78c6a8843 --- /dev/null +++ b/test/acceptance/batch/batch.test.js @@ -0,0 +1,209 @@ +require('../../helper'); + +var assert = require('../../support/assert'); +var queue = require('queue-async'); +var BatchTestClient = require('../../support/batch-test-client'); +var JobStatus = require('../../../batch/job_status'); + +describe('batch happy cases', function() { + + before(function() { + this.batchTestClient = new BatchTestClient(); + }); + + after(function(done) { + this.batchTestClient.drain(done); + }); + + function jobPayload(query) { + return { + query: query + }; + } + + it('should perform job with select', function (done) { + var payload = jobPayload('select * from private_table'); + this.batchTestClient.createJob(payload, function(err, jobResult) { + if (err) { + return done(err); + } + jobResult.getStatus(function (err, job) { + if (err) { + return done(err); + } + assert.equal(job.status, JobStatus.DONE); + return done(); + }); + }); + }); + + it('should perform job with select into', function (done) { + var payload = jobPayload('select * into batch_test_table from (select * from private_table) as job'); + this.batchTestClient.createJob(payload, function(err, jobResult) { + if (err) { + return done(err); + } + jobResult.getStatus(function (err, job) { + if (err) { + return done(err); + } + assert.equal(job.status, JobStatus.DONE); + return done(); + }); + }); + }); + + it('should perform job with select from result table', function (done) { + var payload = jobPayload('select * from batch_test_table'); + this.batchTestClient.createJob(payload, function(err, jobResult) { + if (err) { + return done(err); + } + jobResult.getStatus(function (err, job) { + if (err) { + return done(err); + } + assert.equal(job.status, JobStatus.DONE); + return done(); + }); + }); + }); + + it('should perform all enqueued jobs', function (done) { + var self = this; + + var jobs = [ + 'select * from private_table', + 'select * from private_table', + 'select * from private_table', + 'select * from private_table', + 'select * from private_table', + 'select * from private_table', + 'select * from private_table', + 'select * from private_table', + 'select * from private_table', + 'select * from private_table' + ]; + + var jobsQueue = queue(4); + + jobs.forEach(function(job) { + jobsQueue.defer(function(payload, done) { + self.batchTestClient.createJob(payload, function(err, jobResult) { + if (err) { + return done(err); + } + jobResult.getStatus(done); + }); + }, jobPayload(job)); + }); + + jobsQueue.awaitAll(function (err, jobsCreated) { + if (err) { + return done(err); + } + + jobsCreated.forEach(function(job) { + assert.equal(job.status, JobStatus.DONE); + }); + + return done(); + }); + }); + + it('should set all job as failed', function (done) { + var self = this; + + var jobs = [ + 'select * from unexistent_table', + 'select * from unexistent_table', + 'select * from unexistent_table', + 'select * from unexistent_table', + 'select * from unexistent_table', + 'select * from unexistent_table', + 'select * from unexistent_table', + 'select * from unexistent_table', + 'select * from unexistent_table', + 'select * from unexistent_table' + ]; + + var jobsQueue = queue(4); + + jobs.forEach(function(job) { + jobsQueue.defer(function(payload, done) { + self.batchTestClient.createJob(payload, function(err, jobResult) { + if (err) { + return done(err); + } + jobResult.getStatus(done); + }); + }, jobPayload(job)); + }); + + jobsQueue.awaitAll(function (err, jobsCreated) { + if (err) { + return done(err); + } + + jobsCreated.forEach(function(job) { + assert.equal(job.status, JobStatus.FAILED); + }); + + return done(); + }); + }); + + it('should perform job with array of select', function (done) { + var queries = ['select * from private_table limit 1', 'select * from private_table']; + + var payload = jobPayload(queries); + this.batchTestClient.createJob(payload, function(err, jobResult) { + if (err) { + return done(err); + } + jobResult.getStatus(function (err, job) { + if (err) { + return done(err); + } + assert.equal(job.status, JobStatus.DONE); + return done(); + }); + }); + }); + + it('should set job as failed if last query fails', function (done) { + var queries = ['select * from private_table', 'select * from undefined_table']; + + var payload = jobPayload(queries); + this.batchTestClient.createJob(payload, function(err, jobResult) { + if (err) { + return done(err); + } + jobResult.getStatus(function (err, job) { + if (err) { + return done(err); + } + assert.equal(job.status, JobStatus.FAILED); + return done(); + }); + }); + }); + + it('should set job as failed if first query fails', function (done) { + var queries = ['select * from undefined_table', 'select * from private_table']; + + var payload = jobPayload(queries); + this.batchTestClient.createJob(payload, function(err, jobResult) { + if (err) { + return done(err); + } + jobResult.getStatus(function (err, job) { + if (err) { + return done(err); + } + assert.equal(job.status, JobStatus.FAILED); + return done(); + }); + }); + }); +}); diff --git a/test/acceptance/batch/batch.wip.test.js b/test/acceptance/batch/batch.wip.test.js new file mode 100644 index 000000000..c5e03aa53 --- /dev/null +++ b/test/acceptance/batch/batch.wip.test.js @@ -0,0 +1,77 @@ +require('../../helper'); + +var assert = require('../../support/assert'); +var BatchTestClient = require('../../support/batch-test-client'); + +describe('batch work in progress endpoint happy cases', function() { + + before(function() { + this.batchTestClient = new BatchTestClient(); + }); + + after(function(done) { + this.batchTestClient.drain(done); + }); + + function jobPayload(query) { + return { + query: query + }; + } + + it('should get a list of work in progress jobs group by user', function (done) { + var self = this; + var user = 'vizzuality'; + var queries = ['select pg_sleep(0.5)']; + var payload = jobPayload(queries); + + self.batchTestClient.createJob(payload, function(err, jobResult) { + if (err) { + return done(err); + } + + setTimeout(function () { + self.batchTestClient.getWorkInProgressJobs(function (err, workInProgressJobs) { + if (err) { + return done(err); + } + assert.ok(Array.isArray(workInProgressJobs[user])); + assert.ok(workInProgressJobs[user].length >= 1); + for (var i = 0; i < workInProgressJobs[user].length; i++) { + if (workInProgressJobs[user][i] === jobResult.job.job_id) { + return done(); + } + } + }); + }, 100); + }); + }); + + it('should get a list of work in progress jobs w/o the finished ones', function (done) { + var self = this; + var user = 'vizzuality'; + var queries = ['select pg_sleep(0.1)']; + var payload = jobPayload(queries); + + self.batchTestClient.createJob(payload, function(err, jobResult) { + if (err) { + return done(err); + } + setTimeout(function () { + self.batchTestClient.getWorkInProgressJobs(function (err, workInProgressJobs) { + if (err) { + return done(err); + } + assert.ok(Array.isArray(workInProgressJobs[user])); + assert.ok(workInProgressJobs[user].length >= 1); + for (var i = 0; i < workInProgressJobs[user].length; i++) { + if (workInProgressJobs[user][i] === jobResult.job.job_id) { + return done(new Error('Job should not be in work-in-progress list')); + } + } + return done(); + }); + }, 200); + }); + }); +}); diff --git a/test/acceptance/batch/job.callback-template.test.js b/test/acceptance/batch/job.callback-template.test.js new file mode 100644 index 000000000..0bb41b1e1 --- /dev/null +++ b/test/acceptance/batch/job.callback-template.test.js @@ -0,0 +1,219 @@ +require('../../helper'); + +var assert = require('../../support/assert'); +var redisUtils = require('../../support/redis_utils'); +var server = require('../../../app/server')(); +var querystring = require('qs'); +var metadataBackend = require('cartodb-redis')({ pool: redisUtils.getPool() }); +var batchFactory = require('../../../batch/index'); +var jobStatus = require('../../../batch/job_status'); + +describe('Batch API callback templates', function () { + + function createJob(jobDefinition, callback) { + assert.response(server, { + url: '/api/v2/sql/job?api_key=1234', + headers: { + 'Content-Type': 'application/x-www-form-urlencoded', + host: 'vizzuality.cartodb.com' + }, + method: 'POST', + data: querystring.stringify(jobDefinition) + }, { + status: 201 + }, function (err, res) { + if (err) { + return callback(err); + } + return callback(null, JSON.parse(res.body)); + }); + } + + function getJobStatus(jobId, callback) { + assert.response(server, { + url: '/api/v2/sql/job/' + jobId + '?api_key=1234&', + headers: { + host: 'vizzuality.cartodb.com' + }, + method: 'GET' + }, { + status: 200 + }, function (err, res) { + if (err) { + return callback(err); + } + return callback(null, JSON.parse(res.body)); + }); + } + + function getQueryResult(query, callback) { + assert.response(server, { + url: '/api/v2/sql?' + querystring.stringify({q: query, api_key: 1234}), + headers: { + host: 'vizzuality.cartodb.com' + }, + method: 'GET' + }, { + status: 200 + }, function (err, res) { + if (err) { + return callback(err); + } + return callback(null, JSON.parse(res.body)); + }); + } + + function validateExpectedResponse(actual, expected) { + actual.query.forEach(function(actualQuery, index) { + var expectedQuery = expected.query[index]; + assert.ok(expectedQuery); + Object.keys(expectedQuery).forEach(function(expectedKey) { + assert.equal( + actualQuery[expectedKey], + expectedQuery[expectedKey], + 'Expected value for key "' + expectedKey + '" does not match: ' + actualQuery[expectedKey] + ' ==' + + expectedQuery[expectedKey] + ' at query index=' + index + '. Full response: ' + + JSON.stringify(actual, null, 4) + ); + }); + var propsToCheckDate = ['started_at', 'ended_at']; + propsToCheckDate.forEach(function(propToCheckDate) { + if (actualQuery.hasOwnProperty(propToCheckDate)) { + assert.ok(new Date(actualQuery[propToCheckDate])); + } + }); + }); + + assert.equal(actual.onsuccess, expected.onsuccess); + assert.equal(actual.onerror, expected.onerror); + } + + var batch = batchFactory(metadataBackend, redisUtils.getPool()); + + before(function (done) { + batch.start(); + batch.on('ready', done); + }); + + after(function (done) { + batch.stop(); + redisUtils.clean('batch:*', done); + }); + + describe.skip('should use templates for error_message and job_id onerror callback', function () { + var jobResponse; + before(function(done) { + getQueryResult('create table test_batch_errors (job_id text, error_message text)', function(err) { + if (err) { + return done(err); + } + createJob({ + "query": { + "query": [ + { + "query": "SELECT * FROM invalid_table", + "onerror": "INSERT INTO test_batch_errors " + + "values ('<%= job_id %>', '<%= error_message %>')" + } + ] + } + }, function(err, job) { + jobResponse = job; + return done(err); + }); + }); + }); + + it('should keep the original templated query but use the error message', function (done) { + var expectedQuery = { + query: [ + { + "query": "SELECT * FROM invalid_table", + "onerror": "INSERT INTO test_batch_errors values ('<%= job_id %>', '<%= error_message %>')", + status: 'failed', + fallback_status: 'done' + } + ] + }; + + var interval = setInterval(function () { + getJobStatus(jobResponse.job_id, function(err, job) { + if (job.status === jobStatus.FAILED) { + clearInterval(interval); + validateExpectedResponse(job.query, expectedQuery); + getQueryResult('select * from test_batch_errors', function(err, result) { + if (err) { + return done(err); + } + assert.equal(result.rows[0].job_id, jobResponse.job_id); + assert.equal(result.rows[0].error_message, 'relation "invalid_table" does not exist'); + getQueryResult('drop table test_batch_errors', done); + }); + } else if (job.status === jobStatus.DONE || job.status === jobStatus.CANCELLED) { + clearInterval(interval); + done(new Error('Job ' + job.job_id + ' is ' + job.status + ', expected to be "failed"')); + } + }); + }, 50); + }); + }); + + describe('should use template for job_id onsuccess callback', function () { + var jobResponse; + before(function(done) { + createJob({ + "query": { + "query": [ + { + query: "create table batch_jobs (job_id text)" + }, + { + "query": "SELECT 1", + "onsuccess": "INSERT INTO batch_jobs values ('<%= job_id %>')" + } + ] + } + }, function(err, job) { + jobResponse = job; + return done(err); + }); + }); + + it('should keep the original templated query but use the job_id', function (done) { + var expectedQuery = { + query: [ + { + query: "create table batch_jobs (job_id text)", + status: 'done' + }, + { + query: "SELECT 1", + onsuccess: "INSERT INTO batch_jobs values ('<%= job_id %>')", + status: 'done', + fallback_status: 'done' + } + ] + }; + + var interval = setInterval(function () { + getJobStatus(jobResponse.job_id, function(err, job) { + if (job.status === jobStatus.DONE) { + clearInterval(interval); + validateExpectedResponse(job.query, expectedQuery); + getQueryResult('select * from batch_jobs', function(err, result) { + if (err) { + return done(err); + } + assert.equal(result.rows[0].job_id, jobResponse.job_id); + getQueryResult('drop table batch_jobs', done); + }); + } else if (job.status === jobStatus.FAILED || job.status === jobStatus.CANCELLED) { + clearInterval(interval); + done(new Error('Job ' + job.job_id + ' is ' + job.status + ', expected to be "done"')); + } + }); + }, 50); + }); + }); + +}); diff --git a/test/acceptance/job.fallback.test.js b/test/acceptance/batch/job.fallback.test.js similarity index 70% rename from test/acceptance/job.fallback.test.js rename to test/acceptance/batch/job.fallback.test.js index 83fd904ed..bfa5468e9 100644 --- a/test/acceptance/job.fallback.test.js +++ b/test/acceptance/batch/job.fallback.test.js @@ -1,39 +1,51 @@ -require('../helper'); +require('../../helper'); -var assert = require('../support/assert'); -var app = require(global.settings.app_root + '/app/app')(); +var assert = require('../../support/assert'); +var redisUtils = require('../../support/redis_utils'); +var server = require('../../../app/server')(); var querystring = require('qs'); -var metadataBackend = require('cartodb-redis')({ - host: global.settings.redis_host, - port: global.settings.redis_port, - max: global.settings.redisPool, - idleTimeoutMillis: global.settings.redisIdleTimeoutMillis, - reapIntervalMillis: global.settings.redisReapIntervalMillis -}); -var batchFactory = require('../../batch'); -var jobStatus = require('../../batch/job_status'); +var metadataBackend = require('cartodb-redis')({ pool: redisUtils.getPool() }); +var batchFactory = require('../../../batch/index'); +var jobStatus = require('../../../batch/job_status'); describe('Batch API fallback job', function () { - var batch = batchFactory(metadataBackend); + function validateExpectedResponse(actual, expected) { + actual.query.forEach(function(actualQuery, index) { + var expectedQuery = expected.query[index]; + assert.ok(expectedQuery); + Object.keys(expectedQuery).forEach(function(expectedKey) { + assert.equal(actualQuery[expectedKey], expectedQuery[expectedKey]); + }); + var propsToCheckDate = ['started_at', 'ended_at']; + propsToCheckDate.forEach(function(propToCheckDate) { + if (actualQuery.hasOwnProperty(propToCheckDate)) { + assert.ok(new Date(actualQuery[propToCheckDate])); + } + }); + }); + + assert.equal(actual.onsuccess, expected.onsuccess); + assert.equal(actual.onerror, expected.onerror); + } + + var batch = batchFactory(metadataBackend, redisUtils.getPool()); - before(function () { + before(function (done) { batch.start(); + batch.on('ready', done); }); after(function (done) { batch.stop(); - batch.drain(function () { - metadataBackend.redisCmd(5, 'DEL', [ 'batch:queues:localhost' ], done); - }); + redisUtils.clean('batch:*', done); }); describe('"onsuccess" on first query should be triggered', function () { var fallbackJob = {}; - it('should create a job', function (done) { - assert.response(app, { + assert.response(server, { url: '/api/v2/sql/job?api_key=1234', headers: { 'Content-Type': 'application/x-www-form-urlencoded', @@ -50,7 +62,7 @@ describe('Batch API fallback job', function () { }) }, { status: 201 - }, function (res, err) { + }, function (err, res) { if (err) { return done(err); } @@ -70,7 +82,7 @@ describe('Batch API fallback job', function () { }; var interval = setInterval(function () { - assert.response(app, { + assert.response(server, { url: '/api/v2/sql/job/' + fallbackJob.job_id + '?api_key=1234&', headers: { 'Content-Type': 'application/x-www-form-urlencoded', @@ -79,14 +91,14 @@ describe('Batch API fallback job', function () { method: 'GET' }, { status: 200 - }, function (res, err) { + }, function (err, res) { if (err) { return done(err); } var job = JSON.parse(res.body); if (job.status === jobStatus.DONE) { clearInterval(interval); - assert.deepEqual(job.query, expectedQuery); + validateExpectedResponse(job.query, expectedQuery); done(); } else if (job.status === jobStatus.FAILED || job.status === jobStatus.CANCELLED) { clearInterval(interval); @@ -101,7 +113,7 @@ describe('Batch API fallback job', function () { var fallbackJob = {}; it('should create a job', function (done) { - assert.response(app, { + assert.response(server, { url: '/api/v2/sql/job?api_key=1234', headers: { 'Content-Type': 'application/x-www-form-urlencoded', @@ -118,7 +130,7 @@ describe('Batch API fallback job', function () { }) }, { status: 201 - }, function (res, err) { + }, function (err, res) { if (err) { return done(err); } @@ -133,11 +145,11 @@ describe('Batch API fallback job', function () { "query": "SELECT * FROM untitle_table_4", "onerror": "SELECT * FROM untitle_table_4 limit 1", "status": "done", - "fallback_status": "pending" + "fallback_status": "skipped" }] }; var interval = setInterval(function () { - assert.response(app, { + assert.response(server, { url: '/api/v2/sql/job/' + fallbackJob.job_id + '?api_key=1234&', headers: { 'Content-Type': 'application/x-www-form-urlencoded', @@ -146,14 +158,14 @@ describe('Batch API fallback job', function () { method: 'GET' }, { status: 200 - }, function (res, err) { + }, function (err, res) { if (err) { return done(err); } var job = JSON.parse(res.body); if (job.status === jobStatus.DONE) { clearInterval(interval); - assert.deepEqual(job.query, expectedQuery); + validateExpectedResponse(job.query, expectedQuery); done(); } else if (job.status === jobStatus.FAILED || job.status === jobStatus.CANCELLED) { clearInterval(interval); @@ -168,7 +180,7 @@ describe('Batch API fallback job', function () { var fallbackJob = {}; it('should create a job', function (done) { - assert.response(app, { + assert.response(server, { url: '/api/v2/sql/job?api_key=1234', headers: { 'Content-Type': 'application/x-www-form-urlencoded', @@ -185,7 +197,7 @@ describe('Batch API fallback job', function () { }) }, { status: 201 - }, function (res, err) { + }, function (err, res) { if (err) { return done(err); } @@ -205,7 +217,7 @@ describe('Batch API fallback job', function () { }] }; var interval = setInterval(function () { - assert.response(app, { + assert.response(server, { url: '/api/v2/sql/job/' + fallbackJob.job_id + '?api_key=1234&', headers: { 'Content-Type': 'application/x-www-form-urlencoded', @@ -214,14 +226,14 @@ describe('Batch API fallback job', function () { method: 'GET' }, { status: 200 - }, function (res, err) { + }, function (err, res) { if (err) { return done(err); } var job = JSON.parse(res.body); if (job.status === jobStatus.FAILED) { clearInterval(interval); - assert.deepEqual(job.query, expectedQuery); + validateExpectedResponse(job.query, expectedQuery); done(); } else if (job.status === jobStatus.DONE || job.status === jobStatus.CANCELLED) { clearInterval(interval); @@ -236,7 +248,7 @@ describe('Batch API fallback job', function () { var fallbackJob = {}; it('should create a job', function (done) { - assert.response(app, { + assert.response(server, { url: '/api/v2/sql/job?api_key=1234', headers: { 'Content-Type': 'application/x-www-form-urlencoded', @@ -253,7 +265,7 @@ describe('Batch API fallback job', function () { }) }, { status: 201 - }, function (res, err) { + }, function (err, res) { if (err) { return done(err); } @@ -268,13 +280,13 @@ describe('Batch API fallback job', function () { query: 'SELECT * FROM nonexistent_table /* query should fail */', onsuccess: 'SELECT * FROM untitle_table_4 limit 1', status: 'failed', - fallback_status: 'pending', + fallback_status: 'skipped', failed_reason: 'relation "nonexistent_table" does not exist' }] }; var interval = setInterval(function () { - assert.response(app, { + assert.response(server, { url: '/api/v2/sql/job/' + fallbackJob.job_id + '?api_key=1234&', headers: { 'Content-Type': 'application/x-www-form-urlencoded', @@ -283,14 +295,14 @@ describe('Batch API fallback job', function () { method: 'GET' }, { status: 200 - }, function (res, err) { + }, function (err, res) { if (err) { return done(err); } var job = JSON.parse(res.body); if (job.status === jobStatus.FAILED) { clearInterval(interval); - assert.deepEqual(job.query, expectedQuery); + validateExpectedResponse(job.query, expectedQuery); done(); } else if (job.status === jobStatus.DONE || job.status === jobStatus.CANCELLED) { clearInterval(interval); @@ -306,7 +318,7 @@ describe('Batch API fallback job', function () { var fallbackJob = {}; it('should create a job', function (done) { - assert.response(app, { + assert.response(server, { url: '/api/v2/sql/job?api_key=1234', headers: { 'Content-Type': 'application/x-www-form-urlencoded', @@ -323,7 +335,7 @@ describe('Batch API fallback job', function () { }) }, { status: 201 - }, function (res, err) { + }, function (err, res) { if (err) { return done(err); } @@ -342,7 +354,7 @@ describe('Batch API fallback job', function () { }; var interval = setInterval(function () { - assert.response(app, { + assert.response(server, { url: '/api/v2/sql/job/' + fallbackJob.job_id + '?api_key=1234&', headers: { 'Content-Type': 'application/x-www-form-urlencoded', @@ -351,14 +363,14 @@ describe('Batch API fallback job', function () { method: 'GET' }, { status: 200 - }, function (res, err) { + }, function (err, res) { if (err) { return done(err); } var job = JSON.parse(res.body); if (job.status === jobStatus.DONE && job.fallback_status === jobStatus.DONE) { clearInterval(interval); - assert.deepEqual(job.query, expectedQuery); + validateExpectedResponse(job.query, expectedQuery); done(); } else if (job.status === jobStatus.FAILED || job.status === jobStatus.CANCELLED) { clearInterval(interval); @@ -373,7 +385,7 @@ describe('Batch API fallback job', function () { var fallbackJob = {}; it('should create a job', function (done) { - assert.response(app, { + assert.response(server, { url: '/api/v2/sql/job?api_key=1234', headers: { 'Content-Type': 'application/x-www-form-urlencoded', @@ -390,7 +402,7 @@ describe('Batch API fallback job', function () { }) }, { status: 201 - }, function (res, err) { + }, function (err, res) { if (err) { return done(err); } @@ -410,7 +422,7 @@ describe('Batch API fallback job', function () { }; var interval = setInterval(function () { - assert.response(app, { + assert.response(server, { url: '/api/v2/sql/job/' + fallbackJob.job_id + '?api_key=1234&', headers: { 'Content-Type': 'application/x-www-form-urlencoded', @@ -419,14 +431,14 @@ describe('Batch API fallback job', function () { method: 'GET' }, { status: 200 - }, function (res, err) { + }, function (err, res) { if (err) { return done(err); } var job = JSON.parse(res.body); - if (job.status === jobStatus.FAILED && job.fallback_status === jobStatus.PENDING) { + if (job.status === jobStatus.FAILED && job.fallback_status === jobStatus.SKIPPED) { clearInterval(interval); - assert.deepEqual(job.query, expectedQuery); + validateExpectedResponse(job.query, expectedQuery); done(); } else if (job.status === jobStatus.FAILED || job.status === jobStatus.CANCELLED) { clearInterval(interval); @@ -442,7 +454,7 @@ describe('Batch API fallback job', function () { var fallbackJob = {}; it('should create a job', function (done) { - assert.response(app, { + assert.response(server, { url: '/api/v2/sql/job?api_key=1234', headers: { 'Content-Type': 'application/x-www-form-urlencoded', @@ -459,7 +471,7 @@ describe('Batch API fallback job', function () { }) }, { status: 201 - }, function (res, err) { + }, function (err, res) { if (err) { return done(err); } @@ -479,7 +491,7 @@ describe('Batch API fallback job', function () { }; var interval = setInterval(function () { - assert.response(app, { + assert.response(server, { url: '/api/v2/sql/job/' + fallbackJob.job_id + '?api_key=1234&', headers: { 'Content-Type': 'application/x-www-form-urlencoded', @@ -488,14 +500,14 @@ describe('Batch API fallback job', function () { method: 'GET' }, { status: 200 - }, function (res, err) { + }, function (err, res) { if (err) { return done(err); } var job = JSON.parse(res.body); if (job.status === jobStatus.FAILED && job.fallback_status === jobStatus.DONE) { clearInterval(interval); - assert.deepEqual(job.query, expectedQuery); + validateExpectedResponse(job.query, expectedQuery); done(); } else if (job.status === jobStatus.DONE || job.status === jobStatus.CANCELLED) { clearInterval(interval); @@ -510,7 +522,7 @@ describe('Batch API fallback job', function () { var fallbackJob = {}; it('should create a job', function (done) { - assert.response(app, { + assert.response(server, { url: '/api/v2/sql/job?api_key=1234', headers: { 'Content-Type': 'application/x-www-form-urlencoded', @@ -527,7 +539,7 @@ describe('Batch API fallback job', function () { }) }, { status: 201 - }, function (res, err) { + }, function (err, res) { if (err) { return done(err); } @@ -546,7 +558,7 @@ describe('Batch API fallback job', function () { }; var interval = setInterval(function () { - assert.response(app, { + assert.response(server, { url: '/api/v2/sql/job/' + fallbackJob.job_id + '?api_key=1234&', headers: { 'Content-Type': 'application/x-www-form-urlencoded', @@ -555,14 +567,14 @@ describe('Batch API fallback job', function () { method: 'GET' }, { status: 200 - }, function (res, err) { + }, function (err, res) { if (err) { return done(err); } var job = JSON.parse(res.body); - if (job.status === jobStatus.DONE && job.fallback_status === jobStatus.PENDING) { + if (job.status === jobStatus.DONE && job.fallback_status === jobStatus.SKIPPED) { clearInterval(interval); - assert.deepEqual(job.query, expectedQuery); + validateExpectedResponse(job.query, expectedQuery); done(); } else if (job.status === jobStatus.FAILED || job.status === jobStatus.CANCELLED) { clearInterval(interval); @@ -578,7 +590,7 @@ describe('Batch API fallback job', function () { var fallbackJob = {}; it('should create a job', function (done) { - assert.response(app, { + assert.response(server, { url: '/api/v2/sql/job?api_key=1234', headers: { 'Content-Type': 'application/x-www-form-urlencoded', @@ -596,7 +608,7 @@ describe('Batch API fallback job', function () { }) }, { status: 201 - }, function (res, err) { + }, function (err, res) { if (err) { return done(err); } @@ -617,7 +629,7 @@ describe('Batch API fallback job', function () { }; var interval = setInterval(function () { - assert.response(app, { + assert.response(server, { url: '/api/v2/sql/job/' + fallbackJob.job_id + '?api_key=1234&', headers: { 'Content-Type': 'application/x-www-form-urlencoded', @@ -626,14 +638,14 @@ describe('Batch API fallback job', function () { method: 'GET' }, { status: 200 - }, function (res, err) { + }, function (err, res) { if (err) { return done(err); } var job = JSON.parse(res.body); if (job.status === jobStatus.DONE && job.fallback_status === jobStatus.DONE) { clearInterval(interval); - assert.deepEqual(job.query, expectedQuery); + validateExpectedResponse(job.query, expectedQuery); done(); } else if (job.status === jobStatus.FAILED || job.status === jobStatus.CANCELLED) { clearInterval(interval); @@ -648,7 +660,7 @@ describe('Batch API fallback job', function () { var fallbackJob = {}; it('should create a job', function (done) { - assert.response(app, { + assert.response(server, { url: '/api/v2/sql/job?api_key=1234', headers: { 'Content-Type': 'application/x-www-form-urlencoded', @@ -668,7 +680,7 @@ describe('Batch API fallback job', function () { }) }, { status: 201 - }, function (res, err) { + }, function (err, res) { if (err) { return done(err); } @@ -693,7 +705,7 @@ describe('Batch API fallback job', function () { }; var interval = setInterval(function () { - assert.response(app, { + assert.response(server, { url: '/api/v2/sql/job/' + fallbackJob.job_id + '?api_key=1234&', headers: { 'Content-Type': 'application/x-www-form-urlencoded', @@ -702,14 +714,14 @@ describe('Batch API fallback job', function () { method: 'GET' }, { status: 200 - }, function (res, err) { + }, function (err, res) { if (err) { return done(err); } var job = JSON.parse(res.body); if (job.status === jobStatus.DONE) { clearInterval(interval); - assert.deepEqual(job.query, expectedQuery); + validateExpectedResponse(job.query, expectedQuery); done(); } else if (job.status === jobStatus.FAILED || job.status === jobStatus.CANCELLED) { clearInterval(interval); @@ -724,7 +736,7 @@ describe('Batch API fallback job', function () { var fallbackJob = {}; it('should create a job', function (done) { - assert.response(app, { + assert.response(server, { url: '/api/v2/sql/job?api_key=1234', headers: { 'Content-Type': 'application/x-www-form-urlencoded', @@ -744,7 +756,7 @@ describe('Batch API fallback job', function () { }) }, { status: 201 - }, function (res, err) { + }, function (err, res) { if (err) { return done(err); } @@ -759,18 +771,18 @@ describe('Batch API fallback job', function () { "query": "SELECT * FROM nonexistent_table /* should fail */", "onsuccess": "SELECT * FROM untitle_table_4 limit 1", "status": "failed", - "fallback_status": "pending", + "fallback_status": "skipped", "failed_reason": 'relation "nonexistent_table" does not exist' }, { "query": "SELECT * FROM untitle_table_4 limit 2", "onsuccess": "SELECT * FROM untitle_table_4 limit 3", - "status": "pending", - "fallback_status": "pending" + "status": "skipped", + "fallback_status": "skipped" }] }; var interval = setInterval(function () { - assert.response(app, { + assert.response(server, { url: '/api/v2/sql/job/' + fallbackJob.job_id + '?api_key=1234&', headers: { 'Content-Type': 'application/x-www-form-urlencoded', @@ -779,14 +791,14 @@ describe('Batch API fallback job', function () { method: 'GET' }, { status: 200 - }, function (res, err) { + }, function (err, res) { if (err) { return done(err); } var job = JSON.parse(res.body); if (job.status === jobStatus.FAILED) { clearInterval(interval); - assert.deepEqual(job.query, expectedQuery); + validateExpectedResponse(job.query, expectedQuery); done(); } else if (job.status === jobStatus.DONE || job.status === jobStatus.CANCELLED) { clearInterval(interval); @@ -802,7 +814,7 @@ describe('Batch API fallback job', function () { var fallbackJob = {}; it('should create a job', function (done) { - assert.response(app, { + assert.response(server, { url: '/api/v2/sql/job?api_key=1234', headers: { 'Content-Type': 'application/x-www-form-urlencoded', @@ -822,7 +834,7 @@ describe('Batch API fallback job', function () { }) }, { status: 201 - }, function (res, err) { + }, function (err, res) { if (err) { return done(err); } @@ -842,13 +854,13 @@ describe('Batch API fallback job', function () { "query": "SELECT * FROM nonexistent_table /* should fail */", "onsuccess": "SELECT * FROM untitle_table_4 limit 3", "status": "failed", - "fallback_status": "pending", + "fallback_status": "skipped", "failed_reason": 'relation "nonexistent_table" does not exist' }] }; var interval = setInterval(function () { - assert.response(app, { + assert.response(server, { url: '/api/v2/sql/job/' + fallbackJob.job_id + '?api_key=1234&', headers: { 'Content-Type': 'application/x-www-form-urlencoded', @@ -857,14 +869,14 @@ describe('Batch API fallback job', function () { method: 'GET' }, { status: 200 - }, function (res, err) { + }, function (err, res) { if (err) { return done(err); } var job = JSON.parse(res.body); if (job.status === jobStatus.FAILED) { clearInterval(interval); - assert.deepEqual(job.query, expectedQuery); + validateExpectedResponse(job.query, expectedQuery); done(); } else if (job.status === jobStatus.DONE || job.status === jobStatus.CANCELLED) { clearInterval(interval); @@ -875,12 +887,225 @@ describe('Batch API fallback job', function () { }); }); + describe('"onerror" should not be triggered for any query and "skipped"', function () { + var fallbackJob = {}; + + it('should create a job', function (done) { + assert.response(server, { + url: '/api/v2/sql/job?api_key=1234', + headers: { + 'Content-Type': 'application/x-www-form-urlencoded', + 'host': 'vizzuality.cartodb.com' + }, + method: 'POST', + data: querystring.stringify({ + query: { + query: [{ + query: "SELECT * FROM untitle_table_4 limit 1", + onerror: "SELECT * FROM untitle_table_4 limit 2" + }, { + query: "SELECT * FROM untitle_table_4 limit 3", + onerror: "SELECT * FROM untitle_table_4 limit 4" + }] + } + }) + }, { + status: 201 + }, function (err, res) { + if (err) { + return done(err); + } + fallbackJob = JSON.parse(res.body); + done(); + }); + }); + + it('job should be failed', function (done) { + var expectedQuery = { + query: [{ + query: 'SELECT * FROM untitle_table_4 limit 1', + onerror: 'SELECT * FROM untitle_table_4 limit 2', + status: 'done', + fallback_status: 'skipped' + }, { + query: 'SELECT * FROM untitle_table_4 limit 3', + onerror: 'SELECT * FROM untitle_table_4 limit 4', + status: 'done', + fallback_status: 'skipped' + }] + }; + + var interval = setInterval(function () { + assert.response(server, { + url: '/api/v2/sql/job/' + fallbackJob.job_id + '?api_key=1234&', + headers: { + 'Content-Type': 'application/x-www-form-urlencoded', + 'host': 'vizzuality.cartodb.com' + }, + method: 'GET' + }, { + status: 200 + }, function (err, res) { + if (err) { + return done(err); + } + var job = JSON.parse(res.body); + if (job.status === jobStatus.DONE) { + clearInterval(interval); + validateExpectedResponse(job.query, expectedQuery); + done(); + } else if (job.status === jobStatus.FAILED || job.status === jobStatus.CANCELLED) { + clearInterval(interval); + done(new Error('Job ' + job.job_id + ' is ' + job.status + ', expected to be done')); + } + }); + }, 50); + }); + }); + + describe('"onsuccess" should be "skipped"', function () { + var fallbackJob = {}; + + it('should create a job', function (done) { + assert.response(server, { + url: '/api/v2/sql/job?api_key=1234', + headers: { + 'Content-Type': 'application/x-www-form-urlencoded', + 'host': 'vizzuality.cartodb.com' + }, + method: 'POST', + data: querystring.stringify({ + query: { + query: [{ + query: "SELECT * FROM untitle_table_4 limit 1, /* should fail */", + onsuccess: "SELECT * FROM untitle_table_4 limit 2" + }] + } + }) + }, { + status: 201 + }, function (err, res) { + if (err) { + return done(err); + } + fallbackJob = JSON.parse(res.body); + done(); + }); + }); + + it('job should be failed', function (done) { + var expectedQuery = { + query: [{ + query: 'SELECT * FROM untitle_table_4 limit 1, /* should fail */', + onsuccess: 'SELECT * FROM untitle_table_4 limit 2', + status: 'failed', + fallback_status: 'skipped', + failed_reason: 'syntax error at end of input' + }] + }; + + var interval = setInterval(function () { + assert.response(server, { + url: '/api/v2/sql/job/' + fallbackJob.job_id + '?api_key=1234&', + headers: { + 'Content-Type': 'application/x-www-form-urlencoded', + 'host': 'vizzuality.cartodb.com' + }, + method: 'GET' + }, { + status: 200 + }, function (err, res) { + if (err) { + return done(err); + } + var job = JSON.parse(res.body); + if (job.status === jobStatus.FAILED) { + clearInterval(interval); + validateExpectedResponse(job.query, expectedQuery); + done(); + } else if (job.status === jobStatus.DONE || job.status === jobStatus.CANCELLED) { + clearInterval(interval); + done(new Error('Job ' + job.job_id + ' is ' + job.status + ', expected to be failed')); + } + }); + }, 50); + }); + }); + + + describe('"onsuccess" should not be triggered and "skipped"', function () { + var fallbackJob = {}; + + it('should create a job', function (done) { + assert.response(server, { + url: '/api/v2/sql/job?api_key=1234', + headers: { + 'Content-Type': 'application/x-www-form-urlencoded', + 'host': 'vizzuality.cartodb.com' + }, + method: 'POST', + data: querystring.stringify({ + query: { + query: [{ + query: "SELECT * FROM untitle_table_4 limit 1, /* should fail */", + }], + onsuccess: "SELECT * FROM untitle_table_4 limit 2" + } + }) + }, { + status: 201 + }, function (err, res) { + if (err) { + return done(err); + } + fallbackJob = JSON.parse(res.body); + done(); + }); + }); + + it('job should be failed', function (done) { + var expectedQuery = { + query: [{ + query: 'SELECT * FROM untitle_table_4 limit 1, /* should fail */', + status: 'failed', + failed_reason: 'syntax error at end of input' + }], + onsuccess: 'SELECT * FROM untitle_table_4 limit 2' + }; + + var interval = setInterval(function () { + assert.response(server, { + url: '/api/v2/sql/job/' + fallbackJob.job_id + '?api_key=1234&', + headers: { + 'Content-Type': 'application/x-www-form-urlencoded', + 'host': 'vizzuality.cartodb.com' + }, + method: 'GET' + }, { + status: 200 + }, function (err, res) { + if (err) { + return done(err); + } + var job = JSON.parse(res.body); + if (job.status === jobStatus.FAILED && job.fallback_status === jobStatus.SKIPPED) { + clearInterval(interval); + validateExpectedResponse(job.query, expectedQuery); + done(); + } else if (job.status === jobStatus.DONE || job.status === jobStatus.CANCELLED) { + clearInterval(interval); + done(new Error('Job ' + job.job_id + ' is ' + job.status + ', expected to be failed')); + } + }); + }, 50); + }); + }); describe('"onsuccess" for first query should fail', function () { var fallbackJob = {}; it('should create a job', function (done) { - assert.response(app, { + assert.response(server, { url: '/api/v2/sql/job?api_key=1234', headers: { 'Content-Type': 'application/x-www-form-urlencoded', @@ -900,7 +1125,7 @@ describe('Batch API fallback job', function () { }) }, { status: 201 - }, function (res, err) { + }, function (err, res) { if (err) { return done(err); } @@ -926,7 +1151,7 @@ describe('Batch API fallback job', function () { }; var interval = setInterval(function () { - assert.response(app, { + assert.response(server, { url: '/api/v2/sql/job/' + fallbackJob.job_id + '?api_key=1234&', headers: { 'Content-Type': 'application/x-www-form-urlencoded', @@ -935,14 +1160,14 @@ describe('Batch API fallback job', function () { method: 'GET' }, { status: 200 - }, function (res, err) { + }, function (err, res) { if (err) { return done(err); } var job = JSON.parse(res.body); if (job.status === jobStatus.DONE) { clearInterval(interval); - assert.deepEqual(job.query, expectedQuery); + validateExpectedResponse(job.query, expectedQuery); done(); } else if (job.status === jobStatus.FAILED || job.status === jobStatus.CANCELLED) { clearInterval(interval); @@ -957,7 +1182,7 @@ describe('Batch API fallback job', function () { var fallbackJob = {}; it('should create a job', function (done) { - assert.response(app, { + assert.response(server, { url: '/api/v2/sql/job?api_key=1234', headers: { 'Content-Type': 'application/x-www-form-urlencoded', @@ -977,7 +1202,7 @@ describe('Batch API fallback job', function () { }) }, { status: 201 - }, function (res, err) { + }, function (err, res) { if (err) { return done(err); } @@ -1003,7 +1228,7 @@ describe('Batch API fallback job', function () { }; var interval = setInterval(function () { - assert.response(app, { + assert.response(server, { url: '/api/v2/sql/job/' + fallbackJob.job_id + '?api_key=1234&', headers: { 'Content-Type': 'application/x-www-form-urlencoded', @@ -1012,14 +1237,14 @@ describe('Batch API fallback job', function () { method: 'GET' }, { status: 200 - }, function (res, err) { + }, function (err, res) { if (err) { return done(err); } var job = JSON.parse(res.body); if (job.status === jobStatus.DONE) { clearInterval(interval); - assert.deepEqual(job.query, expectedQuery); + validateExpectedResponse(job.query, expectedQuery); done(); } else if (job.status === jobStatus.FAILED || job.status === jobStatus.CANCELLED) { clearInterval(interval); @@ -1034,7 +1259,7 @@ describe('Batch API fallback job', function () { var fallbackJob = {}; it('should create a job', function (done) { - assert.response(app, { + assert.response(server, { url: '/api/v2/sql/job?api_key=1234', headers: { 'Content-Type': 'application/x-www-form-urlencoded', @@ -1055,7 +1280,7 @@ describe('Batch API fallback job', function () { }) }, { status: 201 - }, function (res, err) { + }, function (err, res) { if (err) { return done(err); } @@ -1081,7 +1306,7 @@ describe('Batch API fallback job', function () { }; var interval = setInterval(function () { - assert.response(app, { + assert.response(server, { url: '/api/v2/sql/job/' + fallbackJob.job_id + '?api_key=1234&', headers: { 'Content-Type': 'application/x-www-form-urlencoded', @@ -1090,14 +1315,14 @@ describe('Batch API fallback job', function () { method: 'GET' }, { status: 200 - }, function (res, err) { + }, function (err, res) { if (err) { return done(err); } var job = JSON.parse(res.body); if (job.status === jobStatus.DONE && job.fallback_status === jobStatus.DONE) { clearInterval(interval); - assert.deepEqual(job.query, expectedQuery); + validateExpectedResponse(job.query, expectedQuery); done(); } else if (job.status === jobStatus.FAILED || job.status === jobStatus.CANCELLED) { clearInterval(interval); @@ -1113,7 +1338,7 @@ describe('Batch API fallback job', function () { var fallbackJob = {}; it('should create a job', function (done) { - assert.response(app, { + assert.response(server, { url: '/api/v2/sql/job?api_key=1234', headers: { 'Content-Type': 'application/x-www-form-urlencoded', @@ -1134,7 +1359,7 @@ describe('Batch API fallback job', function () { }) }, { status: 201 - }, function (res, err) { + }, function (err, res) { if (err) { return done(err); } @@ -1161,7 +1386,7 @@ describe('Batch API fallback job', function () { }; var interval = setInterval(function () { - assert.response(app, { + assert.response(server, { url: '/api/v2/sql/job/' + fallbackJob.job_id + '?api_key=1234&', headers: { 'Content-Type': 'application/x-www-form-urlencoded', @@ -1170,14 +1395,14 @@ describe('Batch API fallback job', function () { method: 'GET' }, { status: 200 - }, function (res, err) { + }, function (err, res) { if (err) { return done(err); } var job = JSON.parse(res.body); if (job.status === jobStatus.DONE && job.fallback_status === jobStatus.DONE) { clearInterval(interval); - assert.deepEqual(job.query, expectedQuery); + validateExpectedResponse(job.query, expectedQuery); done(); } else if (job.status === jobStatus.FAILED || job.status === jobStatus.CANCELLED) { clearInterval(interval); @@ -1193,7 +1418,7 @@ describe('Batch API fallback job', function () { var fallbackJob = {}; it('should create a job', function (done) { - assert.response(app, { + assert.response(server, { url: '/api/v2/sql/job?api_key=1234', headers: { 'Content-Type': 'application/x-www-form-urlencoded', @@ -1211,7 +1436,7 @@ describe('Batch API fallback job', function () { }) }, { status: 201 - }, function (res, err) { + }, function (err, res) { if (err) { return done(err); } @@ -1232,7 +1457,7 @@ describe('Batch API fallback job', function () { }; var interval = setInterval(function () { - assert.response(app, { + assert.response(server, { url: '/api/v2/sql/job/' + fallbackJob.job_id + '?api_key=1234&', headers: { 'Content-Type': 'application/x-www-form-urlencoded', @@ -1241,20 +1466,20 @@ describe('Batch API fallback job', function () { method: 'GET' }, { status: 200 - }, function (res, err) { + }, function (err, res) { if (err) { return done(err); } var job = JSON.parse(res.body); if (job.status === jobStatus.RUNNING && job.fallback_status === jobStatus.PENDING) { clearInterval(interval); - assert.deepEqual(job.query, expectedQuery); + validateExpectedResponse(job.query, expectedQuery); done(); } else if (job.status === jobStatus.DONE || job.status === jobStatus.FAILED || job.status === jobStatus.CANCELLED) { clearInterval(interval); - done(new Error('Job ' + job.job_id + ' is ' + job.status + ', expected to be done')); + done(new Error('Job ' + job.job_id + ' is ' + job.status + ', expected to be running')); } }); }, 50); @@ -1266,12 +1491,12 @@ describe('Batch API fallback job', function () { "query": "SELECT pg_sleep(3)", "onsuccess": "SELECT pg_sleep(0)", "status": "cancelled", - "fallback_status": "pending" + "fallback_status": "skipped" }], "onsuccess": "SELECT pg_sleep(0)" }; - assert.response(app, { + assert.response(server, { url: '/api/v2/sql/job/' + fallbackJob.job_id + '?api_key=1234&', headers: { 'Content-Type': 'application/x-www-form-urlencoded', @@ -1280,13 +1505,13 @@ describe('Batch API fallback job', function () { method: 'DELETE' }, { status: 200 - }, function (res, err) { + }, function (err, res) { if (err) { return done(err); } var job = JSON.parse(res.body); - if (job.status === jobStatus.CANCELLED && job.fallback_status === jobStatus.PENDING) { - assert.deepEqual(job.query, expectedQuery); + if (job.status === jobStatus.CANCELLED && job.fallback_status === jobStatus.SKIPPED) { + validateExpectedResponse(job.query, expectedQuery); done(); } else if (job.status === jobStatus.DONE || job.status === jobStatus.FAILED) { done(new Error('Job ' + job.job_id + ' is ' + job.status + ', expected to be cancelled')); @@ -1299,7 +1524,7 @@ describe('Batch API fallback job', function () { var fallbackJob = {}; it('should create a job', function (done) { - assert.response(app, { + assert.response(server, { url: '/api/v2/sql/job?api_key=1234', headers: { 'Content-Type': 'application/x-www-form-urlencoded', @@ -1317,7 +1542,7 @@ describe('Batch API fallback job', function () { }) }, { status: 201 - }, function (res, err) { + }, function (err, res) { if (err) { return done(err); } @@ -1338,7 +1563,7 @@ describe('Batch API fallback job', function () { }; var interval = setInterval(function () { - assert.response(app, { + assert.response(server, { url: '/api/v2/sql/job/' + fallbackJob.job_id + '?api_key=1234&', headers: { 'Content-Type': 'application/x-www-form-urlencoded', @@ -1347,7 +1572,7 @@ describe('Batch API fallback job', function () { method: 'GET' }, { status: 200 - }, function (res, err) { + }, function (err, res) { if (err) { return done(err); } @@ -1355,7 +1580,7 @@ describe('Batch API fallback job', function () { if (job.query.query[0].status === jobStatus.DONE && job.query.query[0].fallback_status === jobStatus.RUNNING) { clearInterval(interval); - assert.deepEqual(job.query, expectedQuery); + validateExpectedResponse(job.query, expectedQuery); done(); } else if (job.query.query[0].status === jobStatus.DONE || job.query.query[0].status === jobStatus.FAILED || @@ -1380,7 +1605,7 @@ describe('Batch API fallback job', function () { "onsuccess": "SELECT pg_sleep(0)" }; - assert.response(app, { + assert.response(server, { url: '/api/v2/sql/job/' + fallbackJob.job_id + '?api_key=1234&', headers: { 'Content-Type': 'application/x-www-form-urlencoded', @@ -1389,13 +1614,13 @@ describe('Batch API fallback job', function () { method: 'DELETE' }, { status: 200 - }, function (res, err) { + }, function (err, res) { if (err) { return done(err); } var job = JSON.parse(res.body); - if (job.status === jobStatus.CANCELLED && job.fallback_status === jobStatus.PENDING) { - assert.deepEqual(job.query, expectedQuery); + if (job.status === jobStatus.CANCELLED && job.fallback_status === jobStatus.SKIPPED) { + validateExpectedResponse(job.query, expectedQuery); done(); } else if (job.status === jobStatus.DONE || job.status === jobStatus.FAILED) { done(new Error('Job ' + job.job_id + ' is ' + job.status + ', expected to be cancelled')); @@ -1403,4 +1628,165 @@ describe('Batch API fallback job', function () { }); }); }); + + describe('should fail first "onerror" and job "onerror" and skip the other ones', function () { + var fallbackJob = {}; + + it('should create a job', function (done) { + assert.response(server, { + url: '/api/v2/sql/job?api_key=1234', + headers: { + 'Content-Type': 'application/x-www-form-urlencoded', + 'host': 'vizzuality.cartodb.com' + }, + method: 'POST', + data: querystring.stringify({ + "query": { + "query": [{ + "query": "SELECT * FROM atm_madrid limit 1, should fail", + "onerror": "SELECT * FROM atm_madrid limit 2" + }, { + "query": "SELECT * FROM atm_madrid limit 3", + "onerror": "SELECT * FROM atm_madrid limit 4" + }], + "onerror": "SELECT * FROM atm_madrid limit 5" + } + }) + }, { + status: 201 + }, function (err, res) { + if (err) { + return done(err); + } + fallbackJob = JSON.parse(res.body); + done(); + }); + }); + + it('job should fail', function (done) { + var expectedQuery = { + query: [{ + query: 'SELECT * FROM atm_madrid limit 1, should fail', + onerror: 'SELECT * FROM atm_madrid limit 2', + status: 'failed', + fallback_status: 'failed', + failed_reason: 'relation "atm_madrid" does not exist' + }, { + query: 'SELECT * FROM atm_madrid limit 3', + onerror: 'SELECT * FROM atm_madrid limit 4', + status: 'skipped', + fallback_status: 'skipped' + }], + onerror: 'SELECT * FROM atm_madrid limit 5' + }; + + var interval = setInterval(function () { + assert.response(server, { + url: '/api/v2/sql/job/' + fallbackJob.job_id + '?api_key=1234&', + headers: { + 'Content-Type': 'application/x-www-form-urlencoded', + 'host': 'vizzuality.cartodb.com' + }, + method: 'GET' + }, { + status: 200 + }, function (err, res) { + if (err) { + return done(err); + } + var job = JSON.parse(res.body); + if (job.status === jobStatus.FAILED && job.fallback_status === jobStatus.FAILED) { + clearInterval(interval); + validateExpectedResponse(job.query, expectedQuery); + done(); + } else if (job.status === jobStatus.DONE || job.status === jobStatus.CANCELLED) { + clearInterval(interval); + done(new Error('Job ' + job.job_id + ' is ' + job.status + ', expected to be failed')); + } + }); + }, 50); + }); + }); + + describe('should run first "onerror" and job "onerror" and skip the other ones', function () { + var fallbackJob = {}; + + it('should create a job', function (done) { + assert.response(server, { + url: '/api/v2/sql/job?api_key=1234', + headers: { + 'Content-Type': 'application/x-www-form-urlencoded', + 'host': 'vizzuality.cartodb.com' + }, + method: 'POST', + data: querystring.stringify({ + "query": { + "query": [{ + "query": "SELECT * FROM untitle_table_4 limit 1, should fail", + "onerror": "SELECT * FROM untitle_table_4 limit 2" + }, { + "query": "SELECT * FROM untitle_table_4 limit 3", + "onerror": "SELECT * FROM untitle_table_4 limit 4" + }], + "onerror": "SELECT * FROM untitle_table_4 limit 5" + } + }) + }, { + status: 201 + }, function (err, res) { + if (err) { + return done(err); + } + fallbackJob = JSON.parse(res.body); + done(); + }); + }); + + it('job should fail', function (done) { + var expectedQuery = { + "query": [ + { + "query": "SELECT * FROM untitle_table_4 limit 1, should fail", + "onerror": "SELECT * FROM untitle_table_4 limit 2", + "status": "failed", + "fallback_status": "done", + "failed_reason": "LIMIT #,# syntax is not supported" + }, + { + "query": "SELECT * FROM untitle_table_4 limit 3", + "onerror": "SELECT * FROM untitle_table_4 limit 4", + "status": "skipped", + "fallback_status": "skipped" + } + ], + "onerror": "SELECT * FROM untitle_table_4 limit 5" + }; + + var interval = setInterval(function () { + assert.response(server, { + url: '/api/v2/sql/job/' + fallbackJob.job_id + '?api_key=1234&', + headers: { + 'Content-Type': 'application/x-www-form-urlencoded', + 'host': 'vizzuality.cartodb.com' + }, + method: 'GET' + }, { + status: 200 + }, function (err, res) { + if (err) { + return done(err); + } + var job = JSON.parse(res.body); + if (job.status === jobStatus.FAILED && job.fallback_status === jobStatus.DONE) { + clearInterval(interval); + validateExpectedResponse(job.query, expectedQuery); + done(); + } else if (job.status === jobStatus.DONE || job.status === jobStatus.CANCELLED) { + clearInterval(interval); + done(new Error('Job ' + job.job_id + ' is ' + job.status + ', expected to be failed')); + } + }); + }, 50); + }); + }); }); diff --git a/test/acceptance/job.query.limit.test.js b/test/acceptance/batch/job.query.limit.test.js similarity index 69% rename from test/acceptance/job.query.limit.test.js rename to test/acceptance/batch/job.query.limit.test.js index c9cbceabd..5587ecda1 100644 --- a/test/acceptance/job.query.limit.test.js +++ b/test/acceptance/batch/job.query.limit.test.js @@ -12,19 +12,12 @@ * HSET rails:users:vizzuality database_name cartodb_test_user_1_db * */ -require('../helper'); -var JobController = require('../../app/controllers/job_controller'); - -var app = require(global.settings.app_root + '/app/app')(); -var assert = require('../support/assert'); +require('../../helper'); +var JobController = require('../../../app/controllers/job_controller'); +var redisUtils = require('../../support/redis_utils'); +var server = require('../../../app/server')(); +var assert = require('../../support/assert'); var querystring = require('qs'); -var metadataBackend = require('cartodb-redis')({ - host: global.settings.redis_host, - port: global.settings.redis_port, - max: global.settings.redisPool, - idleTimeoutMillis: global.settings.redisIdleTimeoutMillis, - reapIntervalMillis: global.settings.redisReapIntervalMillis -}); function payload(query) { return JSON.stringify({query: query}); @@ -44,14 +37,12 @@ describe('job query limit', function() { } after(function (done) { - // batch services is not activate, so we need empty the queue to avoid unexpected - // behaviour in further tests - metadataBackend.redisCmd(5, 'DEL', [ 'batch:queues:localhost' ], done); + redisUtils.clean('batch:*', done); }); it('POST /api/v2/sql/job with a invalid query size should respond with 400 query too long', function (done){ - assert.response(app, { + assert.response(server, { url: '/api/v2/sql/job?api_key=1234', headers: { 'host': 'vizzuality.cartodb.com', 'Content-Type': 'application/x-www-form-urlencoded' }, method: 'POST', @@ -60,25 +51,7 @@ describe('job query limit', function() { }) }, { status: 400 - }, function (res) { - var error = JSON.parse(res.body); - assert.deepEqual(error, { error: [expectedErrorMessage(queryTooLong)] }); - done(); - }); - }); - - it('PUT /api/v2/sql/job with a invalid query size should respond with 400 query too long', function (done){ - - assert.response(app, { - url: '/api/v2/sql/job/wadus?api_key=1234', - headers: { 'host': 'vizzuality.cartodb.com', 'Content-Type': 'application/x-www-form-urlencoded' }, - method: 'PUT', - data: querystring.stringify({ - query: queryTooLong - }) - }, { - status: 400 - }, function (res) { + }, function (err, res) { var error = JSON.parse(res.body); assert.deepEqual(error, { error: [expectedErrorMessage(queryTooLong)] }); done(); @@ -87,7 +60,7 @@ describe('job query limit', function() { it('POST /api/v2/sql/job with a valid query size should respond with 201 created', function (done){ - assert.response(app, { + assert.response(server, { url: '/api/v2/sql/job?api_key=1234', headers: { 'host': 'vizzuality.cartodb.com', 'Content-Type': 'application/x-www-form-urlencoded' }, method: 'POST', @@ -96,7 +69,7 @@ describe('job query limit', function() { }) }, { status: 201 - }, function (res) { + }, function (err, res) { var job = JSON.parse(res.body); assert.ok(job.job_id); done(); @@ -105,7 +78,7 @@ describe('job query limit', function() { it('POST /api/v2/sql/job with a invalid query size should consider multiple queries', function (done){ var queries = [queryTooLong, 'select 1']; - assert.response(app, { + assert.response(server, { url: '/api/v2/sql/job?api_key=1234', headers: { 'host': 'vizzuality.cartodb.com', 'Content-Type': 'application/x-www-form-urlencoded' }, method: 'POST', @@ -114,7 +87,7 @@ describe('job query limit', function() { }) }, { status: 400 - }, function (res) { + }, function (err, res) { var error = JSON.parse(res.body); assert.deepEqual(error, { error: [expectedErrorMessage(queries)] }); done(); @@ -131,7 +104,7 @@ describe('job query limit', function() { onsuccess: "SELECT * FROM untitle_table_4 limit 3" }] }; - assert.response(app, { + assert.response(server, { url: '/api/v2/sql/job?api_key=1234', headers: { 'host': 'vizzuality.cartodb.com', 'Content-Type': 'application/x-www-form-urlencoded' }, method: 'POST', @@ -140,7 +113,7 @@ describe('job query limit', function() { }) }, { status: 400 - }, function (res) { + }, function (err, res) { var error = JSON.parse(res.body); assert.deepEqual(error, { error: [expectedErrorMessage(fallbackQueries)] }); done(); diff --git a/test/acceptance/batch/job.query.order.test.js b/test/acceptance/batch/job.query.order.test.js new file mode 100644 index 000000000..2b26be338 --- /dev/null +++ b/test/acceptance/batch/job.query.order.test.js @@ -0,0 +1,57 @@ +require('../../helper'); +var assert = require('../../support/assert'); + +var BatchTestClient = require('../../support/batch-test-client'); +var JobStatus = require('../../../batch/job_status'); + +describe('job query order', function() { + + before(function() { + this.batchTestClient = new BatchTestClient(); + }); + + after(function (done) { + return this.batchTestClient.drain(done); + }); + + function createJob(queries) { + return { + query: queries + }; + } + + it('should run job queries in order (single consumer)', function (done) { + var jobRequest1 = createJob(["select 1", "select 2"]); + var jobRequest2 = createJob(["select 3"]); + + this.batchTestClient.createJob(jobRequest1, function(err, jobResult1) { + if (err) { + return done(err); + } + this.batchTestClient.createJob(jobRequest2, function(err, jobResult2) { + if (err) { + return done(err); + } + + jobResult1.getStatus(function (err, job1) { + if (err) { + return done(err); + } + jobResult2.getStatus(function(err, job2) { + if (err) { + return done(err); + } + assert.equal(job1.status, JobStatus.DONE); + assert.equal(job2.status, JobStatus.DONE); + assert.ok( + new Date(job1.updated_at).getTime() < new Date(job2.updated_at).getTime(), + 'job1 (' + job1.updated_at + ') should finish before job2 (' + job2.updated_at + ')' + ); + done(); + }); + }); + }); + }.bind(this)); + }); + +}); diff --git a/test/acceptance/batch/job.query.timeout.test.js b/test/acceptance/batch/job.query.timeout.test.js new file mode 100644 index 000000000..65df00d25 --- /dev/null +++ b/test/acceptance/batch/job.query.timeout.test.js @@ -0,0 +1,97 @@ +require('../../helper'); +var assert = require('../../support/assert'); + +var BatchTestClient = require('../../support/batch-test-client'); +var JobStatus = require('../../../batch/job_status'); + +describe('job query timeout', function() { + + before(function() { + this.batchQueryTimeout = global.settings.batch_query_timeout; + this.batchTestClient = new BatchTestClient(); + }); + + after(function (done) { + global.settings.batch_query_timeout = this.batchQueryTimeout; + return this.batchTestClient.drain(done); + }); + + function createTimeoutQuery(query, timeout) { + return { + query: { + query: [ + { + timeout: timeout, + query: query + } + ] + } + }; + } + + it('should run query with higher user timeout', function (done) { + var jobRequest = createTimeoutQuery("select pg_sleep(0.1)", 200); + this.batchTestClient.createJob(jobRequest, function(err, jobResult) { + if (err) { + return done(err); + } + jobResult.getStatus(function(err, job) { + if (err) { + return done(err); + } + assert.equal(job.status, JobStatus.DONE); + done(); + }); + }); + }); + + it('should fail to run query with lower user timeout', function (done) { + var jobRequest = createTimeoutQuery("select pg_sleep(0.1)", 50); + this.batchTestClient.createJob(jobRequest, function(err, jobResult) { + if (err) { + return done(err); + } + jobResult.getStatus(function(err, job) { + if (err) { + return done(err); + } + assert.equal(job.status, JobStatus.FAILED); + done(); + }); + }); + }); + + it('should fail to run query with user timeout if it is higher than config', function (done) { + global.settings.batch_query_timeout = 100; + var jobRequest = createTimeoutQuery("select pg_sleep(1)", 2000); + this.batchTestClient.createJob(jobRequest, function(err, jobResult) { + if (err) { + return done(err); + } + jobResult.getStatus(function(err, job) { + if (err) { + return done(err); + } + assert.equal(job.status, JobStatus.FAILED); + done(); + }); + }); + }); + + it('should fail to run query with user timeout if set to 0 (ignored timeout)', function (done) { + global.settings.batch_query_timeout = 100; + var jobRequest = createTimeoutQuery("select pg_sleep(1)", 0); + this.batchTestClient.createJob(jobRequest, function(err, jobResult) { + if (err) { + return done(err); + } + jobResult.getStatus(function(err, job) { + if (err) { + return done(err); + } + assert.equal(job.status, JobStatus.FAILED); + done(); + }); + }); + }); +}); diff --git a/test/acceptance/batch/job.test.js b/test/acceptance/batch/job.test.js new file mode 100644 index 000000000..5c3f2b5e5 --- /dev/null +++ b/test/acceptance/batch/job.test.js @@ -0,0 +1,218 @@ +/** + * + * Requires the database and tables setup in config/environments/test.js to exist + * Ensure the user is present in the pgbouncer auth file too + * TODO: Add OAuth tests. + * + * To run this test, ensure that cartodb_test_user_1_db metadata exists + * in Redis for the vizzuality.cartodb.com domain + * + * SELECT 5 + * HSET rails:users:vizzuality id 1 + * HSET rails:users:vizzuality database_name cartodb_test_user_1_db + * + */ +require('../../helper'); + +var server = require('../../../app/server')(); +var assert = require('../../support/assert'); +var redisUtils = require('../../support/redis_utils'); +var querystring = require('querystring'); + +describe('job module', function() { + var job = {}; + + after(function (done) { + redisUtils.clean('batch:*', done); + }); + + it('POST /api/v2/sql/job should respond with 200 and the created job', function (done){ + assert.response(server, { + url: '/api/v2/sql/job?api_key=1234', + headers: { 'host': 'vizzuality.cartodb.com', 'Content-Type': 'application/x-www-form-urlencoded' }, + method: 'POST', + data: querystring.stringify({ + query: "SELECT * FROM untitle_table_4" + }) + }, { + status: 201 + }, function(err, res) { + job = JSON.parse(res.body); + assert.deepEqual(res.headers['content-type'], 'application/json; charset=utf-8'); + assert.ok(job.job_id); + assert.equal(job.query, "SELECT * FROM untitle_table_4"); + assert.equal(job.user, "vizzuality"); + done(); + }); + }); + + it('POST /api/v2/sql/job without query should respond with 400 and the corresponding message of error', + function (done){ + assert.response(server, { + url: '/api/v2/sql/job?api_key=1234', + headers: { 'host': 'vizzuality.cartodb.com', 'Content-Type': 'application/x-www-form-urlencoded' }, + method: 'POST', + data: querystring.stringify({}) + }, { + status: 400 + }, function(err, res) { + var error = JSON.parse(res.body); + assert.deepEqual(error, { error: [ 'You must indicate a valid SQL' ] }); + done(); + }); + }); + + it('POST /api/v2/sql/job with bad query param should respond with 400 and message of error', function (done){ + assert.response(server, { + url: '/api/v2/sql/job?api_key=1234', + headers: { 'host': 'vizzuality.cartodb.com', 'Content-Type': 'application/x-www-form-urlencoded' }, + method: 'POST', + data: querystring.stringify({ + q: "SELECT * FROM untitle_table_4" + }) + }, { + status: 400 + }, function(err, res) { + var error = JSON.parse(res.body); + assert.deepEqual(error, { error: [ 'You must indicate a valid SQL' ] }); + done(); + }); + }); + + it('POST /api/v2/sql/job with wrong api key should respond with 401 permission denied', function (done){ + assert.response(server, { + url: '/api/v2/sql/job?api_key=wrong', + headers: { 'host': 'vizzuality.cartodb.com', 'Content-Type': 'application/x-www-form-urlencoded' }, + method: 'POST', + data: querystring.stringify({ + query: "SELECT * FROM untitle_table_4" + }) + }, { + status: 401 + }, function(err, res) { + var error = JSON.parse(res.body); + assert.deepEqual(error, { error: [ 'permission denied' ] }); + done(); + }); + }); + + it('POST /api/v2/sql/job with wrong host header should respond with 404 not found', function (done){ + assert.response(server, { + url: '/api/v2/sql/job?api_key=wrong', + headers: { 'host': 'wrong-host.cartodb.com', 'Content-Type': 'application/x-www-form-urlencoded' }, + method: 'POST', + data: querystring.stringify({ + query: "SELECT * FROM untitle_table_4" + }) + }, { + status: 404 + }, function(err, res) { + var error = JSON.parse(res.body); + assert.deepEqual(error, { + error: [ + 'Sorry, we can\'t find CartoDB user \'wrong-host\'. ' + + 'Please check that you have entered the correct domain.' + ] + }); + done(); + }); + }); + + it('GET /api/v2/sql/job/:job_id should respond with 200 and the requested job', function (done){ + assert.response(server, { + url: '/api/v2/sql/job/' + job.job_id + '?api_key=1234', + headers: { 'host': 'vizzuality.cartodb.com', 'Content-Type': 'application/x-www-form-urlencoded' }, + method: 'GET' + }, { + status: 200 + }, function(err, res) { + var jobGot = JSON.parse(res.body); + assert.deepEqual(res.headers['content-type'], 'application/json; charset=utf-8'); + assert.equal(jobGot.query, "SELECT * FROM untitle_table_4"); + assert.equal(jobGot.user, "vizzuality"); + done(); + }); + }); + + it('GET /api/v2/sql/job/:job_id with wrong api key should respond with 401 permission denied', function (done){ + assert.response(server, { + url: '/api/v2/sql/job/' + job.job_id + '?api_key=wrong', + headers: { 'host': 'vizzuality.cartodb.com', 'Content-Type': 'application/x-www-form-urlencoded' }, + method: 'GET' + }, { + status: 401 + }, function(err, res) { + var error = JSON.parse(res.body); + assert.deepEqual(error, { error: [ 'permission denied' ] }); + done(); + }); + }); + + it('GET /api/v2/sql/job/:jobId with wrong jobId header respond with 400 and an error', function (done){ + assert.response(server, { + url: '/api/v2/sql/job/irrelevantJob?api_key=1234', + headers: { 'host': 'vizzuality.cartodb.com', 'Content-Type': 'application/x-www-form-urlencoded' }, + method: 'GET' + }, { + status: 400 + }, function(err, res) { + var error = JSON.parse(res.body); + console.log(error); + assert.deepEqual(error , { + error: ['Job with id irrelevantJob not found'] + }); + done(); + }); + }); + + it('DELETE /api/v2/sql/job/:job_id should respond with 200 and the requested job', function (done){ + assert.response(server, { + url: '/api/v2/sql/job/' + job.job_id + '?api_key=1234', + headers: { 'host': 'vizzuality.cartodb.com', 'Content-Type': 'application/x-www-form-urlencoded' }, + method: 'DELETE' + }, { + status: 200 + }, function(err, res) { + var jobCancelled = JSON.parse(res.body); + assert.deepEqual(res.headers['content-type'], 'application/json; charset=utf-8'); + assert.equal(jobCancelled.job_id, job.job_id); + assert.equal(jobCancelled.query, "SELECT * FROM untitle_table_4"); + assert.equal(jobCancelled.user, "vizzuality"); + assert.equal(jobCancelled.status, "cancelled"); + done(); + }); + }); + + it('DELETE /api/v2/sql/job/:job_id with wrong api key should respond with 401 permission denied', function (done){ + assert.response(server, { + url: '/api/v2/sql/job/' + job.job_id + '?api_key=wrong', + headers: { 'host': 'vizzuality.cartodb.com', 'Content-Type': 'application/x-www-form-urlencoded' }, + method: 'DELETE' + }, { + status: 401 + }, function(err, res) { + var error = JSON.parse(res.body); + assert.deepEqual(error, { error: [ 'permission denied' ] }); + done(); + }); + }); + + it('DELETE /api/v2/sql/job/ with wrong host header respond with 404 not found', function (done){ + assert.response(server, { + url: '/api/v2/sql/job/' + job.job_id + '?api_key=1234', + headers: { 'host': 'wrong-host.cartodb.com', 'Content-Type': 'application/x-www-form-urlencoded' }, + method: 'DELETE' + }, { + status: 404 + }, function(err, res) { + var error = JSON.parse(res.body); + assert.deepEqual(error , { + error: [ + 'Sorry, we can\'t find CartoDB user \'wrong-host\'. ' + + 'Please check that you have entered the correct domain.' + ] + }); + done(); + }); + }); +}); diff --git a/test/acceptance/batch/job.timing.test.js b/test/acceptance/batch/job.timing.test.js new file mode 100644 index 000000000..d49027601 --- /dev/null +++ b/test/acceptance/batch/job.timing.test.js @@ -0,0 +1,195 @@ +require('../../helper'); + +var assert = require('../../support/assert'); +var redisUtils = require('../../support/redis_utils'); +var server = require('../../../app/server')(); +var querystring = require('qs'); +var metadataBackend = require('cartodb-redis')({ pool: redisUtils.getPool() }); +var batchFactory = require('../../../batch'); +var jobStatus = require('../../../batch/job_status'); + +describe('Batch API query timing', function () { + + function createJob(jobDefinition, callback) { + assert.response(server, { + url: '/api/v2/sql/job?api_key=1234', + headers: { + 'Content-Type': 'application/x-www-form-urlencoded', + host: 'vizzuality.cartodb.com' + }, + method: 'POST', + data: querystring.stringify(jobDefinition) + }, { + status: 201 + }, function (err, res) { + if (err) { + return callback(err); + } + return callback(null, JSON.parse(res.body)); + }); + } + + function getJobStatus(jobId, callback) { + assert.response(server, { + url: '/api/v2/sql/job/' + jobId + '?api_key=1234&', + headers: { + host: 'vizzuality.cartodb.com' + }, + method: 'GET' + }, { + status: 200 + }, function (err, res) { + if (err) { + return callback(err); + } + return callback(null, JSON.parse(res.body)); + }); + } + + function validateExpectedResponse(actual, expected) { + actual.query.forEach(function(actualQuery, index) { + var expectedQuery = expected.query[index]; + assert.ok(expectedQuery); + Object.keys(expectedQuery).forEach(function(expectedKey) { + assert.equal(actualQuery[expectedKey], expectedQuery[expectedKey]); + }); + var propsToCheckDate = ['started_at', 'ended_at']; + propsToCheckDate.forEach(function(propToCheckDate) { + if (actualQuery.hasOwnProperty(propToCheckDate)) { + assert.ok(new Date(actualQuery[propToCheckDate])); + } + }); + }); + + assert.equal(actual.onsuccess, expected.onsuccess); + assert.equal(actual.onerror, expected.onerror); + } + + var batch = batchFactory(metadataBackend, redisUtils.getPool()); + + before(function (done) { + batch.start(); + batch.on('ready', done); + }); + + after(function (done) { + batch.stop(); + redisUtils.clean('batch:*', done); + }); + + describe('should report start and end time for each query with fallback queries', function () { + var jobResponse; + before(function(done) { + createJob({ + "query": { + "query": [ + { + "query": "SELECT * FROM untitle_table_4 limit 1", + "onerror": "SELECT * FROM untitle_table_4 limit 2" + }, + { + "query": "SELECT * FROM untitle_table_4 limit 3", + "onerror": "SELECT * FROM untitle_table_4 limit 4" + } + ], + "onerror": "SELECT * FROM untitle_table_4 limit 5" + } + }, function(err, job) { + jobResponse = job; + return done(err); + }); + }); + + it('should expose started_at and ended_at for all queries with fallback mechanism', function (done) { + var expectedQuery = { + query: [{ + query: 'SELECT * FROM untitle_table_4 limit 1', + onerror: 'SELECT * FROM untitle_table_4 limit 2', + status: 'done', + fallback_status: 'skipped' + }, { + query: 'SELECT * FROM untitle_table_4 limit 3', + onerror: 'SELECT * FROM untitle_table_4 limit 4', + status: 'done', + fallback_status: 'skipped' + }], + onerror: 'SELECT * FROM untitle_table_4 limit 5' + }; + + var interval = setInterval(function () { + getJobStatus(jobResponse.job_id, function(err, job) { + if (job.status === jobStatus.DONE) { + clearInterval(interval); + validateExpectedResponse(job.query, expectedQuery); + job.query.query.forEach(function(actualQuery) { + assert.ok(actualQuery.started_at); + assert.ok(actualQuery.ended_at); + }); + done(); + } else if (job.status === jobStatus.FAILED || job.status === jobStatus.CANCELLED) { + clearInterval(interval); + done(new Error('Job ' + job.job_id + ' is ' + job.status + ', expected to be "done"')); + } + }); + }, 50); + }); + }); + + describe('should report start and end time for each query also for failing queries', function () { + var jobResponse; + before(function(done) { + createJob({ + "query": { + "query": [ + { + "query": "SELECT * FROM untitle_table_4 limit 1", + "onerror": "SELECT * FROM untitle_table_4 limit 2" + }, + { + "query": "SELECT * FROM untitle_table_4 limit 3 failed", + "onerror": "SELECT * FROM untitle_table_4 limit 4" + } + ], + "onerror": "SELECT * FROM untitle_table_4 limit 5" + } + }, function(err, job) { + jobResponse = job; + return done(err); + }); + }); + + it('should expose started_at and ended_at for all queries with fallback mechanism (failed)', function (done) { + var expectedQuery = { + query: [{ + query: 'SELECT * FROM untitle_table_4 limit 1', + onerror: 'SELECT * FROM untitle_table_4 limit 2', + status: 'done', + fallback_status: 'skipped' + }, { + query: 'SELECT * FROM untitle_table_4 limit 3 failed', + onerror: 'SELECT * FROM untitle_table_4 limit 4', + status: 'failed', + fallback_status: 'done' + }], + onerror: 'SELECT * FROM untitle_table_4 limit 5' + }; + + var interval = setInterval(function () { + getJobStatus(jobResponse.job_id, function(err, job) { + if (job.status === jobStatus.FAILED) { + clearInterval(interval); + validateExpectedResponse(job.query, expectedQuery); + job.query.query.forEach(function(actualQuery) { + assert.ok(actualQuery.started_at); + assert.ok(actualQuery.ended_at); + }); + done(); + } else if (job.status === jobStatus.DONE || job.status === jobStatus.CANCELLED) { + clearInterval(interval); + done(new Error('Job ' + job.job_id + ' is ' + job.status + ', expected to be "failed"')); + } + }); + }, 50); + }); + }); +}); diff --git a/test/acceptance/job.use-case-1.test.js b/test/acceptance/batch/job.use-case-1.test.js similarity index 62% rename from test/acceptance/job.use-case-1.test.js rename to test/acceptance/batch/job.use-case-1.test.js index 2c965baca..b0888656a 100644 --- a/test/acceptance/job.use-case-1.test.js +++ b/test/acceptance/batch/job.use-case-1.test.js @@ -12,39 +12,32 @@ * HSET rails:users:vizzuality database_name cartodb_test_user_1_db * */ -require('../helper'); +require('../../helper'); -var app = require(global.settings.app_root + '/app/app')(); -var assert = require('../support/assert'); +var server = require('../../../app/server')(); +var assert = require('../../support/assert'); +var redisUtils = require('../../support/redis_utils'); var querystring = require('querystring'); -var metadataBackend = require('cartodb-redis')({ - host: global.settings.redis_host, - port: global.settings.redis_port, - max: global.settings.redisPool, - idleTimeoutMillis: global.settings.redisIdleTimeoutMillis, - reapIntervalMillis: global.settings.redisReapIntervalMillis -}); -var batchFactory = require('../../batch'); +var metadataBackend = require('cartodb-redis')({ pool: redisUtils.getPool() }); +var batchFactory = require('../../../batch/index'); describe('Use case 1: cancel and modify a done job', function () { + var batch = batchFactory(metadataBackend, redisUtils.getPool()); - var batch = batchFactory(metadataBackend); - - before(function () { + before(function (done) { batch.start(); + batch.on('ready', done); }); after(function (done) { batch.stop(); - batch.drain(function () { - metadataBackend.redisCmd(5, 'DEL', [ 'batch:queues:localhost' ], done); - }); + redisUtils.clean('batch:*', done); }); var doneJob = {}; it('Step 1, should create a job', function (done) { - assert.response(app, { + assert.response(server, { url: '/api/v2/sql/job?api_key=1234', headers: { 'host': 'vizzuality.cartodb.com', 'Content-Type': 'application/x-www-form-urlencoded' }, method: 'POST', @@ -53,7 +46,7 @@ describe('Use case 1: cancel and modify a done job', function () { }) }, { status: 201 - }, function (res) { + }, function (err, res) { doneJob = JSON.parse(res.body); done(); }); @@ -61,13 +54,13 @@ describe('Use case 1: cancel and modify a done job', function () { it('Step 2, job should be done', function (done) { var interval = setInterval(function () { - assert.response(app, { + assert.response(server, { url: '/api/v2/sql/job/' + doneJob.job_id + '?api_key=1234', headers: { 'host': 'vizzuality.cartodb.com', 'Content-Type': 'application/x-www-form-urlencoded' }, method: 'GET' }, { status: 200 - }, function (res) { + }, function (err, res) { var job = JSON.parse(res.body); if (job.status === "done") { clearInterval(interval); @@ -83,33 +76,16 @@ describe('Use case 1: cancel and modify a done job', function () { }); it('Step 3, cancel a done job should give an error', function (done){ - assert.response(app, { + assert.response(server, { url: '/api/v2/sql/job/' + doneJob.job_id + '?api_key=1234', headers: { 'host': 'vizzuality.cartodb.com', 'Content-Type': 'application/x-www-form-urlencoded' }, method: 'DELETE' }, { status: 400 - }, function(res) { + }, function(err, res) { var errors = JSON.parse(res.body); assert.equal(errors.error[0], "Cannot set status from done to cancelled"); done(); }); }); - - it('Step 4, modify a done job should give an error', function (done){ - assert.response(app, { - url: '/api/v2/sql/job/' + doneJob.job_id + '?api_key=1234', - headers: { 'host': 'vizzuality.cartodb.com', 'Content-Type': 'application/x-www-form-urlencoded' }, - method: 'PUT', - data: querystring.stringify({ - query: "SELECT cartodb_id FROM untitle_table_4" - }) - }, { - status: 400 - }, function(res) { - var errors = JSON.parse(res.body); - assert.equal(errors.error[0], "Job is not pending, it cannot be updated"); - done(); - }); - }); }); diff --git a/test/acceptance/job.use-case-10.test.js b/test/acceptance/batch/job.use-case-10.test.js similarity index 63% rename from test/acceptance/job.use-case-10.test.js rename to test/acceptance/batch/job.use-case-10.test.js index a327bd9f7..b43519a0d 100644 --- a/test/acceptance/job.use-case-10.test.js +++ b/test/acceptance/batch/job.use-case-10.test.js @@ -12,39 +12,32 @@ * HSET rails:users:vizzuality database_name cartodb_test_user_1_db * */ -require('../helper'); +require('../../helper'); -var app = require(global.settings.app_root + '/app/app')(); -var assert = require('../support/assert'); +var server = require('../../../app/server')(); +var assert = require('../../support/assert'); +var redisUtils = require('../../support/redis_utils'); var querystring = require('querystring'); -var metadataBackend = require('cartodb-redis')({ - host: global.settings.redis_host, - port: global.settings.redis_port, - max: global.settings.redisPool, - idleTimeoutMillis: global.settings.redisIdleTimeoutMillis, - reapIntervalMillis: global.settings.redisReapIntervalMillis -}); -var batchFactory = require('../../batch'); +var metadataBackend = require('cartodb-redis')({ pool: redisUtils.getPool() }); +var batchFactory = require('../../../batch/index'); describe('Use case 10: cancel and modify a done multiquery job', function () { + var batch = batchFactory(metadataBackend, redisUtils.getPool()); - var batch = batchFactory(metadataBackend); - - before(function () { + before(function (done) { batch.start(); + batch.on('ready', done); }); after(function (done) { batch.stop(); - batch.drain(function () { - metadataBackend.redisCmd(5, 'DEL', [ 'batch:queues:localhost' ], done); - }); + redisUtils.clean('batch:*', done); }); var doneJob = {}; it('Step 1, should create a multiquery job', function (done) { - assert.response(app, { + assert.response(server, { url: '/api/v2/sql/job?api_key=1234', headers: { 'host': 'vizzuality.cartodb.com', 'Content-Type': 'application/x-www-form-urlencoded' }, method: 'POST', @@ -57,7 +50,7 @@ describe('Use case 10: cancel and modify a done multiquery job', function () { }) }, { status: 201 - }, function (res) { + }, function (err, res) { doneJob = JSON.parse(res.body); done(); }); @@ -65,13 +58,13 @@ describe('Use case 10: cancel and modify a done multiquery job', function () { it('Step 2, multiquery job should be done', function (done) { var interval = setInterval(function () { - assert.response(app, { + assert.response(server, { url: '/api/v2/sql/job/' + doneJob.job_id + '?api_key=1234', headers: { 'host': 'vizzuality.cartodb.com', 'Content-Type': 'application/x-www-form-urlencoded' }, method: 'GET' }, { status: 200 - }, function (res) { + }, function (err, res) { var job = JSON.parse(res.body); if (job.status === "done") { clearInterval(interval); @@ -87,36 +80,16 @@ describe('Use case 10: cancel and modify a done multiquery job', function () { }); it('Step 3, cancel a done multiquery job should give an error', function (done){ - assert.response(app, { + assert.response(server, { url: '/api/v2/sql/job/' + doneJob.job_id + '?api_key=1234', headers: { 'host': 'vizzuality.cartodb.com', 'Content-Type': 'application/x-www-form-urlencoded' }, method: 'DELETE' }, { status: 400 - }, function(res) { + }, function(err, res) { var errors = JSON.parse(res.body); assert.equal(errors.error[0], "Cannot set status from done to cancelled"); done(); }); }); - - it('Step 4, modify a done multiquery job should give an error', function (done){ - assert.response(app, { - url: '/api/v2/sql/job/' + doneJob.job_id + '?api_key=1234', - headers: { 'host': 'vizzuality.cartodb.com', 'Content-Type': 'application/x-www-form-urlencoded' }, - method: 'PUT', - data: querystring.stringify({ - query: [ - "SELECT * FROM untitle_table_4", - "SELECT * FROM untitle_table_4" - ] - }) - }, { - status: 400 - }, function(res) { - var errors = JSON.parse(res.body); - assert.equal(errors.error[0], "Job is not pending, it cannot be updated"); - done(); - }); - }); }); diff --git a/test/acceptance/job.use-case-2.test.js b/test/acceptance/batch/job.use-case-2.test.js similarity index 68% rename from test/acceptance/job.use-case-2.test.js rename to test/acceptance/batch/job.use-case-2.test.js index c2552d63e..0e88a29d7 100644 --- a/test/acceptance/job.use-case-2.test.js +++ b/test/acceptance/batch/job.use-case-2.test.js @@ -12,40 +12,33 @@ * HSET rails:users:vizzuality database_name cartodb_test_user_1_db * */ -require('../helper'); +require('../../helper'); -var app = require(global.settings.app_root + '/app/app')(); -var assert = require('../support/assert'); +var server = require('../../../app/server')(); +var assert = require('../../support/assert'); +var redisUtils = require('../../support/redis_utils'); var querystring = require('querystring'); -var metadataBackend = require('cartodb-redis')({ - host: global.settings.redis_host, - port: global.settings.redis_port, - max: global.settings.redisPool, - idleTimeoutMillis: global.settings.redisIdleTimeoutMillis, - reapIntervalMillis: global.settings.redisReapIntervalMillis -}); -var batchFactory = require('../../batch'); +var metadataBackend = require('cartodb-redis')({ pool: redisUtils.getPool() }); +var batchFactory = require('../../../batch/index'); describe('Use case 2: cancel a running job', function() { + var batch = batchFactory(metadataBackend, redisUtils.getPool()); - var batch = batchFactory(metadataBackend); - - before(function () { + before(function (done) { batch.start(); + batch.on('ready', done); }); after(function (done) { batch.stop(); - batch.drain(function () { - metadataBackend.redisCmd(5, 'DEL', [ 'batch:queues:localhost' ], done); - }); + redisUtils.clean('batch:*', done); }); var runningJob = {}; var cancelledJob = {}; it('Step 1, should create a new job', function (done){ - assert.response(app, { + assert.response(server, { url: '/api/v2/sql/job?api_key=1234', headers: { 'host': 'vizzuality.cartodb.com', 'Content-Type': 'application/x-www-form-urlencoded' }, method: 'POST', @@ -54,7 +47,7 @@ describe('Use case 2: cancel a running job', function() { }) }, { status: 201 - }, function(res) { + }, function(err, res) { runningJob = JSON.parse(res.body); done(); }); @@ -62,13 +55,13 @@ describe('Use case 2: cancel a running job', function() { it('Step 2, job should be running', function (done){ var interval = setInterval(function () { - assert.response(app, { + assert.response(server, { url: '/api/v2/sql/job/' + runningJob.job_id + '?api_key=1234', headers: { 'host': 'vizzuality.cartodb.com', 'Content-Type': 'application/x-www-form-urlencoded' }, method: 'GET' }, { status: 200 - }, function(res) { + }, function(err, res) { var job = JSON.parse(res.body); if (job.status === "running") { clearInterval(interval); @@ -82,13 +75,13 @@ describe('Use case 2: cancel a running job', function() { }); it('Step 3, cancel a job', function (done){ - assert.response(app, { + assert.response(server, { url: '/api/v2/sql/job/' + runningJob.job_id + '?api_key=1234', headers: { 'host': 'vizzuality.cartodb.com', 'Content-Type': 'application/x-www-form-urlencoded' }, method: 'DELETE' }, { status: 200 - }, function(res) { + }, function(err, res) { cancelledJob = JSON.parse(res.body); assert.equal(cancelledJob.status, "cancelled"); done(); @@ -96,13 +89,13 @@ describe('Use case 2: cancel a running job', function() { }); it('Step 4, job should be cancelled', function (done){ - assert.response(app, { + assert.response(server, { url: '/api/v2/sql/job/' + runningJob.job_id + '?api_key=1234', headers: { 'host': 'vizzuality.cartodb.com', 'Content-Type': 'application/x-www-form-urlencoded' }, method: 'GET' }, { status: 200 - }, function(res) { + }, function(err, res) { var job = JSON.parse(res.body); if (job.status === "cancelled") { done(); @@ -113,33 +106,16 @@ describe('Use case 2: cancel a running job', function() { }); it('Step 5, cancel a cancelled should give an error', function (done) { - assert.response(app, { + assert.response(server, { url: '/api/v2/sql/job/' + cancelledJob.job_id + '?api_key=1234', headers: { 'host': 'vizzuality.cartodb.com', 'Content-Type': 'application/x-www-form-urlencoded' }, method: 'DELETE' }, { status: 400 - }, function(res) { + }, function(err, res) { var errors = JSON.parse(res.body); assert.equal(errors.error[0], "Cannot set status from cancelled to cancelled"); done(); }); }); - - it('Step 5, modify a cancelled job should give an error', function (done){ - assert.response(app, { - url: '/api/v2/sql/job/' + cancelledJob.job_id + '?api_key=1234', - headers: { 'host': 'vizzuality.cartodb.com', 'Content-Type': 'application/x-www-form-urlencoded' }, - method: 'PUT', - data: querystring.stringify({ - query: "SELECT cartodb_id FROM untitle_table_4" - }) - }, { - status: 400 - }, function(res) { - var errors = JSON.parse(res.body); - assert.equal(errors.error[0], "Job is not pending, it cannot be updated"); - done(); - }); - }); }); diff --git a/test/acceptance/job.use-case-3.test.js b/test/acceptance/batch/job.use-case-3.test.js similarity index 78% rename from test/acceptance/job.use-case-3.test.js rename to test/acceptance/batch/job.use-case-3.test.js index a4d360559..550a715a5 100644 --- a/test/acceptance/job.use-case-3.test.js +++ b/test/acceptance/batch/job.use-case-3.test.js @@ -12,40 +12,33 @@ * HSET rails:users:vizzuality database_name cartodb_test_user_1_db * */ -require('../helper'); +require('../../helper'); -var app = require(global.settings.app_root + '/app/app')(); -var assert = require('../support/assert'); +var server = require('../../../app/server')(); +var assert = require('../../support/assert'); +var redisUtils = require('../../support/redis_utils'); var querystring = require('querystring'); -var metadataBackend = require('cartodb-redis')({ - host: global.settings.redis_host, - port: global.settings.redis_port, - max: global.settings.redisPool, - idleTimeoutMillis: global.settings.redisIdleTimeoutMillis, - reapIntervalMillis: global.settings.redisReapIntervalMillis -}); -var batchFactory = require('../../batch'); +var metadataBackend = require('cartodb-redis')({ pool: redisUtils.getPool() }); +var batchFactory = require('../../../batch/index'); describe('Use case 3: cancel a pending job', function() { + var batch = batchFactory(metadataBackend, redisUtils.getPool()); - var batch = batchFactory(metadataBackend); - - before(function () { + before(function (done) { batch.start(); + batch.on('ready', done); }); after(function (done) { batch.stop(); - batch.drain(function () { - metadataBackend.redisCmd(5, 'DEL', [ 'batch:queues:localhost' ], done); - }); + redisUtils.clean('batch:*', done); }); var runningJob = {}; var pendingJob = {}; it('Step 1, should create a job', function (done) { - assert.response(app, { + assert.response(server, { url: '/api/v2/sql/job?api_key=1234', headers: { 'host': 'vizzuality.cartodb.com', 'Content-Type': 'application/x-www-form-urlencoded' }, method: 'POST', @@ -54,14 +47,14 @@ describe('Use case 3: cancel a pending job', function() { }) }, { status: 201 - }, function (res) { + }, function (err, res) { runningJob = JSON.parse(res.body); done(); }); }); it('Step 2, should create a another job', function (done) { - assert.response(app, { + assert.response(server, { url: '/api/v2/sql/job?api_key=1234', headers: { 'host': 'vizzuality.cartodb.com', 'Content-Type': 'application/x-www-form-urlencoded' }, method: 'POST', @@ -70,7 +63,7 @@ describe('Use case 3: cancel a pending job', function() { }) }, { status: 201 - }, function(res) { + }, function(err, res) { pendingJob = JSON.parse(res.body); done(); }); @@ -78,13 +71,13 @@ describe('Use case 3: cancel a pending job', function() { it('Step 3, job should be pending', function (done){ var interval = setInterval(function () { - assert.response(app, { + assert.response(server, { url: '/api/v2/sql/job/' + pendingJob.job_id + '?api_key=1234', headers: { 'host': 'vizzuality.cartodb.com', 'Content-Type': 'application/x-www-form-urlencoded' }, method: 'GET' }, { status: 200 - }, function(res) { + }, function(err, res) { var job = JSON.parse(res.body); if (job.status === "pending") { clearInterval(interval); @@ -98,13 +91,13 @@ describe('Use case 3: cancel a pending job', function() { }); it('Step 4, cancel a pending job should be cancelled', function (done){ - assert.response(app, { + assert.response(server, { url: '/api/v2/sql/job/' + pendingJob.job_id + '?api_key=1234', headers: { 'host': 'vizzuality.cartodb.com', 'Content-Type': 'application/x-www-form-urlencoded' }, method: 'DELETE' }, { status: 200 - }, function(res) { + }, function(err, res) { var jobGot = JSON.parse(res.body); assert.equal(jobGot.job_id, pendingJob.job_id); assert.equal(jobGot.status, "cancelled"); @@ -113,13 +106,13 @@ describe('Use case 3: cancel a pending job', function() { }); it('Step 5, running job should be cancelled', function (done){ - assert.response(app, { + assert.response(server, { url: '/api/v2/sql/job/' + runningJob.job_id + '?api_key=1234', headers: { 'host': 'vizzuality.cartodb.com', 'Content-Type': 'application/x-www-form-urlencoded' }, method: 'DELETE' }, { status: 200 - }, function(res) { + }, function(err, res) { var cancelledJob = JSON.parse(res.body); assert.equal(cancelledJob.status, "cancelled"); done(); diff --git a/test/acceptance/job.use-case-4.test.js b/test/acceptance/batch/job.use-case-4.test.js similarity index 64% rename from test/acceptance/job.use-case-4.test.js rename to test/acceptance/batch/job.use-case-4.test.js index 83eb35036..22d175ecc 100644 --- a/test/acceptance/job.use-case-4.test.js +++ b/test/acceptance/batch/job.use-case-4.test.js @@ -12,40 +12,33 @@ * HSET rails:users:vizzuality database_name cartodb_test_user_1_db * */ -require('../helper'); +require('../../helper'); -var app = require(global.settings.app_root + '/app/app')(); -var assert = require('../support/assert'); +var server = require('../../../app/server')(); +var assert = require('../../support/assert'); +var redisUtils = require('../../support/redis_utils'); var querystring = require('querystring'); -var metadataBackend = require('cartodb-redis')({ - host: global.settings.redis_host, - port: global.settings.redis_port, - max: global.settings.redisPool, - idleTimeoutMillis: global.settings.redisIdleTimeoutMillis, - reapIntervalMillis: global.settings.redisReapIntervalMillis -}); -var batchFactory = require('../../batch'); +var metadataBackend = require('cartodb-redis')({ pool: redisUtils.getPool() }); +var batchFactory = require('../../../batch'); describe('Use case 4: modify a pending job', function() { + var batch = batchFactory(metadataBackend, redisUtils.getPool()); - var batch = batchFactory(metadataBackend); - - before(function () { + before(function (done) { batch.start(); + batch.on('ready', done); }); after(function (done) { batch.stop(); - batch.drain(function () { - metadataBackend.redisCmd(5, 'DEL', [ 'batch:queues:localhost' ], done); - }); + redisUtils.clean('batch:*', done); }); var runningJob = {}; var pendingJob = {}; it('Step 1, should create a job', function (done) { - assert.response(app, { + assert.response(server, { url: '/api/v2/sql/job?api_key=1234', headers: { 'host': 'vizzuality.cartodb.com', 'Content-Type': 'application/x-www-form-urlencoded' }, method: 'POST', @@ -54,14 +47,14 @@ describe('Use case 4: modify a pending job', function() { }) }, { status: 201 - }, function(res) { + }, function(err, res) { runningJob = JSON.parse(res.body); done(); }); }); it('Step 2, should create another job', function (done) { - assert.response(app, { + assert.response(server, { url: '/api/v2/sql/job?api_key=1234', headers: { 'host': 'vizzuality.cartodb.com', 'Content-Type': 'application/x-www-form-urlencoded' }, method: 'POST', @@ -70,7 +63,7 @@ describe('Use case 4: modify a pending job', function() { }) }, { status: 201 - }, function(res) { + }, function(err, res) { pendingJob = JSON.parse(res.body); done(); }); @@ -78,13 +71,13 @@ describe('Use case 4: modify a pending job', function() { it('Step 3, job should be pending', function (done){ var interval = setInterval(function () { - assert.response(app, { + assert.response(server, { url: '/api/v2/sql/job/' + pendingJob.job_id + '?api_key=1234', headers: { 'host': 'vizzuality.cartodb.com', 'Content-Type': 'application/x-www-form-urlencoded' }, method: 'GET' }, { status: 200 - }, function(res) { + }, function(err, res) { var job = JSON.parse(res.body); if (job.status === "pending") { clearInterval(interval); @@ -97,32 +90,14 @@ describe('Use case 4: modify a pending job', function() { }, 50); }); - it('Step 4, job should be modified', function (done){ - assert.response(app, { - url: '/api/v2/sql/job/' + pendingJob.job_id + '?api_key=1234', - headers: { 'host': 'vizzuality.cartodb.com', 'Content-Type': 'application/x-www-form-urlencoded' }, - method: 'PUT', - data: querystring.stringify({ - query: "SELECT cartodb_id FROM untitle_table_4" - }) - }, { - status: 200 - }, function(res) { - var jobGot = JSON.parse(res.body); - assert.equal(jobGot.job_id, pendingJob.job_id); - assert.equal(jobGot.query, "SELECT cartodb_id FROM untitle_table_4"); - done(); - }); - }); - it('Step 5, running job should be cancelled', function (done){ - assert.response(app, { + assert.response(server, { url: '/api/v2/sql/job/' + runningJob.job_id + '?api_key=1234', headers: { 'host': 'vizzuality.cartodb.com', 'Content-Type': 'application/x-www-form-urlencoded' }, method: 'DELETE' }, { status: 200 - }, function(res) { + }, function(err, res) { var cancelledJob = JSON.parse(res.body); assert.equal(cancelledJob.status, "cancelled"); done(); diff --git a/test/acceptance/job.use-case-5.test.js b/test/acceptance/batch/job.use-case-5.test.js similarity index 52% rename from test/acceptance/job.use-case-5.test.js rename to test/acceptance/batch/job.use-case-5.test.js index 2c395d3ae..c54154cbf 100644 --- a/test/acceptance/job.use-case-5.test.js +++ b/test/acceptance/batch/job.use-case-5.test.js @@ -12,39 +12,32 @@ * HSET rails:users:vizzuality database_name cartodb_test_user_1_db * */ -require('../helper'); +require('../../helper'); -var app = require(global.settings.app_root + '/app/app')(); -var assert = require('../support/assert'); +var server = require('../../../app/server')(); +var assert = require('../../support/assert'); +var redisUtils = require('../../support/redis_utils'); var querystring = require('querystring'); -var metadataBackend = require('cartodb-redis')({ - host: global.settings.redis_host, - port: global.settings.redis_port, - max: global.settings.redisPool, - idleTimeoutMillis: global.settings.redisIdleTimeoutMillis, - reapIntervalMillis: global.settings.redisReapIntervalMillis -}); -var batchFactory = require('../../batch'); +var metadataBackend = require('cartodb-redis')({ pool: redisUtils.getPool() }); +var batchFactory = require('../../../batch'); describe('Use case 5: modify a running job', function() { + var batch = batchFactory(metadataBackend, redisUtils.getPool()); - var batch = batchFactory(metadataBackend); - - before(function () { + before(function (done) { batch.start(); + batch.on('ready', done); }); after(function (done) { batch.stop(); - batch.drain(function () { - metadataBackend.redisCmd(5, 'DEL', [ 'batch:queues:localhost' ], done); - }); + redisUtils.clean('batch:*', done); }); var runningJob = {}; it('Step 1, should create job', function (done) { - assert.response(app, { + assert.response(server, { url: '/api/v2/sql/job?api_key=1234', headers: { 'host': 'vizzuality.cartodb.com', 'Content-Type': 'application/x-www-form-urlencoded' }, method: 'POST', @@ -53,7 +46,7 @@ describe('Use case 5: modify a running job', function() { }) }, { status: 201 - }, function (res) { + }, function (err, res) { runningJob = JSON.parse(res.body); done(); }); @@ -61,13 +54,13 @@ describe('Use case 5: modify a running job', function() { it('Step 2, job should be running', function (done){ var interval = setInterval(function () { - assert.response(app, { + assert.response(server, { url: '/api/v2/sql/job/' + runningJob.job_id + '?api_key=1234', headers: { 'host': 'vizzuality.cartodb.com', 'Content-Type': 'application/x-www-form-urlencoded' }, method: 'GET' }, { status: 200 - }, function(res) { + }, function(err, res) { var job = JSON.parse(res.body); if (job.status === "running") { clearInterval(interval); @@ -80,51 +73,17 @@ describe('Use case 5: modify a running job', function() { }, 50); }); - it('Step 3, modify a running job should give an error', function (done){ - assert.response(app, { - url: '/api/v2/sql/job/' + runningJob.job_id + '?api_key=1234', - headers: { 'host': 'vizzuality.cartodb.com', 'Content-Type': 'application/x-www-form-urlencoded' }, - method: 'PUT', - data: querystring.stringify({ - query: "SELECT cartodb_id FROM untitle_table_4" - }) - }, { - status: 400 - }, function(res) { - var errors = JSON.parse(res.body); - assert.equal(errors.error[0], "Job is not pending, it cannot be updated"); - done(); - }); - }); - it('Step 4, running job should be cancelled', function (done){ - assert.response(app, { + assert.response(server, { url: '/api/v2/sql/job/' + runningJob.job_id + '?api_key=1234', headers: { 'host': 'vizzuality.cartodb.com', 'Content-Type': 'application/x-www-form-urlencoded' }, method: 'DELETE' }, { status: 200 - }, function(res) { + }, function(err, res) { var cancelledJob = JSON.parse(res.body); assert.equal(cancelledJob.status, "cancelled"); done(); }); }); - - it('Step 5, modify again a cancelled job should give an error', function (done){ - assert.response(app, { - url: '/api/v2/sql/job/' + runningJob.job_id + '?api_key=1234', - headers: { 'host': 'vizzuality.cartodb.com', 'Content-Type': 'application/x-www-form-urlencoded' }, - method: 'PUT', - data: querystring.stringify({ - query: "SELECT cartodb_id FROM untitle_table_4" - }) - }, { - status: 400 - }, function(res) { - var errors = JSON.parse(res.body); - assert.equal(errors.error[0], "Job is not pending, it cannot be updated"); - done(); - }); - }); }); diff --git a/test/acceptance/batch/job.use-case-6.test.js b/test/acceptance/batch/job.use-case-6.test.js new file mode 100644 index 000000000..0ef007a32 --- /dev/null +++ b/test/acceptance/batch/job.use-case-6.test.js @@ -0,0 +1,75 @@ +/** + * + * Requires the database and tables setup in config/environments/test.js to exist + * Ensure the user is present in the pgbouncer auth file too + * TODO: Add OAuth tests. + * + * To run this test, ensure that cartodb_test_user_1_db metadata exists + * in Redis for the vizzuality.cartodb.com domain + * + * SELECT 5 + * HSET rails:users:vizzuality id 1 + * HSET rails:users:vizzuality database_name cartodb_test_user_1_db + * + */ +require('../../helper'); + +var server = require('../../../app/server')(); +var assert = require('../../support/assert'); +var redisUtils = require('../../support/redis_utils'); +var querystring = require('querystring'); +var metadataBackend = require('cartodb-redis')({ pool: redisUtils.getPool() }); +var batchFactory = require('../../../batch'); + +describe('Use case 6: modify a done job', function() { + var batch = batchFactory(metadataBackend, redisUtils.getPool()); + + before(function (done) { + batch.start(); + batch.on('ready', done); + }); + + after(function (done) { + batch.stop(); + redisUtils.clean('batch:*', done); + }); + + var doneJob = {}; + + it('Step 1, should create job', function (done) { + assert.response(server, { + url: '/api/v2/sql/job?api_key=1234', + headers: { 'host': 'vizzuality.cartodb.com', 'Content-Type': 'application/x-www-form-urlencoded' }, + method: 'POST', + data: querystring.stringify({ + query: "SELECT * FROM untitle_table_4" + }) + }, { + status: 201 + }, function (err, res) { + doneJob = JSON.parse(res.body); + done(); + }); + }); + + it('Step 2, job should be done', function (done) { + var interval = setInterval(function () { + assert.response(server, { + url: '/api/v2/sql/job/' + doneJob.job_id + '?api_key=1234', + headers: { 'host': 'vizzuality.cartodb.com', 'Content-Type': 'application/x-www-form-urlencoded' }, + method: 'GET' + }, { + status: 200 + }, function(err, res) { + var job = JSON.parse(res.body); + if (job.status === "done") { + clearInterval(interval); + done(); + } else if (job.status === "failed" || job.status === "cancelled") { + clearInterval(interval); + done(new Error('Job ' + job.job_id + ' is ' + job.status + ', expected to be done')); + } + }); + }, 50); + }); +}); diff --git a/test/acceptance/job.use-case-7.test.js b/test/acceptance/batch/job.use-case-7.test.js similarity index 74% rename from test/acceptance/job.use-case-7.test.js rename to test/acceptance/batch/job.use-case-7.test.js index 7fdefab40..0957e3567 100644 --- a/test/acceptance/job.use-case-7.test.js +++ b/test/acceptance/batch/job.use-case-7.test.js @@ -12,39 +12,32 @@ * HSET rails:users:vizzuality database_name cartodb_test_user_1_db * */ -require('../helper'); +require('../../helper'); -var app = require(global.settings.app_root + '/app/app')(); -var assert = require('../support/assert'); +var server = require('../../../app/server')(); +var assert = require('../../support/assert'); +var redisUtils = require('../../support/redis_utils'); var querystring = require('querystring'); -var metadataBackend = require('cartodb-redis')({ - host: global.settings.redis_host, - port: global.settings.redis_port, - max: global.settings.redisPool, - idleTimeoutMillis: global.settings.redisIdleTimeoutMillis, - reapIntervalMillis: global.settings.redisReapIntervalMillis -}); -var batchFactory = require('../../batch'); +var metadataBackend = require('cartodb-redis')({ pool: redisUtils.getPool() }); +var batchFactory = require('../../../batch'); describe('Use case 7: cancel a job with quotes', function() { + var batch = batchFactory(metadataBackend, redisUtils.getPool()); - var batch = batchFactory(metadataBackend); - - before(function () { + before(function (done) { batch.start(); + batch.on('ready', done); }); after(function (done) { batch.stop(); - batch.drain(function () { - metadataBackend.redisCmd(5, 'DEL', [ 'batch:queues:localhost' ], done); - }); + redisUtils.clean('batch:*', done); }); var runningJob = {}; it('Step 1, should create job with quotes', function (done) { - assert.response(app, { + assert.response(server, { url: '/api/v2/sql/job?api_key=1234', headers: { 'host': 'vizzuality.cartodb.com', 'Content-Type': 'application/x-www-form-urlencoded' }, method: 'POST', @@ -53,7 +46,7 @@ describe('Use case 7: cancel a job with quotes', function() { }) }, { status: 201 - }, function (res) { + }, function (err, res) { runningJob = JSON.parse(res.body); done(); }); @@ -61,13 +54,13 @@ describe('Use case 7: cancel a job with quotes', function() { it('Step 2, job should be running', function (done){ var interval = setInterval(function () { - assert.response(app, { + assert.response(server, { url: '/api/v2/sql/job/' + runningJob.job_id + '?api_key=1234', headers: { 'host': 'vizzuality.cartodb.com', 'Content-Type': 'application/x-www-form-urlencoded' }, method: 'GET' }, { status: 200 - }, function(res) { + }, function(err, res) { var job = JSON.parse(res.body); if (job.status === "running") { clearInterval(interval); @@ -81,13 +74,13 @@ describe('Use case 7: cancel a job with quotes', function() { }); it('Step 3, running job should be cancelled', function (done){ - assert.response(app, { + assert.response(server, { url: '/api/v2/sql/job/' + runningJob.job_id + '?api_key=1234', headers: { 'host': 'vizzuality.cartodb.com', 'Content-Type': 'application/x-www-form-urlencoded' }, method: 'DELETE' }, { status: 200 - }, function(res) { + }, function(err, res) { var cancelledJob = JSON.parse(res.body); assert.equal(cancelledJob.status, "cancelled"); done(); diff --git a/test/acceptance/job.use-case-8.test.js b/test/acceptance/batch/job.use-case-8.test.js similarity index 67% rename from test/acceptance/job.use-case-8.test.js rename to test/acceptance/batch/job.use-case-8.test.js index 2ea0fa19d..be1eb3c5d 100644 --- a/test/acceptance/job.use-case-8.test.js +++ b/test/acceptance/batch/job.use-case-8.test.js @@ -12,40 +12,33 @@ * HSET rails:users:vizzuality database_name cartodb_test_user_1_db * */ -require('../helper'); +require('../../helper'); -var app = require(global.settings.app_root + '/app/app')(); -var assert = require('../support/assert'); +var server = require('../../../app/server')(); +var assert = require('../../support/assert'); +var redisUtils = require('../../support/redis_utils'); var querystring = require('querystring'); -var metadataBackend = require('cartodb-redis')({ - host: global.settings.redis_host, - port: global.settings.redis_port, - max: global.settings.redisPool, - idleTimeoutMillis: global.settings.redisIdleTimeoutMillis, - reapIntervalMillis: global.settings.redisReapIntervalMillis -}); -var batchFactory = require('../../batch'); +var metadataBackend = require('cartodb-redis')({ pool: redisUtils.getPool() }); +var batchFactory = require('../../../batch'); describe('Use case 8: cancel a running multiquery job', function() { + var batch = batchFactory(metadataBackend, redisUtils.getPool()); - var batch = batchFactory(metadataBackend); - - before(function () { + before(function (done) { batch.start(); + batch.on('ready', done); }); after(function (done) { batch.stop(); - batch.drain(function () { - metadataBackend.redisCmd(5, 'DEL', [ 'batch:queues:localhost' ], done); - }); + redisUtils.clean('batch:*', done); }); var runningJob = {}; var cancelledJob = {}; it('Step 1, should create a new multiquery job', function (done) { - assert.response(app, { + assert.response(server, { url: '/api/v2/sql/job?api_key=1234', headers: { 'host': 'vizzuality.cartodb.com', 'Content-Type': 'application/x-www-form-urlencoded' }, method: 'POST', @@ -58,7 +51,7 @@ describe('Use case 8: cancel a running multiquery job', function() { }) }, { status: 201 - }, function(res) { + }, function(err, res) { runningJob = JSON.parse(res.body); done(); }); @@ -66,13 +59,13 @@ describe('Use case 8: cancel a running multiquery job', function() { it('Step 2, multiquery job should be running', function (done){ var interval = setInterval(function () { - assert.response(app, { + assert.response(server, { url: '/api/v2/sql/job/' + runningJob.job_id + '?api_key=1234', headers: { 'host': 'vizzuality.cartodb.com', 'Content-Type': 'application/x-www-form-urlencoded' }, method: 'GET' }, { status: 200 - }, function(res) { + }, function(err, res) { var job = JSON.parse(res.body); if (job.status === "running") { clearInterval(interval); @@ -86,13 +79,13 @@ describe('Use case 8: cancel a running multiquery job', function() { }); it('Step 3, cancel a multiquery job', function (done){ - assert.response(app, { + assert.response(server, { url: '/api/v2/sql/job/' + runningJob.job_id + '?api_key=1234', headers: { 'host': 'vizzuality.cartodb.com', 'Content-Type': 'application/x-www-form-urlencoded' }, method: 'DELETE' }, { status: 200 - }, function(res) { + }, function(err, res) { cancelledJob = JSON.parse(res.body); assert.equal(cancelledJob.status, "cancelled"); done(); @@ -100,13 +93,13 @@ describe('Use case 8: cancel a running multiquery job', function() { }); it('Step 4, multiquery job should be cancelled', function (done){ - assert.response(app, { + assert.response(server, { url: '/api/v2/sql/job/' + runningJob.job_id + '?api_key=1234', headers: { 'host': 'vizzuality.cartodb.com', 'Content-Type': 'application/x-www-form-urlencoded' }, method: 'GET' }, { status: 200 - }, function(res) { + }, function(err, res) { var job = JSON.parse(res.body); if (job.status === "cancelled") { done(); @@ -117,38 +110,16 @@ describe('Use case 8: cancel a running multiquery job', function() { }); it('Step 5, cancel a cancelled multiquery job should give an error', function (done) { - assert.response(app, { + assert.response(server, { url: '/api/v2/sql/job/' + cancelledJob.job_id + '?api_key=1234', headers: { 'host': 'vizzuality.cartodb.com', 'Content-Type': 'application/x-www-form-urlencoded' }, method: 'DELETE' }, { status: 400 - }, function(res) { + }, function(err, res) { var errors = JSON.parse(res.body); assert.equal(errors.error[0], "Cannot set status from cancelled to cancelled"); done(); }); }); - - it('Step 5, modify a cancelled multiquery job should give an error', function (done){ - assert.response(app, { - url: '/api/v2/sql/job/' + cancelledJob.job_id + '?api_key=1234', - headers: { 'host': 'vizzuality.cartodb.com', 'Content-Type': 'application/x-www-form-urlencoded' }, - method: 'PUT', - data: querystring.stringify({ - query: [ - "select pg_sleep(1)", - "select pg_sleep(1)", - "select pg_sleep(1)", - "select pg_sleep(1)" - ] - }) - }, { - status: 400 - }, function(res) { - var errors = JSON.parse(res.body); - assert.equal(errors.error[0], "Job is not pending, it cannot be updated"); - done(); - }); - }); }); diff --git a/test/acceptance/job.use-case-9.test.js b/test/acceptance/batch/job.use-case-9.test.js similarity index 62% rename from test/acceptance/job.use-case-9.test.js rename to test/acceptance/batch/job.use-case-9.test.js index ab56ee7c6..c0e81af02 100644 --- a/test/acceptance/job.use-case-9.test.js +++ b/test/acceptance/batch/job.use-case-9.test.js @@ -12,40 +12,33 @@ * HSET rails:users:vizzuality database_name cartodb_test_user_1_db * */ -require('../helper'); +require('../../helper'); -var app = require(global.settings.app_root + '/app/app')(); -var assert = require('../support/assert'); +var server = require('../../../app/server')(); +var assert = require('../../support/assert'); +var redisUtils = require('../../support/redis_utils'); var querystring = require('querystring'); -var metadataBackend = require('cartodb-redis')({ - host: global.settings.redis_host, - port: global.settings.redis_port, - max: global.settings.redisPool, - idleTimeoutMillis: global.settings.redisIdleTimeoutMillis, - reapIntervalMillis: global.settings.redisReapIntervalMillis -}); -var batchFactory = require('../../batch'); +var metadataBackend = require('cartodb-redis')({ pool: redisUtils.getPool() }); +var batchFactory = require('../../../batch'); describe('Use case 9: modify a pending multiquery job', function() { + var batch = batchFactory(metadataBackend, redisUtils.getPool()); - var batch = batchFactory(metadataBackend); - - before(function () { + before(function (done) { batch.start(); + batch.on('ready', done); }); after(function (done) { batch.stop(); - batch.drain(function () { - metadataBackend.redisCmd(5, 'DEL', [ 'batch:queues:localhost' ], done); - }); + redisUtils.clean('batch:*', done); }); var runningJob = {}; var pendingJob = {}; it('Step 1, should create a multiquery job', function (done) { - assert.response(app, { + assert.response(server, { url: '/api/v2/sql/job?api_key=1234', headers: { 'host': 'vizzuality.cartodb.com', 'Content-Type': 'application/x-www-form-urlencoded' }, method: 'POST', @@ -57,14 +50,14 @@ describe('Use case 9: modify a pending multiquery job', function() { }) }, { status: 201 - }, function(res) { + }, function(err, res) { runningJob = JSON.parse(res.body); done(); }); }); it('Step 2, should create another multiquery job', function (done) { - assert.response(app, { + assert.response(server, { url: '/api/v2/sql/job?api_key=1234', headers: { 'host': 'vizzuality.cartodb.com', 'Content-Type': 'application/x-www-form-urlencoded' }, method: 'POST', @@ -76,7 +69,7 @@ describe('Use case 9: modify a pending multiquery job', function() { }) }, { status: 201 - }, function(res) { + }, function(err, res) { pendingJob = JSON.parse(res.body); done(); }); @@ -84,13 +77,13 @@ describe('Use case 9: modify a pending multiquery job', function() { it('Step 3, multiquery job should be pending', function (done){ var interval = setInterval(function () { - assert.response(app, { + assert.response(server, { url: '/api/v2/sql/job/' + pendingJob.job_id + '?api_key=1234', headers: { 'host': 'vizzuality.cartodb.com', 'Content-Type': 'application/x-www-form-urlencoded' }, method: 'GET' }, { status: 200 - }, function(res) { + }, function(err, res) { var job = JSON.parse(res.body); if (job.status === "pending") { clearInterval(interval); @@ -103,41 +96,14 @@ describe('Use case 9: modify a pending multiquery job', function() { }, 50); }); - it('Step 4, multiquery job should be modified', function (done) { - assert.response(app, { - url: '/api/v2/sql/job/' + pendingJob.job_id + '?api_key=1234', - headers: { 'host': 'vizzuality.cartodb.com', 'Content-Type': 'application/x-www-form-urlencoded' }, - method: 'PUT', - data: querystring.stringify({ - query: [ - "SELECT * FROM untitle_table_4", - "SELECT * FROM untitle_table_4 limit 1" - ] - }) - }, { - status: 200 - }, function(res) { - var jobGot = JSON.parse(res.body); - assert.equal(jobGot.job_id, pendingJob.job_id); - assert.deepEqual(jobGot.query, [{ - query: 'SELECT * FROM untitle_table_4', - status: 'pending' - }, { - query: 'SELECT * FROM untitle_table_4 limit 1', - status: 'pending' - }]); - done(); - }); - }); - it('Step 5, running multiquery job should be cancelled', function (done){ - assert.response(app, { + assert.response(server, { url: '/api/v2/sql/job/' + runningJob.job_id + '?api_key=1234', headers: { 'host': 'vizzuality.cartodb.com', 'Content-Type': 'application/x-www-form-urlencoded' }, method: 'DELETE' }, { status: 200 - }, function(res) { + }, function(err, res) { var cancelledJob = JSON.parse(res.body); assert.equal(cancelledJob.status, "cancelled"); done(); diff --git a/test/acceptance/batch/leader-multiple-users-query-order.test.js b/test/acceptance/batch/leader-multiple-users-query-order.test.js new file mode 100644 index 000000000..782dee196 --- /dev/null +++ b/test/acceptance/batch/leader-multiple-users-query-order.test.js @@ -0,0 +1,127 @@ +require('../../helper'); +var assert = require('../../support/assert'); + +var TestClient = require('../../support/test-client'); +var BatchTestClient = require('../../support/batch-test-client'); +var JobStatus = require('../../../batch/job_status'); + +describe('multiple batch clients and users, job query order', function() { + + before(function(done) { + this.batchTestClientA = new BatchTestClient({ name: 'consumerA' }); + this.batchTestClientB = new BatchTestClient({ name: 'consumerB' }); + + this.testClient = new TestClient(); + this.testClient.getResult( + [ + 'drop table if exists ordered_inserts_a', + 'drop table if exists ordered_inserts_bbbbb', + 'create table ordered_inserts_a (status numeric)', + 'create table ordered_inserts_bbbbb (status numeric)' + ].join(';'), + done + ); + }); + + after(function (done) { + this.batchTestClientA.drain(function(err) { + if (err) { + return done(err); + } + + this.batchTestClientB.drain(done); + }.bind(this)); + }); + + function createJob(queries) { + return { + query: queries + }; + } + + it('should run job queries in order (multiple consumers)', function (done) { + var jobRequestA1 = createJob([ + "insert into ordered_inserts_a values(1)", + "select pg_sleep(0.25)", + "insert into ordered_inserts_a values(2)" + ]); + var jobRequestA2 = createJob([ + "insert into ordered_inserts_a values(3)" + ]); + + var jobRequestB1 = createJob([ + "insert into ordered_inserts_bbbbb values(1)" + ]); + + var self = this; + + this.batchTestClientA.createJob(jobRequestA1, function(err, jobResultA1) { + if (err) { + return done(err); + } + + var override = { host: 'cartodb250user.cartodb.com' }; + self.batchTestClientB.createJob(jobRequestB1, override, function(err, jobResultB1) { + if (err) { + return done(err); + } + + // we don't care about the producer + self.batchTestClientB.createJob(jobRequestA2, function(err, jobResultA2) { + if (err) { + return done(err); + } + + jobResultA1.getStatus(function (err, jobA1) { + if (err) { + return done(err); + } + jobResultA2.getStatus(function(err, jobA2) { + if (err) { + return done(err); + } + jobResultB1.getStatus(function(err, jobB1) { + assert.equal(jobA1.status, JobStatus.DONE); + assert.equal(jobA2.status, JobStatus.DONE); + assert.equal(jobB1.status, JobStatus.DONE); + + assert.ok( + new Date(jobA1.updated_at).getTime() < new Date(jobA2.updated_at).getTime(), + 'A1 (' + jobA1.updated_at + ') ' + + 'should finish before A2 (' + jobA2.updated_at + ')' + ); + assert.ok( + new Date(jobB1.updated_at).getTime() < new Date(jobA1.updated_at).getTime(), + 'B1 (' + jobA1.updated_at + ') ' + + 'should finish before A1 (' + jobA1.updated_at + ')' + ); + + function statusMapper (status) { return { status: status }; } + + self.testClient.getResult('select * from ordered_inserts_a', function(err, rows) { + assert.ok(!err); + + // cartodb250user and vizzuality test users share database + var expectedRows = [1, 2, 3].map(statusMapper); + assert.deepEqual(rows, expectedRows); + + var query = 'select * from ordered_inserts_bbbbb'; + self.testClient.getResult(query, override, function(err, rows) { + assert.ok(!err); + + var expectedRows = [1].map(statusMapper); + assert.deepEqual(rows, expectedRows); + + done(); + }); + }); + }); + + }); + }); + }); + }); + }); + }); + +}); diff --git a/test/acceptance/batch/leader.job.query.order.test.js b/test/acceptance/batch/leader.job.query.order.test.js new file mode 100644 index 000000000..962ebda91 --- /dev/null +++ b/test/acceptance/batch/leader.job.query.order.test.js @@ -0,0 +1,86 @@ +require('../../helper'); +var assert = require('../../support/assert'); + +var TestClient = require('../../support/test-client'); +var BatchTestClient = require('../../support/batch-test-client'); +var JobStatus = require('../../../batch/job_status'); + +describe('multiple batch clients job query order', function() { + + before(function(done) { + this.batchTestClient1 = new BatchTestClient({ name: 'consumerA' }); + this.batchTestClient2 = new BatchTestClient({ name: 'consumerB' }); + + this.testClient = new TestClient(); + this.testClient.getResult( + 'drop table if exists ordered_inserts; create table ordered_inserts (status numeric)', + done + ); + }); + + after(function (done) { + this.batchTestClient1.drain(function(err) { + if (err) { + return done(err); + } + + this.batchTestClient2.drain(done); + }.bind(this)); + }); + + function createJob(queries) { + return { + query: queries + }; + } + + it('should run job queries in order (multiple consumers)', function (done) { + var jobRequest1 = createJob([ + "insert into ordered_inserts values(1)", + "select pg_sleep(0.25)", + "insert into ordered_inserts values(2)" + ]); + var jobRequest2 = createJob([ + "insert into ordered_inserts values(3)" + ]); + + var self = this; + + this.batchTestClient1.createJob(jobRequest1, function(err, jobResult1) { + if (err) { + return done(err); + } + this.batchTestClient2.createJob(jobRequest2, function(err, jobResult2) { + if (err) { + return done(err); + } + + jobResult1.getStatus(function (err, job1) { + if (err) { + return done(err); + } + jobResult2.getStatus(function(err, job2) { + if (err) { + return done(err); + } + assert.equal(job1.status, JobStatus.DONE); + assert.equal(job2.status, JobStatus.DONE); + + self.testClient.getResult('select * from ordered_inserts', function(err, rows) { + assert.ok(!err); + + assert.deepEqual(rows, [{ status: 1 }, { status: 2 }, { status: 3 }]); + assert.ok( + new Date(job1.updated_at).getTime() < new Date(job2.updated_at).getTime(), + 'job1 (' + job1.updated_at + ') should finish before job2 (' + job2.updated_at + ')' + ); + done(); + }); + + }); + }); + }); + }.bind(this)); + }); + +}); diff --git a/test/acceptance/batch/queued-jobs-limit.test.js b/test/acceptance/batch/queued-jobs-limit.test.js new file mode 100644 index 000000000..f62089b8f --- /dev/null +++ b/test/acceptance/batch/queued-jobs-limit.test.js @@ -0,0 +1,83 @@ +require('../../helper'); + +var assert = require('../../support/assert'); +var redisUtils = require('../../support/redis_utils'); +var batchFactory = require('../../../batch/index'); +var metadataBackend = require('cartodb-redis')({ pool: redisUtils.getPool() }); +var TestClient = require('../../support/test-client'); + +describe('max queued jobs', function() { + + before(function(done) { + this.batch_max_queued_jobs = global.settings.batch_max_queued_jobs; + global.settings.batch_max_queued_jobs = 1; + this.server = require('../../../app/server')(); + this.testClient = new TestClient(); + this.testClient.getResult( + 'drop table if exists max_queued_jobs_inserts; create table max_queued_jobs_inserts (status numeric)', + done + ); + }); + + after(function (done) { + var self = this; + global.settings.batch_max_queued_jobs = this.batch_max_queued_jobs; + var batch = batchFactory(metadataBackend, redisUtils.getPool()); + batch.start(); + batch.on('ready', function() { + // this is not ideal as the first job might not be committed yet + setTimeout(function() { + batch.stop(function() { + self.testClient.getResult('select count(*) from max_queued_jobs_inserts', function(err, rows) { + assert.ok(!err); + assert.equal(rows[0].count, 1); + + redisUtils.clean('batch:*', done); + }); + }); + }, 100); + }); + }); + + function createJob(server, status, callback) { + assert.response( + server, + { + url: '/api/v2/sql/job?api_key=1234', + headers: { + host: 'vizzuality.cartodb.com', + 'Content-Type': 'application/json' + }, + method: 'POST', + data: JSON.stringify({ + query: "insert into max_queued_jobs_inserts values (1)" + }) + }, + { + status: status + }, + function(err, res) { + if (err) { + return callback(err); + } + + return callback(null, JSON.parse(res.body)); + } + ); + } + + it('POST /api/v2/sql/job should respond with 200 and the created job', function (done) { + var self = this; + createJob(this.server, 201, function(err) { + assert.ok(!err); + + createJob(self.server, 400, function(err, res) { + assert.ok(!err); + assert.equal(res.error[0], "Failed to create job. Max number of jobs (" + + global.settings.batch_max_queued_jobs + ") queued reached"); + done(); + }); + }); + }); + +}); diff --git a/test/acceptance/batch/scheduler-basic.test.js b/test/acceptance/batch/scheduler-basic.test.js new file mode 100644 index 000000000..ba5decbd7 --- /dev/null +++ b/test/acceptance/batch/scheduler-basic.test.js @@ -0,0 +1,97 @@ +require('../../helper'); +var assert = require('../../support/assert'); + +var TestClient = require('../../support/test-client'); +var BatchTestClient = require('../../support/batch-test-client'); +var JobStatus = require('../../../batch/job_status'); + +describe('basic scheduling', function() { + + before(function(done) { + this.batchTestClientA = new BatchTestClient({ name: 'consumerA' }); + this.batchTestClientB = new BatchTestClient({ name: 'consumerB' }); + + this.testClient = new TestClient(); + this.testClient.getResult( + [ + 'drop table if exists ordered_inserts_a', + 'create table ordered_inserts_a (status numeric)' + ].join(';'), + done + ); + }); + + after(function (done) { + this.batchTestClientA.drain(function(err) { + if (err) { + return done(err); + } + + this.batchTestClientB.drain(done); + }.bind(this)); + }); + + function createJob(queries) { + return { + query: queries + }; + } + + it('should run job queries in order (multiple consumers)', function (done) { + var jobRequestA1 = createJob([ + "insert into ordered_inserts_a values(1)", + "select pg_sleep(0.25)", + "insert into ordered_inserts_a values(2)" + ]); + var jobRequestA2 = createJob([ + "insert into ordered_inserts_a values(3)" + ]); + + var self = this; + + this.batchTestClientA.createJob(jobRequestA1, function(err, jobResultA1) { + if (err) { + return done(err); + } + + // we don't care about the producer + self.batchTestClientB.createJob(jobRequestA2, function(err, jobResultA2) { + if (err) { + return done(err); + } + + jobResultA1.getStatus(function (err, jobA1) { + if (err) { + return done(err); + } + + jobResultA2.getStatus(function(err, jobA2) { + if (err) { + return done(err); + } + assert.equal(jobA1.status, JobStatus.DONE); + assert.equal(jobA2.status, JobStatus.DONE); + + assert.ok( + new Date(jobA1.updated_at).getTime() < new Date(jobA2.updated_at).getTime(), + 'A1 (' + jobA1.updated_at + ') ' + + 'should finish before A2 (' + jobA2.updated_at + ')' + ); + + function statusMapper (status) { return { status: status }; } + + self.testClient.getResult('select * from ordered_inserts_a', function(err, rows) { + assert.ok(!err); + + // cartodb250user and vizzuality test users share database + var expectedRows = [1, 2, 3].map(statusMapper); + assert.deepEqual(rows, expectedRows); + + return done(); + }); + }); + }); + }); + }); + }); +}); diff --git a/test/acceptance/export/arraybuffer.js b/test/acceptance/export/arraybuffer.js index e3f7b66c6..2a297b85a 100644 --- a/test/acceptance/export/arraybuffer.js +++ b/test/acceptance/export/arraybuffer.js @@ -2,24 +2,21 @@ require('../../helper'); require('../../support/assert'); -var app = require(global.settings.app_root + '/app/app')(); +var server = require('../../../app/server')(); var assert = require('assert'); var querystring = require('querystring'); -// allow lots of emitters to be set to silence warning -app.setMaxListeners(0); - describe('export.arraybuffer', function() { it('GET /api/v1/sql as arraybuffer ', function(done){ - assert.response(app, { + assert.response(server, { url: '/api/v1/sql?' + querystring.stringify({ q: 'SELECT cartodb_id,name,1::integer,187.9 FROM untitle_table_4', format: 'arraybuffer' }), headers: {host: 'vizzuality.cartodb.com'}, method: 'GET' - },{ }, function(res){ + },{ }, function(err, res){ assert.equal(res.statusCode, 200, res.body); assert.equal(res.headers['content-type'], "application/octet-stream"); done(); @@ -27,14 +24,14 @@ it('GET /api/v1/sql as arraybuffer ', function(done){ }); it('GET /api/v1/sql as arraybuffer does not support geometry types ', function(done){ - assert.response(app, { + assert.response(server, { url: '/api/v1/sql?' + querystring.stringify({ q: 'SELECT cartodb_id, the_geom FROM untitle_table_4', format: 'arraybuffer' }), headers: {host: 'vizzuality.cartodb.com'}, method: 'GET' - },{ }, function(res){ + },{ }, function(err, res){ assert.equal(res.statusCode, 400, res.body); var result = JSON.parse(res.body); assert.equal(result.error[0], "geometry types are not supported"); diff --git a/test/acceptance/export/csv.js b/test/acceptance/export/csv.js index 180b5db1b..98ea94a97 100644 --- a/test/acceptance/export/csv.js +++ b/test/acceptance/export/csv.js @@ -2,29 +2,26 @@ require('../../helper'); require('../../support/assert'); -var app = require(global.settings.app_root + '/app/app')(); +var server = require('../../../app/server')(); var assert = require('assert'); var querystring = require('querystring'); -// allow lots of emitters to be set to silence warning -app.setMaxListeners(0); - describe('export.csv', function() { it('CSV format', function(done){ - assert.response(app, { + assert.response(server, { url: '/api/v1/sql?' + querystring.stringify({ q: 'SELECT * FROM untitle_table_4 WHERE cartodb_id = 1', format: 'csv' }), headers: {host: 'vizzuality.cartodb.com'}, method: 'GET' - },{ }, function(res){ + },{ }, function(err, res){ assert.equal(res.statusCode, 200, res.body); - var cd = res.header('Content-Disposition'); + var cd = res.headers['content-disposition']; assert.equal(true, /^attachment/.test(cd), 'CSV is not disposed as attachment: ' + cd); assert.equal(true, /filename=cartodb-query.csv/gi.test(cd)); - var ct = res.header('Content-Type'); + var ct = res.headers['content-type']; assert.equal(true, /header=present/.test(ct), "CSV doesn't advertise header presence: " + ct); var rows = res.body.split(/\r\n/); @@ -39,7 +36,7 @@ it('CSV format', function(done){ }); it('CSV format, bigger than 81920 bytes', function(done){ - assert.response(app, { + assert.response(server, { url: '/api/v1/sql', data: querystring.stringify({ q: 'SELECT 0 as fname FROM generate_series(0,81920)', @@ -47,7 +44,7 @@ it('CSV format, bigger than 81920 bytes', function(done){ }), headers: {host: 'vizzuality.cartodb.com', 'Content-Type': 'application/x-www-form-urlencoded' }, method: 'POST' - },{ }, function(res){ + },{ }, function(err, res){ assert.ok(res.body.length > 81920, 'CSV smaller than expected: ' + res.body.length); done(); }); @@ -55,33 +52,33 @@ it('CSV format, bigger than 81920 bytes', function(done){ it('CSV format from POST', function(done){ - assert.response(app, { + assert.response(server, { url: '/api/v1/sql', data: querystring.stringify({q: "SELECT * FROM untitle_table_4 LIMIT 1", format: 'csv'}), headers: {host: 'vizzuality.cartodb.com', 'Content-Type': 'application/x-www-form-urlencoded' }, method: 'POST' - },{ }, function(res){ + },{ }, function(err, res){ assert.equal(res.statusCode, 200, res.body); - var cd = res.header('Content-Disposition'); + var cd = res.headers['content-disposition']; assert.equal(true, /^attachment/.test(cd), 'CSV is not disposed as attachment: ' + cd); assert.equal(true, /filename=cartodb-query.csv/gi.test(cd)); - var ct = res.header('Content-Type'); + var ct = res.headers['content-type']; assert.equal(true, /header=present/.test(ct), "CSV doesn't advertise header presence: " + ct); done(); }); }); it('CSV format, custom filename', function(done){ - assert.response(app, { + assert.response(server, { url: '/api/v1/sql?q=SELECT%20*%20FROM%20untitle_table_4%20LIMIT%201&format=csv&filename=mycsv.csv', headers: {host: 'vizzuality.cartodb.com'}, method: 'GET' - },{ }, function(res){ + },{ }, function(err, res){ assert.equal(res.statusCode, 200, res.body); - var cd = res.header('Content-Disposition'); + var cd = res.headers['content-disposition']; assert.equal(true, /^attachment/.test(cd), 'CSV is not disposed as attachment: ' + cd); assert.equal(true, /filename=mycsv.csv/gi.test(cd), cd); - var ct = res.header('Content-Type'); + var ct = res.headers['content-type']; assert.equal(true, /header=present/.test(ct), "CSV doesn't advertise header presence: " + ct); var row0 = res.body.substring(0, res.body.search(/[\n\r]/)).split(','); var checkFields = { name: true, cartodb_id: true, the_geom: true, the_geom_webmercator: true }; @@ -98,12 +95,12 @@ it('CSV format, custom filename', function(done){ }); it('skipfields controls fields included in CSV output', function(done){ - assert.response(app, { + assert.response(server, { url: '/api/v1/sql?q=SELECT%20*%20FROM%20untitle_table_4%20LIMIT%201&format=csv' + '&skipfields=unexistant,cartodb_id', headers: {host: 'vizzuality.cartodb.com'}, method: 'GET' - },{ }, function(res){ + },{ }, function(err, res){ assert.equal(res.statusCode, 200, res.body); var row0 = res.body.substring(0, res.body.search(/[\n\r]/)).split(','); var checkFields = { name: true, cartodb_id: false, the_geom: true, the_geom_webmercator: true }; @@ -120,12 +117,12 @@ it('skipfields controls fields included in CSV output', function(done){ }); it('GET /api/v1/sql as csv', function(done){ - assert.response(app, { + assert.response(server, { url: '/api/v1/sql?q=SELECT%20cartodb_id,ST_AsEWKT(the_geom)%20as%20geom%20FROM%20untitle_table_4%20LIMIT%201' + '&format=csv', headers: {host: 'vizzuality.cartodb.com'}, method: 'GET' - },{ }, function(res){ + },{ }, function(err, res){ assert.equal(res.statusCode, 200, res.body); var expected = 'cartodb_id,geom\r\n1,"SRID=4326;POINT(-3.699732 40.423012)"\r\n'; assert.equal(res.body, expected); @@ -135,11 +132,11 @@ it('GET /api/v1/sql as csv', function(done){ // See https://github.com/Vizzuality/CartoDB-SQL-API/issues/60 it('GET /api/v1/sql as csv with no rows', function(done){ - assert.response(app, { + assert.response(server, { url: '/api/v1/sql?q=SELECT%20true%20WHERE%20false&format=csv', headers: {host: 'vizzuality.cartodb.com'}, method: 'GET' - },{ }, function(res){ + },{ }, function(err, res){ assert.equal(res.statusCode, 200, res.body); var obtained_lines = res.body.split('\r\n'); assert.ok(obtained_lines.length <= 2, // may or may not have an header @@ -150,11 +147,11 @@ it('GET /api/v1/sql as csv with no rows', function(done){ }); it('GET /api/v1/sql as csv, properly escaped', function(done){ - assert.response(app, { + assert.response(server, { url: '/api/v1/sql?q=SELECT%20cartodb_id,%20address%20FROM%20untitle_table_4%20LIMIT%201&format=csv', headers: {host: 'vizzuality.cartodb.com'}, method: 'GET' - },{ }, function(res){ + },{ }, function(err, res){ assert.equal(res.statusCode, 200, res.body); var expected = 'cartodb_id,address\r\n1,"Calle de Pérez Galdós 9, Madrid, Spain"\r\n'; assert.equal(res.body, expected); @@ -166,7 +163,7 @@ it('GET /api/v1/sql as csv, concurrently', function(done){ var concurrency = 4; var waiting = concurrency; - function validate(res){ + function validate(err, res){ var expected = 'cartodb_id,address\r\n1,"Calle de Pérez Galdós 9, Madrid, Spain"\r\n'; assert.equal(res.body, expected); if ( ! --waiting ) { @@ -174,7 +171,7 @@ it('GET /api/v1/sql as csv, concurrently', function(done){ } } for (var i=0; i 81920, 'KML smaller than expected: ' + res.body.length); @@ -166,13 +161,13 @@ it('KML format, bigger than 81920 bytes', function(done){ }); it('KML format, skipfields', function(done){ - assert.response(app, { + assert.response(server, { url: '/api/v1/sql?q=SELECT%20*%20FROM%20untitle_table_4%20LIMIT%201&format=kml&skipfields=address,cartodb_id', headers: {host: 'vizzuality.cartodb.com'}, method: 'GET' - },{ }, function(res){ + },{ }, function(err, res){ assert.equal(res.statusCode, 200, res.body); - var cd = res.header('Content-Disposition'); + var cd = res.headers['content-disposition']; assert.equal(true, /^attachment/.test(cd), 'KML is not disposed as attachment: ' + cd); assert.equal(true, /filename=cartodb-query.kml/gi.test(cd), 'Unexpected KML filename: ' + cd); var row0 = res.body; @@ -189,13 +184,13 @@ it('KML format, skipfields', function(done){ }); it('KML format, unauthenticated, custom filename', function(done){ - assert.response(app, { + assert.response(server, { url: '/api/v1/sql?q=SELECT%20*%20FROM%20untitle_table_4%20LIMIT%201&format=kml&filename=kmltest', headers: {host: 'vizzuality.cartodb.com'}, method: 'GET' - },{ }, function(res){ + },{ }, function(err, res){ assert.equal(res.statusCode, 200, res.body); - var cd = res.header('Content-Disposition'); + var cd = res.headers['content-disposition']; assert.equal(true, /^attachment/.test(cd), 'KML is not disposed as attachment: ' + cd); assert.equal(true, /filename=kmltest.kml/gi.test(cd), 'Unexpected KML filename: ' + cd); var name = extractFolderName(res.body); @@ -205,13 +200,13 @@ it('KML format, unauthenticated, custom filename', function(done){ }); it('KML format, authenticated', function(done){ - assert.response(app, { + assert.response(server, { url: '/api/v1/sql?q=SELECT%20*%20FROM%20untitle_table_4%20LIMIT%201&format=kml&api_key=1234', headers: {host: 'vizzuality.cartodb.com'}, method: 'GET' - },{ }, function(res){ + },{ }, function(err, res){ assert.equal(res.statusCode, 200, res.body); - var cd = res.header('Content-Disposition'); + var cd = res.headers['content-disposition']; assert.equal(true, /filename=cartodb-query.kml/gi.test(cd), 'Unexpected KML filename: ' + cd); done(); }); @@ -228,57 +223,39 @@ it('KML format, unauthenticated, concurrent requests', function(done){ var concurrency = 4; var waiting = concurrency; - function onResponse(res) { - //console.log("Response started"); - res.body = ''; - //res.setEncoding('binary'); - res.on('data', function(chunk){ res.body += chunk; }); - res.on('end', function(){ - //console.log("Response ended"); - assert.equal(res.statusCode, 200, res.body); - assert.ok(res.body); - var snippet = res.body.substr(0, 5); - assert.equal(snippet, "' + @@ -294,14 +271,14 @@ it('GET /api/v1/sql as kml with no rows', function(done){ // See https://github.com/Vizzuality/CartoDB-SQL-API/issues/90 it('GET /api/v1/sql as kml with ending semicolon', function(done){ - assert.response(app, { + assert.response(server, { url: '/api/v1/sql?' + querystring.stringify({ q: 'SELECT true WHERE false;', format: 'kml' }), headers: {host: 'vizzuality.cartodb.com'}, method: 'GET' - },{ }, function(res){ + },{ }, function(err, res){ assert.equal(res.statusCode, 200, res.body); // NOTE: GDAL-1.11+ added 'id="root_doc"' attribute to the output var pat = new RegExp('^<\\?xml version="1.0" encoding="utf-8" \\?>' + @@ -317,14 +294,14 @@ it('GET /api/v1/sql as kml with ending semicolon', function(done){ // See https://github.com/CartoDB/cartodb/issues/276 it('check point coordinates, unauthenticated', function(done){ - assert.response(app, { + assert.response(server, { url: '/api/v1/sql?' + querystring.stringify({ q: 'SELECT * from untitle_table_4 WHERE cartodb_id = -1', format: 'kml' }), headers: {host: 'vizzuality.cartodb.com'}, method: 'GET' - },{ }, function(res){ + },{ }, function(err, res){ assert.equal(res.statusCode, 200, res.body); var coords = extractCoordinates(res.body); assert(coords, 'No coordinates in ' + res.body); @@ -335,7 +312,7 @@ it('check point coordinates, unauthenticated', function(done){ // See https://github.com/CartoDB/cartodb/issues/276 it('check point coordinates, authenticated', function(done){ - assert.response(app, { + assert.response(server, { url: '/api/v1/sql?' + querystring.stringify({ q: 'SELECT * from untitle_table_4 WHERE cartodb_id = -1', api_key: 1234, @@ -343,7 +320,7 @@ it('check point coordinates, authenticated', function(done){ }), headers: {host: 'vizzuality.cartodb.com'}, method: 'GET' - },{ }, function(res){ + },{ }, function(err, res){ assert.equal(res.statusCode, 200, res.body); var coords = extractCoordinates(res.body); assert(coords, 'No coordinates in ' + res.body); @@ -357,7 +334,7 @@ it('check point coordinates, authenticated', function(done){ it('expects ' + limit + ' placemarks in public table', function(done){ - assert.response(app, { + assert.response(server, { url: '/api/v1/sql', data: querystring.stringify({ q: "SELECT * from populated_places_simple_reduced limit " + limit, @@ -369,7 +346,7 @@ it('check point coordinates, authenticated', function(done){ { status: 200 }, - function(res) { + function(err, res) { assert.equal(res.body.match(//g).length, limit); done(); } @@ -378,7 +355,7 @@ it('check point coordinates, authenticated', function(done){ it('expects ' + limit + ' placemarks in private table using the API KEY', function(done){ - assert.response(app, { + assert.response(server, { url: '/api/v1/sql?' + querystring.stringify({ q: "SELECT * from populated_places_simple_reduced limit " + limit, api_key: 1234, @@ -390,7 +367,7 @@ it('check point coordinates, authenticated', function(done){ { status: 200 }, - function(res) { + function(err, res) { assert.equal(res.body.match(//g).length, limit); done(); } @@ -399,7 +376,7 @@ it('check point coordinates, authenticated', function(done){ it('should work with queries returning no results', function(done) { assert.response( - app, + server, { url: "/api/v1/sql?" + querystring.stringify({ q: "SELECT * FROM populated_places_simple_reduced LIMIT 0", @@ -414,7 +391,7 @@ it('check point coordinates, authenticated', function(done){ { status: 200 }, - function(res) { + function(err, res) { assert.equal(res.body.match(//g), null); done(); } diff --git a/test/acceptance/export/shapefile.js b/test/acceptance/export/shapefile.js index 30f3f7478..ec04a389c 100644 --- a/test/acceptance/export/shapefile.js +++ b/test/acceptance/export/shapefile.js @@ -1,6 +1,6 @@ require('../../helper'); -var app = require(global.settings.app_root + '/app/app')(); +var server = require('../../../app/server')(); var assert = require('../../support/assert'); var querystring = require('querystring'); var shapefile = require('shapefile'); @@ -8,28 +8,25 @@ var _ = require('underscore'); var zipfile = require('zipfile'); var fs = require('fs'); -// allow lots of emitters to be set to silence warning -app.setMaxListeners(0); - describe('export.shapefile', function() { // SHP tests it('SHP format, unauthenticated', function(done){ - assert.response(app, { + assert.response(server, { url: '/api/v1/sql?q=SELECT%20*%20FROM%20untitle_table_4%20LIMIT%201&format=shp', headers: {host: 'vizzuality.cartodb.com'}, encoding: 'binary', method: 'GET' - },{ }, function(res){ + },{ }, function(err, res){ assert.equal(res.statusCode, 200, res.body); - var cd = res.header('Content-Disposition'); + var cd = res.headers['content-disposition']; assert.equal(true, /^attachment/.test(cd), 'SHP is not disposed as attachment: ' + cd); assert.equal(true, /filename=cartodb-query.zip/gi.test(cd)); var tmpfile = '/tmp/myshape.zip'; - var err = fs.writeFileSync(tmpfile, res.body, 'binary'); - if (err) { - return done(err); + var writeErr = fs.writeFileSync(tmpfile, res.body, 'binary'); + if (writeErr) { + return done(writeErr); } var zf = new zipfile.ZipFile(tmpfile); assert.ok(_.contains(zf.names, 'cartodb-query.shp'), 'SHP zipfile does not contain .shp: ' + zf.names); @@ -43,14 +40,14 @@ it('SHP format, unauthenticated', function(done){ }); it('SHP format, unauthenticated, POST', function(done){ - assert.response(app, { + assert.response(server, { url: '/api/v1/sql', data: 'q=SELECT%20*%20FROM%20untitle_table_4%20LIMIT%201&format=shp', headers: {host: 'vizzuality.cartodb.com', 'Content-Type': 'application/x-www-form-urlencoded' }, method: 'POST' - },{ }, function(res){ + },{ }, function(err, res){ assert.equal(res.statusCode, 200, res.body); - var cd = res.header('Content-Disposition'); + var cd = res.headers['content-disposition']; assert.equal(true, /^attachment/.test(cd), 'SHP is not disposed as attachment: ' + cd); assert.equal(true, /filename=cartodb-query.zip/gi.test(cd), 'Unexpected SHP filename: ' + cd); done(); @@ -58,7 +55,7 @@ it('SHP format, unauthenticated, POST', function(done){ }); it('SHP format, big size, POST', function(done){ - assert.response(app, { + assert.response(server, { url: '/api/v1/sql', data: querystring.stringify({ q: 'SELECT 0 as fname, st_makepoint(i,i) FROM generate_series(0,81920) i', @@ -66,9 +63,9 @@ it('SHP format, big size, POST', function(done){ }), headers: {host: 'vizzuality.cartodb.com', 'Content-Type': 'application/x-www-form-urlencoded' }, method: 'POST' - },{ }, function(res){ + },{ }, function(err, res){ assert.equal(res.statusCode, 200, res.body); - var cd = res.header('Content-Disposition'); + var cd = res.headers['content-disposition']; assert.equal(true, /^attachment/.test(cd), 'SHP is not disposed as attachment: ' + cd); assert.equal(true, /filename=cartodb-query.zip/gi.test(cd), 'Unexpected SHP filename: ' + cd); assert.ok(res.body.length > 81920, 'SHP smaller than expected: ' + res.body.length); @@ -77,20 +74,20 @@ it('SHP format, big size, POST', function(done){ }); it('SHP format, unauthenticated, with custom filename', function(done){ - assert.response(app, { + assert.response(server, { url: '/api/v1/sql?q=SELECT%20*%20FROM%20untitle_table_4%20LIMIT%201&format=shp&filename=myshape', headers: {host: 'vizzuality.cartodb.com'}, encoding: 'binary', method: 'GET' - },{ }, function(res){ + },{ }, function(err, res){ assert.equal(res.statusCode, 200, res.body); - var cd = res.header('Content-Disposition'); + var cd = res.headers['content-disposition']; assert.equal(true, /^attachment/.test(cd), 'SHP is not disposed as attachment: ' + cd); assert.equal(true, /filename=myshape.zip/gi.test(cd)); var tmpfile = '/tmp/myshape.zip'; - var err = fs.writeFileSync(tmpfile, res.body, 'binary'); - if (err) { - return done(err); + var writeErr = fs.writeFileSync(tmpfile, res.body, 'binary'); + if (writeErr) { + return done(writeErr); } var zf = new zipfile.ZipFile(tmpfile); assert.ok(_.contains(zf.names, 'myshape.shp'), 'SHP zipfile does not contain .shp: ' + zf.names); @@ -103,21 +100,21 @@ it('SHP format, unauthenticated, with custom filename', function(done){ }); it('SHP format, unauthenticated, with custom, dangerous filename', function(done){ - assert.response(app, { + assert.response(server, { url: '/api/v1/sql?q=SELECT%20*%20FROM%20untitle_table_4%20LIMIT%201&format=shp&filename=b;"%20()[]a', headers: {host: 'vizzuality.cartodb.com'}, encoding: 'binary', method: 'GET' - },{ }, function(res){ + },{ }, function(err, res){ assert.equal(res.statusCode, 200, res.body); var fname = "b_______a"; - var cd = res.header('Content-Disposition'); + var cd = res.headers['content-disposition']; assert.equal(true, /^attachment/.test(cd), 'SHP is not disposed as attachment: ' + cd); assert.equal(true, /filename=b_______a.zip/gi.test(cd), 'Unexpected SHP filename: ' + cd); var tmpfile = '/tmp/myshape.zip'; - var err = fs.writeFileSync(tmpfile, res.body, 'binary'); - if (err) { - return done(err); + var writeErr = fs.writeFileSync(tmpfile, res.body, 'binary'); + if (writeErr) { + return done(writeErr); } var zf = new zipfile.ZipFile(tmpfile); assert.ok(_.contains(zf.names, fname + '.shp'), 'SHP zipfile does not contain .shp: ' + zf.names); @@ -130,19 +127,19 @@ it('SHP format, unauthenticated, with custom, dangerous filename', function(done }); it('SHP format, authenticated', function(done){ - assert.response(app, { + assert.response(server, { url: '/api/v1/sql?q=SELECT%20*%20FROM%20untitle_table_4%20LIMIT%201&format=shp&api_key=1234', headers: {host: 'vizzuality.cartodb.com'}, encoding: 'binary', method: 'GET' - },{ }, function(res){ + },{ }, function(err, res){ assert.equal(res.statusCode, 200, res.body); - var cd = res.header('Content-Disposition'); + var cd = res.headers['content-disposition']; assert.equal(true, /filename=cartodb-query.zip/gi.test(cd)); var tmpfile = '/tmp/myshape.zip'; - var err = fs.writeFileSync(tmpfile, res.body, 'binary'); - if (err) { - return done(err); + var writeErr = fs.writeFileSync(tmpfile, res.body, 'binary'); + if (writeErr) { + return done(writeErr); } var zf = new zipfile.ZipFile(tmpfile); assert.ok(_.contains(zf.names, 'cartodb-query.shp'), 'SHP zipfile does not contain .shp: ' + zf.names); @@ -163,17 +160,17 @@ it('SHP format, unauthenticated, with utf8 data', function(done){ format: 'shp', filename: 'myshape' }); - assert.response(app, { + assert.response(server, { url: '/api/v1/sql?' + query, headers: {host: 'vizzuality.cartodb.com'}, encoding: 'binary', method: 'GET' - },{ }, function(res){ + },{ }, function(err, res){ assert.equal(res.statusCode, 200, res.body); var tmpfile = '/tmp/myshape.zip'; - var err = fs.writeFileSync(tmpfile, res.body, 'binary'); - if (err) { - return done(err); + var writeErr = fs.writeFileSync(tmpfile, res.body, 'binary'); + if (writeErr) { + return done(writeErr); } var zf = new zipfile.ZipFile(tmpfile); var buffer = zf.readFileSync('myshape.dbf'); @@ -190,12 +187,12 @@ it('mixed type geometry', function(done){ q: "SELECT 'POINT(0 0)'::geometry as g UNION ALL SELECT 'LINESTRING(0 0, 1 0)'::geometry", format: 'shp' }); - assert.response(app, { + assert.response(server, { url: '/api/v1/sql?' + query, headers: {host: 'vizzuality.cartodb.com'}, encoding: 'binary', method: 'GET' - },{ }, function(res){ + },{ }, function(err, res){ assert.deepEqual(res.headers['content-type'], 'application/json; charset=utf-8'); assert.deepEqual(res.headers['content-disposition'], 'inline'); assert.equal(res.statusCode, 400, res.statusCode + ': ' +res.body); @@ -215,12 +212,12 @@ it('errors are not confused with warnings', function(done){ ].join(" UNION ALL "), format: 'shp' }); - assert.response(app, { + assert.response(server, { url: '/api/v1/sql?' + query, headers: {host: 'vizzuality.cartodb.com'}, encoding: 'binary', method: 'GET' - },{ }, function(res){ + },{ }, function(err, res){ assert.deepEqual(res.headers['content-type'], 'application/json; charset=utf-8'); assert.deepEqual(res.headers['content-disposition'], 'inline'); assert.equal(res.statusCode, 400, res.statusCode + ': ' +res.body); @@ -238,17 +235,17 @@ it('skipfields controls fields included in SHP output', function(done){ skipfields: 'skipme', filename: 'myshape' }); - assert.response(app, { + assert.response(server, { url: '/api/v1/sql?' + query, headers: {host: 'vizzuality.cartodb.com'}, encoding: 'binary', method: 'GET' - },{ }, function(res){ + },{ }, function(err, res){ assert.equal(res.statusCode, 200, res.body); var tmpfile = '/tmp/myshape.zip'; - var err = fs.writeFileSync(tmpfile, res.body, 'binary'); - if (err) { - return done(err); + var writeErr = fs.writeFileSync(tmpfile, res.body, 'binary'); + if (writeErr) { + return done(writeErr); } var zf = new zipfile.ZipFile(tmpfile); var buffer = zf.readFileSync('myshape.dbf'); @@ -262,14 +259,14 @@ it('skipfields controls fields included in SHP output', function(done){ it('SHP format, concurrently', function(done){ var concurrency = 1; var waiting = concurrency; - function validate(res){ - var cd = res.header('Content-Disposition'); + function validate(err, res){ + var cd = res.headers['content-disposition']; assert.equal(true, /^attachment/.test(cd), 'SHP is not disposed as attachment: ' + cd); assert.equal(true, /filename=cartodb-query.zip/gi.test(cd)); var tmpfile = '/tmp/myshape.zip'; - var err = fs.writeFileSync(tmpfile, res.body, 'binary'); - if (err) { - return done(err); + var writeErr = fs.writeFileSync(tmpfile, res.body, 'binary'); + if (writeErr) { + return done(writeErr); } var zf = new zipfile.ZipFile(tmpfile); assert.ok(_.contains(zf.names, 'cartodb-query.shp'), 'SHP zipfile does not contain .shp: ' + zf.names); @@ -284,7 +281,7 @@ it('SHP format, concurrently', function(done){ } for (var i=0; i') > 0, res.body ); // TODO: test viewBox done(); @@ -34,17 +31,17 @@ it('POST /api/v1/sql with SVG format', function(done){ q: "SELECT 1 as cartodb_id, ST_MakeLine(ST_MakePoint(10, 10), ST_MakePoint(1034, 778)) AS the_geom ", format: "svg" }); - assert.response(app, { + assert.response(server, { url: '/api/v1/sql', data: query, headers: {host: 'vizzuality.cartodb.com', 'Content-Type': 'application/x-www-form-urlencoded' }, method: 'POST' - },{ }, function(res){ + },{ }, function(err, res){ assert.equal(res.statusCode, 200, res.body); - var cd = res.header('Content-Disposition'); + var cd = res.headers['content-disposition']; assert.equal(true, /^attachment/.test(cd), 'SVG is not disposed as attachment: ' + cd); assert.ok(/filename=cartodb-query.svg/gi.test(cd), cd); - assert.equal(res.header('Content-Type'), 'image/svg+xml; charset=utf-8'); + assert.equal(res.headers['content-type'], 'image/svg+xml; charset=utf-8'); assert.ok( res.body.indexOf('') > 0, res.body ); // TODO: test viewBox done(); @@ -57,15 +54,15 @@ it('GET /api/v1/sql with SVG format and custom filename', function(done){ format: "svg", filename: 'mysvg' }); - assert.response(app, { + assert.response(server, { url: '/api/v1/sql?' + query, headers: {host: 'vizzuality.cartodb.com'}, method: 'GET' - },{ }, function(res){ + },{ }, function(err, res){ assert.equal(res.statusCode, 200, res.body); - var cd = res.header('Content-Disposition'); + var cd = res.headers['content-disposition']; assert.ok(/filename=mysvg.svg/gi.test(cd), cd); - assert.equal(res.header('Content-Type'), 'image/svg+xml; charset=utf-8'); + assert.equal(res.headers['content-type'], 'image/svg+xml; charset=utf-8'); assert.ok( res.body.indexOf('') > 0, res.body ); // TODO: test viewBox done(); @@ -77,15 +74,15 @@ it('GET /api/v1/sql with SVG format and centered point', function(done){ q: "SELECT 1 as cartodb_id, ST_MakePoint(5000, -54) AS the_geom ", format: "svg" }); - assert.response(app, { + assert.response(server, { url: '/api/v1/sql?' + query, headers: {host: 'vizzuality.cartodb.com'}, method: 'GET' - },{ }, function(res){ + },{ }, function(err, res){ assert.equal(res.statusCode, 200, res.body); - var cd = res.header('Content-Disposition'); + var cd = res.headers['content-disposition']; assert.ok(/filename=cartodb-query.svg/gi.test(cd), cd); - assert.equal(res.header('Content-Type'), 'image/svg+xml; charset=utf-8'); + assert.equal(res.headers['content-type'], 'image/svg+xml; charset=utf-8'); assert.ok( res.body.indexOf('cx="0" cy="0"') > 0, res.body ); // TODO: test viewBox // TODO: test radius @@ -99,29 +96,29 @@ it('GET /api/v1/sql with SVG format and trimmed decimals', function(done){ format: "svg", dp: 2 }; - assert.response(app, { + assert.response(server, { url: '/api/v1/sql?' + querystring.stringify(queryobj), headers: {host: 'vizzuality.cartodb.com'}, method: 'GET' - },{ }, function(res){ + },{ }, function(err, res){ assert.equal(res.statusCode, 200, res.body); - var cd = res.header('Content-Disposition'); + var cd = res.headers['content-disposition']; assert.ok(/filename=cartodb-query.svg/gi.test(cd), cd); - assert.equal(res.header('Content-Type'), 'image/svg+xml; charset=utf-8'); + assert.equal(res.headers['content-type'], 'image/svg+xml; charset=utf-8'); assert.ok( res.body.indexOf('') > 0, res.body ); // TODO: test viewBox queryobj.dp = 3; - assert.response(app, { + assert.response(server, { url: '/api/v1/sql?' + querystring.stringify(queryobj), headers: {host: 'vizzuality.cartodb.com'}, method: 'GET' - },{}, function(res) { + },{}, function(err, res) { assert.equal(res.statusCode, 200, res.body); - var cd = res.header('Content-Disposition'); + var cd = res.headers['content-disposition']; assert.equal(true, /^attachment/.test(cd), 'SVG is not disposed as attachment: ' + cd); assert.ok(/filename=cartodb-query.svg/gi.test(cd), cd); - assert.equal(res.header('Content-Type'), 'image/svg+xml; charset=utf-8'); + assert.equal(res.headers['content-type'], 'image/svg+xml; charset=utf-8'); assert.ok( res.body.indexOf('') > 0, res.body ); // TODO: test viewBox done(); @@ -137,11 +134,11 @@ it('SVG format with "the_geom" in skipfields', function(done){ format: "svg", skipfields: "the_geom" }); - assert.response(app, { + assert.response(server, { url: '/api/v1/sql?' + query, headers: {host: 'vizzuality.cartodb.com'}, method: 'GET' - },{ }, function(res){ + },{ }, function(err, res){ assert.equal(res.statusCode, 400, res.statusCode + ': ' + res.body); assert.deepEqual(res.headers['content-type'], 'application/json; charset=utf-8'); assert.deepEqual(res.headers['content-disposition'], 'inline'); @@ -157,11 +154,11 @@ it('SVG format with missing "the_geom" field', function(done){ q: "SELECT 1 as cartodb_id, ST_MakePoint(5000, -54) AS something_else ", format: "svg" }); - assert.response(app, { + assert.response(server, { url: '/api/v1/sql?' + query, headers: {host: 'vizzuality.cartodb.com'}, method: 'GET' - },{ }, function(res){ + },{ }, function(err, res){ assert.equal(res.statusCode, 400, res.statusCode + ': ' + res.body); assert.deepEqual(JSON.parse(res.body), { error:['column "the_geom" does not exist'] @@ -172,7 +169,7 @@ it('SVG format with missing "the_geom" field', function(done){ it('should close on error and error must be the only key in the body', function(done) { assert.response( - app, + server, { url: "/api/v1/sql?" + querystring.stringify({ q: "SELECT the_geom, 100/(cartodb_id - 3) cdb_ratio FROM untitle_table_4", @@ -186,7 +183,7 @@ it('SVG format with missing "the_geom" field', function(done){ { status: 400 }, - function(res) { + function(err, res) { var parsedBody = JSON.parse(res.body); assert.deepEqual(Object.keys(parsedBody), ['error']); assert.deepEqual(parsedBody.error, ["division by zero"]); diff --git a/test/acceptance/export/topojson.js b/test/acceptance/export/topojson.js index cd6f28cec..8316d86b1 100644 --- a/test/acceptance/export/topojson.js +++ b/test/acceptance/export/topojson.js @@ -1,14 +1,10 @@ require('../../helper'); -var app = require(global.settings.app_root + '/app/app')(); +var server = require('../../../app/server')(); var assert = require('../../support/assert'); var querystring = require('querystring'); var _ = require('underscore'); -// allow lots of emitters to be set to silence warning -app.setMaxListeners(0); - - describe('export.topojson', function() { // TOPOJSON tests @@ -29,7 +25,7 @@ describe('export.topojson', function() { } it('GET two polygons sharing an edge as topojson', function(done){ - assert.response(app, + assert.response(server, getRequest( "SELECT 1 as gid, 'U' as name, 'POLYGON((-5 0,5 0,0 5,-5 0))'::geometry as the_geom " + " UNION ALL " + @@ -38,8 +34,8 @@ it('GET two polygons sharing an edge as topojson', function(done){ { status: 200 }, - function(res) { - var cd = res.header('Content-Disposition'); + function(err, res) { + var cd = res.headers['content-disposition']; assert.equal(true, /^attachment/.test(cd), 'TOPOJSON is not disposed as attachment: ' + cd); assert.equal(true, /filename=cartodb-query.topojson/gi.test(cd)); var topojson = JSON.parse(res.body); @@ -135,7 +131,7 @@ it('GET two polygons sharing an edge as topojson', function(done){ }); it('null geometries', function(done){ - assert.response(app, getRequest( + assert.response(server, getRequest( "SELECT 1 as gid, 'U' as name, 'POLYGON((-5 0,5 0,0 5,-5 0))'::geometry as the_geom " + " UNION ALL " + "SELECT 2, 'D', null::geometry as the_geom " @@ -143,8 +139,8 @@ it('null geometries', function(done){ { status: 200 }, - function(res) { - var cd = res.header('Content-Disposition'); + function(err, res) { + var cd = res.headers['content-disposition']; assert.equal(true, /^attachment/.test(cd), 'TOPOJSON is not disposed as attachment: ' + cd); assert.equal(true, /filename=cartodb-query.topojson/gi.test(cd)); var topojson = JSON.parse(res.body); @@ -193,7 +189,7 @@ it('null geometries', function(done){ }); it('skipped fields are not returned', function(done) { - assert.response(app, + assert.response(server, getRequest( "SELECT 1 as gid, 'U' as name, 'POLYGON((-5 0,5 0,0 5,-5 0))'::geometry as the_geom", { @@ -203,7 +199,7 @@ it('null geometries', function(done){ { status: 200 }, - function(res) { + function(err, res) { var parsedBody = JSON.parse(res.body); assert.equal(parsedBody.objects[0].properties.gid, 1, 'gid was expected property'); assert.ok(!parsedBody.objects[0].properties.name); @@ -214,7 +210,7 @@ it('null geometries', function(done){ it('jsonp callback is invoked', function(done){ assert.response( - app, + server, getRequest( "SELECT 1 as gid, 'U' as name, 'POLYGON((-5 0,5 0,0 5,-5 0))'::geometry as the_geom", { @@ -224,7 +220,7 @@ it('null geometries', function(done){ { status: 200 }, - function(res) { + function(err, res) { assert.equal(res.statusCode, 200, res.statusCode + ': ' + res.body); var didRunJsonCallback = false; // jshint ignore:start @@ -242,7 +238,7 @@ it('null geometries', function(done){ it('should close on error and error must be the only key in the body', function(done) { assert.response( - app, + server, { url: "/api/v1/sql?" + querystring.stringify({ q: "SELECT the_geom, 100/(cartodb_id - 3) cdb_ratio FROM untitle_table_4", @@ -256,7 +252,7 @@ it('null geometries', function(done){ { status: 400 }, - function(res) { + function(err, res) { var parsedBody = JSON.parse(res.body); assert.deepEqual(Object.keys(parsedBody), ['error']); assert.deepEqual(parsedBody.error, ["division by zero"]); diff --git a/test/acceptance/frontend_abort.js b/test/acceptance/frontend_abort.js index 15a155979..16cb10808 100644 --- a/test/acceptance/frontend_abort.js +++ b/test/acceptance/frontend_abort.js @@ -30,19 +30,16 @@ it('aborts request', function(done){ var db_port_backup = global.settings.db_port; global.settings.db_host = 'localhost'; global.settings.db_port = sql_server_port; - var app = require(global.settings.app_root + '/app/app')(); + var server = require('../../app/server')(); var timeout; step( function sendQuery() { - var next = this; - assert.response(app, { + assert.response(server, { url: '/api/v1/sql?q=SELECT+1', method: 'GET', timeout: 1, headers: {host: 'vizzuality.localhost' } - },{}, function(res, err) { - next(err, res); - }); + },{}, this); }, function checkResponse(err/*, res*/) { assert(err); // expect timeout diff --git a/test/acceptance/health_check.js b/test/acceptance/health_check.js index e54631804..7a94e6cba 100644 --- a/test/acceptance/health_check.js +++ b/test/acceptance/health_check.js @@ -2,7 +2,7 @@ require('../helper'); require('../support/assert'); var assert = require('assert'); -var app = require(global.settings.app_root + '/app/app')(); +var server = require('../../app/server')(); describe('health checks', function() { @@ -26,12 +26,12 @@ describe('health checks', function() { it('returns 200 and ok=true with disabled configuration', function(done) { global.settings.health.enabled = false; - assert.response(app, + assert.response(server, healthCheckRequest, { status: 200 }, - function(res, err) { + function(err, res) { assert.ok(!err); var parsed = JSON.parse(res.body); @@ -45,12 +45,12 @@ describe('health checks', function() { }); it('returns 200 and ok=true with enabled configuration', function(done) { - assert.response(app, + assert.response(server, healthCheckRequest, { status: 200 }, - function(res, err) { + function(err, res) { assert.ok(!err); var parsed = JSON.parse(res.body); diff --git a/test/acceptance/job.test.js b/test/acceptance/job.test.js deleted file mode 100644 index d81a6cca6..000000000 --- a/test/acceptance/job.test.js +++ /dev/null @@ -1,483 +0,0 @@ -/** - * - * Requires the database and tables setup in config/environments/test.js to exist - * Ensure the user is present in the pgbouncer auth file too - * TODO: Add OAuth tests. - * - * To run this test, ensure that cartodb_test_user_1_db metadata exists - * in Redis for the vizzuality.cartodb.com domain - * - * SELECT 5 - * HSET rails:users:vizzuality id 1 - * HSET rails:users:vizzuality database_name cartodb_test_user_1_db - * - */ -require('../helper'); - -var app = require(global.settings.app_root + '/app/app')(); -var assert = require('../support/assert'); -var querystring = require('querystring'); -var metadataBackend = require('cartodb-redis')({ - host: global.settings.redis_host, - port: global.settings.redis_port, - max: global.settings.redisPool, - idleTimeoutMillis: global.settings.redisIdleTimeoutMillis, - reapIntervalMillis: global.settings.redisReapIntervalMillis -}); - -describe('job module', function() { - var job = {}; - - after(function (done) { - // batch services is not activate, so we need empty the queue to avoid unexpected - // behaviour in further tests - metadataBackend.redisCmd(5, 'DEL', [ 'batch:queues:localhost' ], done); - }); - - it('POST /api/v2/sql/job should respond with 200 and the created job', function (done){ - assert.response(app, { - url: '/api/v2/sql/job?api_key=1234', - headers: { 'host': 'vizzuality.cartodb.com', 'Content-Type': 'application/x-www-form-urlencoded' }, - method: 'POST', - data: querystring.stringify({ - query: "SELECT * FROM untitle_table_4" - }) - }, { - status: 201 - }, function(res) { - job = JSON.parse(res.body); - assert.deepEqual(res.headers['content-type'], 'application/json; charset=utf-8'); - assert.ok(job.job_id); - assert.equal(job.query, "SELECT * FROM untitle_table_4"); - assert.equal(job.user, "vizzuality"); - done(); - }); - }); - - it('POST /api/v2/sql/job without query should respond with 400 and the corresponding message of error', - function (done){ - assert.response(app, { - url: '/api/v2/sql/job?api_key=1234', - headers: { 'host': 'vizzuality.cartodb.com', 'Content-Type': 'application/x-www-form-urlencoded' }, - method: 'POST', - data: querystring.stringify({}) - }, { - status: 400 - }, function(res) { - var error = JSON.parse(res.body); - assert.deepEqual(error, { error: [ 'You must indicate a valid SQL' ] }); - done(); - }); - }); - - it('POST /api/v2/sql/job with bad query param should respond with 400 and message of error', function (done){ - assert.response(app, { - url: '/api/v2/sql/job?api_key=1234', - headers: { 'host': 'vizzuality.cartodb.com', 'Content-Type': 'application/x-www-form-urlencoded' }, - method: 'POST', - data: querystring.stringify({ - q: "SELECT * FROM untitle_table_4" - }) - }, { - status: 400 - }, function(res) { - var error = JSON.parse(res.body); - assert.deepEqual(error, { error: [ 'You must indicate a valid SQL' ] }); - done(); - }); - }); - - it('POST /api/v2/sql/job with wrong api key should respond with 401 permission denied', function (done){ - assert.response(app, { - url: '/api/v2/sql/job?api_key=wrong', - headers: { 'host': 'vizzuality.cartodb.com', 'Content-Type': 'application/x-www-form-urlencoded' }, - method: 'POST', - data: querystring.stringify({ - query: "SELECT * FROM untitle_table_4" - }) - }, { - status: 401 - }, function(res) { - var error = JSON.parse(res.body); - assert.deepEqual(error, { error: [ 'permission denied' ] }); - done(); - }); - }); - - it('POST /api/v2/sql/job with wrong host header should respond with 404 not found', function (done){ - assert.response(app, { - url: '/api/v2/sql/job?api_key=wrong', - headers: { 'host': 'wrong-host.cartodb.com', 'Content-Type': 'application/x-www-form-urlencoded' }, - method: 'POST', - data: querystring.stringify({ - query: "SELECT * FROM untitle_table_4" - }) - }, { - status: 404 - }, function(res) { - var error = JSON.parse(res.body); - assert.deepEqual(error, { - error: [ - 'Sorry, we can\'t find CartoDB user \'wrong-host\'. ' + - 'Please check that you have entered the correct domain.' - ] - }); - done(); - }); - }); - - it('GET /api/v2/sql/job/:job_id should respond with 200 and the requested job', function (done){ - assert.response(app, { - url: '/api/v2/sql/job/' + job.job_id + '?api_key=1234', - headers: { 'host': 'vizzuality.cartodb.com', 'Content-Type': 'application/x-www-form-urlencoded' }, - method: 'GET' - }, { - status: 200 - }, function(res) { - var jobGot = JSON.parse(res.body); - assert.deepEqual(res.headers['content-type'], 'application/json; charset=utf-8'); - assert.equal(jobGot.query, "SELECT * FROM untitle_table_4"); - assert.equal(jobGot.user, "vizzuality"); - done(); - }); - }); - - it('GET /api/v2/sql/job/:job_id with wrong api key should respond with 401 permission denied', function (done){ - assert.response(app, { - url: '/api/v2/sql/job/' + job.job_id + '?api_key=wrong', - headers: { 'host': 'vizzuality.cartodb.com', 'Content-Type': 'application/x-www-form-urlencoded' }, - method: 'GET' - }, { - status: 401 - }, function(res) { - var error = JSON.parse(res.body); - assert.deepEqual(error, { error: [ 'permission denied' ] }); - done(); - }); - }); - - it('GET /api/v2/sql/job/ with wrong host header respond with 404 not found', function (done){ - assert.response(app, { - url: '/api/v2/sql/job?api_key=1234', - headers: { 'host': 'wrong-host.cartodb.com', 'Content-Type': 'application/x-www-form-urlencoded' }, - method: 'GET' - }, { - status: 404 - }, function(res) { - var error = JSON.parse(res.body); - assert.deepEqual(error , { - error: [ - 'Sorry, we can\'t find CartoDB user \'wrong-host\'. ' + - 'Please check that you have entered the correct domain.' - ] - }); - done(); - }); - }); - - it('GET /api/v2/sql/job/:jobId with wrong jobId header respond with 400 and an error', function (done){ - assert.response(app, { - url: '/api/v2/sql/job/irrelevantJob?api_key=1234', - headers: { 'host': 'vizzuality.cartodb.com', 'Content-Type': 'application/x-www-form-urlencoded' }, - method: 'GET' - }, { - status: 400 - }, function(res) { - var error = JSON.parse(res.body); - console.log(error); - assert.deepEqual(error , { - error: ['Job with id irrelevantJob not found'] - }); - done(); - }); - }); - - it('PUT /api/v2/sql/job/:job_id should respond 200 and the updated job', function (done) { - var query ="SELECT cartodb_id FROM untitle_table_4"; - assert.response(app, { - url: '/api/v2/sql/job/' + job.job_id + '?api_key=1234', - headers: { 'host': 'vizzuality.cartodb.com', 'Content-Type': 'application/x-www-form-urlencoded' }, - method: 'PUT', - data: querystring.stringify({ - query: query - }) - }, { - status: 200 - }, function(res) { - var updatedJob = JSON.parse(res.body); - assert.deepEqual(res.headers['content-type'], 'application/json; charset=utf-8'); - assert.equal(updatedJob.job_id, job.job_id); - assert.equal(updatedJob.query, query); - assert.equal(updatedJob.user, "vizzuality"); - done(); - }); - }); - - it('PUT /api/v2/sql/job/:job_id without query should respond with 400 and message of error', function (done){ - assert.response(app, { - url: '/api/v2/sql/job/' + job.job_id + '?api_key=1234', - headers: { 'host': 'vizzuality.cartodb.com', 'Content-Type': 'application/x-www-form-urlencoded' }, - method: 'PUT', - data: querystring.stringify({}) - }, { - status: 400 - }, function(res) { - var error = JSON.parse(res.body); - assert.deepEqual(error, { error: [ 'You must indicate a valid SQL' ] }); - done(); - }); - }); - - it('PUT /api/v2/sql/job with bad query param should respond with 400 and message of error', function (done){ - assert.response(app, { - url: '/api/v2/sql/job/' + job.job_id + '?api_key=1234', - headers: { 'host': 'vizzuality.cartodb.com', 'Content-Type': 'application/x-www-form-urlencoded' }, - method: 'PUT', - data: querystring.stringify({ - q: "SELECT * FROM untitle_table_4" - }) - }, { - status: 400 - }, function(res) { - var error = JSON.parse(res.body); - assert.deepEqual(error, { error: [ 'You must indicate a valid SQL' ] }); - done(); - }); - }); - - it('PUT /api/v2/sql/job/:job_id with wrong api key should respond with 401 permission denied', function (done) { - var query ="SELECT cartodb_id FROM untitle_table_4"; - assert.response(app, { - url: '/api/v2/sql/job/' + job.job_id + '?api_key=wrong', - headers: { 'host': 'vizzuality.cartodb.com', 'Content-Type': 'application/x-www-form-urlencoded' }, - method: 'PUT', - data: querystring.stringify({ - query: query - }) - }, { - status: 401 - }, function(res) { - var error = JSON.parse(res.body); - assert.deepEqual(error, { error: [ 'permission denied' ] }); - done(); - }); - }); - - it('PUT /api/v2/sql/job with wrong host header should respond with 404 not found', function (done){ - assert.response(app, { - url: '/api/v2/sql/job/' + job.job_id + '?api_key=wrong', - headers: { 'host': 'wrong-host.cartodb.com', 'Content-Type': 'application/x-www-form-urlencoded' }, - method: 'PUT', - data: querystring.stringify({ - query: "SELECT * FROM untitle_table_4" - }) - }, { - status: 404 - }, function(res) { - var error = JSON.parse(res.body); - assert.deepEqual(error , { - error: [ - 'Sorry, we can\'t find CartoDB user \'wrong-host\'. ' + - 'Please check that you have entered the correct domain.' - ] - }); - done(); - }); - }); - - it('PATCH /api/v2/sql/job/:job_id should respond 200 and the updated job', function (done) { - var query ="SELECT * FROM untitle_table_4"; - assert.response(app, { - url: '/api/v2/sql/job/' + job.job_id + '?api_key=1234', - headers: { 'host': 'vizzuality.cartodb.com', 'Content-Type': 'application/x-www-form-urlencoded' }, - method: 'PATCH', - data: querystring.stringify({ - query: query - }) - }, { - status: 200 - }, function(res) { - var updatedJob = JSON.parse(res.body); - assert.deepEqual(res.headers['content-type'], 'application/json; charset=utf-8'); - assert.equal(updatedJob.job_id, job.job_id); - assert.equal(updatedJob.query, query); - assert.equal(updatedJob.user, "vizzuality"); - done(); - }); - }); - - it('PATCH /api/v2/sql/job/:job_id without query should respond with 400 and message of error', function (done){ - assert.response(app, { - url: '/api/v2/sql/job/' + job.job_id + '?api_key=1234', - headers: { 'host': 'vizzuality.cartodb.com', 'Content-Type': 'application/x-www-form-urlencoded' }, - method: 'PATCH', - data: querystring.stringify({}) - }, { - status: 400 - }, function(res) { - var error = JSON.parse(res.body); - assert.deepEqual(error, { error: [ 'You must indicate a valid SQL' ] }); - done(); - }); - }); - - it('PATCH /api/v2/sql/job with bad query param should respond with 400 and message of error', function (done){ - assert.response(app, { - url: '/api/v2/sql/job/' + job.job_id + '?api_key=1234', - headers: { 'host': 'vizzuality.cartodb.com', 'Content-Type': 'application/x-www-form-urlencoded' }, - method: 'PATCH', - data: querystring.stringify({ - q: "SELECT * FROM untitle_table_4" - }) - }, { - status: 400 - }, function(res) { - var error = JSON.parse(res.body); - assert.deepEqual(error, { error: [ 'You must indicate a valid SQL' ] }); - done(); - }); - }); - - it('PATCH /api/v2/sql/job/:job_id with wrong api key should respond with 401 permission denied', function (done) { - var query ="SELECT * FROM untitle_table_4"; - assert.response(app, { - url: '/api/v2/sql/job/' + job.job_id + '?api_key=wrong', - headers: { 'host': 'vizzuality.cartodb.com', 'Content-Type': 'application/x-www-form-urlencoded' }, - method: 'PATCH', - data: querystring.stringify({ - query: query - }) - }, { - status: 401 - }, function(res) { - var error = JSON.parse(res.body); - assert.deepEqual(error, { error: [ 'permission denied' ] }); - done(); - }); - }); - - it('PATCH /api/v2/sql/job with wrong host header should respond with 404 not found', function (done){ - assert.response(app, { - url: '/api/v2/sql/job/' + job.job_id + '?api_key=wrong', - headers: { 'host': 'wrong-host.cartodb.com', 'Content-Type': 'application/x-www-form-urlencoded' }, - method: 'PATCH', - data: querystring.stringify({ - query: "SELECT * FROM untitle_table_4" - }) - }, { - status: 404 - }, function(res) { - var error = JSON.parse(res.body); - assert.deepEqual(error , { - error: [ - 'Sorry, we can\'t find CartoDB user \'wrong-host\'. ' + - 'Please check that you have entered the correct domain.' - ] - }); - done(); - }); - }); - - it('GET /api/v2/sql/job/ should respond with 200 and a job\'s list', function (done){ - assert.response(app, { - url: '/api/v2/sql/job?api_key=1234', - headers: { 'host': 'vizzuality.cartodb.com', 'Content-Type': 'application/x-www-form-urlencoded' }, - method: 'GET' - }, { - status: 200 - }, function(res) { - var jobs = JSON.parse(res.body); - assert.deepEqual(res.headers['content-type'], 'application/json; charset=utf-8'); - assert.ok(jobs instanceof Array); - assert.ok(jobs.length > 0); - assert.ok(jobs[0].job_id); - assert.ok(jobs[0].status); - assert.ok(jobs[0].query); - done(); - }); - }); - - it('GET /api/v2/sql/job/ with wrong api key should respond with 401 permission denied', function (done){ - assert.response(app, { - url: '/api/v2/sql/job?api_key=wrong', - headers: { 'host': 'vizzuality.cartodb.com', 'Content-Type': 'application/x-www-form-urlencoded' }, - method: 'GET' - }, { - status: 401 - }, function(res) { - var error = JSON.parse(res.body); - assert.deepEqual(error, { error: [ 'permission denied' ] }); - done(); - }); - }); - - it('GET /api/v2/sql/job/ without host header respond with 404 not found', function (done){ - assert.response(app, { - url: '/api/v2/sql/job?api_key=1234', - headers: { 'host': 'wrong-host.cartodb.com', 'Content-Type': 'application/x-www-form-urlencoded' }, - method: 'GET' - }, { - status: 404 - }, function(res) { - var error = JSON.parse(res.body); - assert.deepEqual(error , { - error: [ - 'Sorry, we can\'t find CartoDB user \'wrong-host\'. ' + - 'Please check that you have entered the correct domain.' - ] - }); - done(); - }); - }); - - it('DELETE /api/v2/sql/job/:job_id should respond with 200 and the requested job', function (done){ - assert.response(app, { - url: '/api/v2/sql/job/' + job.job_id + '?api_key=1234', - headers: { 'host': 'vizzuality.cartodb.com', 'Content-Type': 'application/x-www-form-urlencoded' }, - method: 'DELETE' - }, { - status: 200 - }, function(res) { - var jobCancelled = JSON.parse(res.body); - assert.deepEqual(res.headers['content-type'], 'application/json; charset=utf-8'); - assert.equal(jobCancelled.job_id, job.job_id); - assert.equal(jobCancelled.query, "SELECT * FROM untitle_table_4"); - assert.equal(jobCancelled.user, "vizzuality"); - assert.equal(jobCancelled.status, "cancelled"); - done(); - }); - }); - - it('DELETE /api/v2/sql/job/:job_id with wrong api key should respond with 401 permission denied', function (done){ - assert.response(app, { - url: '/api/v2/sql/job/' + job.job_id + '?api_key=wrong', - headers: { 'host': 'vizzuality.cartodb.com', 'Content-Type': 'application/x-www-form-urlencoded' }, - method: 'DELETE' - }, { - status: 401 - }, function(res) { - var error = JSON.parse(res.body); - assert.deepEqual(error, { error: [ 'permission denied' ] }); - done(); - }); - }); - - it('DELETE /api/v2/sql/job/ with wrong host header respond with 404 not found', function (done){ - assert.response(app, { - url: '/api/v2/sql/job/' + job.job_id + '?api_key=1234', - headers: { 'host': 'wrong-host.cartodb.com', 'Content-Type': 'application/x-www-form-urlencoded' }, - method: 'DELETE' - }, { - status: 404 - }, function(res) { - var error = JSON.parse(res.body); - assert.deepEqual(error , { - error: [ - 'Sorry, we can\'t find CartoDB user \'wrong-host\'. ' + - 'Please check that you have entered the correct domain.' - ] - }); - done(); - }); - }); -}); diff --git a/test/acceptance/job.use-case-6.test.js b/test/acceptance/job.use-case-6.test.js deleted file mode 100644 index d8b9ebfc7..000000000 --- a/test/acceptance/job.use-case-6.test.js +++ /dev/null @@ -1,116 +0,0 @@ -/** - * - * Requires the database and tables setup in config/environments/test.js to exist - * Ensure the user is present in the pgbouncer auth file too - * TODO: Add OAuth tests. - * - * To run this test, ensure that cartodb_test_user_1_db metadata exists - * in Redis for the vizzuality.cartodb.com domain - * - * SELECT 5 - * HSET rails:users:vizzuality id 1 - * HSET rails:users:vizzuality database_name cartodb_test_user_1_db - * - */ -require('../helper'); - -var app = require(global.settings.app_root + '/app/app')(); -var assert = require('../support/assert'); -var querystring = require('querystring'); -var metadataBackend = require('cartodb-redis')({ - host: global.settings.redis_host, - port: global.settings.redis_port, - max: global.settings.redisPool, - idleTimeoutMillis: global.settings.redisIdleTimeoutMillis, - reapIntervalMillis: global.settings.redisReapIntervalMillis -}); -var batchFactory = require('../../batch'); - -describe('Use case 6: modify a done job', function() { - - var batch = batchFactory(metadataBackend); - - before(function () { - batch.start(); - }); - - after(function (done) { - batch.stop(); - batch.drain(function () { - metadataBackend.redisCmd(5, 'DEL', [ 'batch:queues:localhost' ], done); - }); - }); - - var doneJob = {}; - - it('Step 1, should create job', function (done) { - assert.response(app, { - url: '/api/v2/sql/job?api_key=1234', - headers: { 'host': 'vizzuality.cartodb.com', 'Content-Type': 'application/x-www-form-urlencoded' }, - method: 'POST', - data: querystring.stringify({ - query: "SELECT * FROM untitle_table_4" - }) - }, { - status: 201 - }, function (res) { - doneJob = JSON.parse(res.body); - done(); - }); - }); - - it('Step 2, job should be done', function (done) { - var interval = setInterval(function () { - assert.response(app, { - url: '/api/v2/sql/job/' + doneJob.job_id + '?api_key=1234', - headers: { 'host': 'vizzuality.cartodb.com', 'Content-Type': 'application/x-www-form-urlencoded' }, - method: 'GET' - }, { - status: 200 - }, function(res) { - var job = JSON.parse(res.body); - if (job.status === "done") { - clearInterval(interval); - done(); - } else if (job.status === "failed" || job.status === "cancelled") { - clearInterval(interval); - done(new Error('Job ' + job.job_id + ' is ' + job.status + ', expected to be done')); - } - }); - }, 50); - }); - - it('Step 3, modify a done job should give an error', function (done){ - assert.response(app, { - url: '/api/v2/sql/job/' + doneJob.job_id + '?api_key=1234', - headers: { 'host': 'vizzuality.cartodb.com', 'Content-Type': 'application/x-www-form-urlencoded' }, - method: 'PUT', - data: querystring.stringify({ - query: "SELECT cartodb_id FROM untitle_table_4" - }) - }, { - status: 400 - }, function(res) { - var errors = JSON.parse(res.body); - assert.equal(errors.error[0], "Job is not pending, it cannot be updated"); - done(); - }); - }); - - it('Step 5, modify a cancelled job should give an error', function (done){ - assert.response(app, { - url: '/api/v2/sql/job/' + doneJob.job_id + '?api_key=1234', - headers: { 'host': 'vizzuality.cartodb.com', 'Content-Type': 'application/x-www-form-urlencoded' }, - method: 'PUT', - data: querystring.stringify({ - query: "SELECT cartodb_id FROM untitle_table_4" - }) - }, { - status: 400 - }, function(res) { - var errors = JSON.parse(res.body); - assert.equal(errors.error[0], "Job is not pending, it cannot be updated"); - done(); - }); - }); -}); diff --git a/test/acceptance/last-modified-header.js b/test/acceptance/last-modified-header.js index e6ffd14fc..09e0c6068 100644 --- a/test/acceptance/last-modified-header.js +++ b/test/acceptance/last-modified-header.js @@ -1,6 +1,6 @@ require('../helper'); -var app = require(global.settings.app_root + '/app/app')(); +var server = require('../../app/server')(); var assert = require('../support/assert'); var qs = require('querystring'); @@ -37,7 +37,7 @@ describe('last modified header', function() { }).join(' UNION ALL '), api_key: 1234 }); - assert.response(app, + assert.response(server, { url: '/api/v1/sql?' + query, headers: { @@ -48,7 +48,7 @@ describe('last modified header', function() { { statusCode: 200 }, - function(res) { + function(err, res) { assert.equal(res.headers['last-modified'], scenario.expectedLastModified); done(); } @@ -66,7 +66,7 @@ describe('last modified header', function() { Date.now = function() { return fixedDateNow; }; - assert.response(app, + assert.response(server, { url: '/api/v1/sql?' + query, headers: { @@ -77,7 +77,7 @@ describe('last modified header', function() { { statusCode: 200 }, - function(res) { + function(err, res) { Date.now = dateNowFn; assert.equal(res.headers['last-modified'], new Date(fixedDateNow).toUTCString()); done(); @@ -95,7 +95,7 @@ describe('last modified header', function() { Date.now = function() { return fixedDateNow; }; - assert.response(app, + assert.response(server, { url: '/api/v1/sql?' + query, headers: { @@ -106,7 +106,7 @@ describe('last modified header', function() { { statusCode: 200 }, - function(res) { + function(err, res) { Date.now = dateNowFn; assert.equal(res.headers['last-modified'], new Date(fixedDateNow).toUTCString()); done(); diff --git a/test/acceptance/logging.js b/test/acceptance/logging.js index 5574a0179..65a308d62 100644 --- a/test/acceptance/logging.js +++ b/test/acceptance/logging.js @@ -1,6 +1,6 @@ require('../helper'); -var app = require(global.settings.app_root + '/app/app'); +var appServer = require('../../app/server'); var assert = require('../support/assert'); var qs = require('querystring'); var log4js = require('log4js'); @@ -32,7 +32,7 @@ describe('Logging SQL query on POST requests', function() { } ] }); - server = app(); + server = appServer(); }); after(function() { @@ -105,7 +105,7 @@ describe('Logging SQL query on POST requests', function() { return result; }; - assert.response(server, scenario.request, RESPONSE_OK, function(res, err) { + assert.response(server, scenario.request, RESPONSE_OK, function(err) { assert.ok(!err); assert.equal(called, 1); @@ -137,7 +137,7 @@ describe('Logging SQL query on POST requests', function() { } }, RESPONSE_OK, - function(res, err) { + function(err) { assert.ok(!err); assert.equal(called, 1); diff --git a/test/acceptance/query-tables-api-cache.js b/test/acceptance/query-tables-api-cache.js index 3409ef8a7..ee7126ac4 100644 --- a/test/acceptance/query-tables-api-cache.js +++ b/test/acceptance/query-tables-api-cache.js @@ -2,14 +2,14 @@ require('../helper'); var qs = require('querystring'); -var app = require(global.settings.app_root + '/app/app')(); +var server = require('../../app/server')(); var assert = require('../support/assert'); describe('query-tables-api', function() { function getCacheStatus(callback) { assert.response( - app, + server, { method: 'GET', url: '/api/v1/cachestatus' @@ -17,7 +17,7 @@ describe('query-tables-api', function() { { status: 200 }, - function(res) { + function(err, res) { callback(null, JSON.parse(res.body)); } ); @@ -38,7 +38,7 @@ describe('query-tables-api', function() { }; it('should create a key in affected tables cache', function(done) { - assert.response(app, request, RESPONSE_OK, function(res, err) { + assert.response(server, request, RESPONSE_OK, function(err) { assert.ok(!err, err); getCacheStatus(function(err, cacheStatus) { @@ -52,7 +52,7 @@ describe('query-tables-api', function() { }); it('should use cache to retrieve affected tables', function(done) { - assert.response(app, request, RESPONSE_OK, function(res, err) { + assert.response(server, request, RESPONSE_OK, function(err) { assert.ok(!err, err); getCacheStatus(function(err, cacheStatus) { @@ -76,7 +76,7 @@ describe('query-tables-api', function() { }, method: 'GET' }; - assert.response(app, authenticatedRequest, RESPONSE_OK, function(res, err) { + assert.response(server, authenticatedRequest, RESPONSE_OK, function(err) { assert.ok(!err, err); getCacheStatus(function(err, cacheStatus) { diff --git a/test/acceptance/regressions.js b/test/acceptance/regressions.js index dd79deaeb..f81f08e07 100644 --- a/test/acceptance/regressions.js +++ b/test/acceptance/regressions.js @@ -1,6 +1,6 @@ require('../helper'); -var app = require(global.settings.app_root + '/app/app')(); +var server = require('../../app/server')(); var assert = require('../support/assert'); var qs = require('querystring'); @@ -25,31 +25,34 @@ describe('regressions', function() { statusCode: 200 }; - assert.response(app, createRequest('CREATE TABLE "foo.bar" (a int);'), responseOk, - function(res, err) { + assert.response(server, createRequest('CREATE TABLE "foo.bar" (a int);'), responseOk, + function(err) { if (err) { return done(err); } - assert.response(app, createRequest('INSERT INTO "foo.bar" (a) values (1), (2)'), responseOk, - function(res, err) { + assert.response(server, createRequest('INSERT INTO "foo.bar" (a) values (1), (2)'), responseOk, + function(err, res) { if (err) { return done(err); } var parsedBody = JSON.parse(res.body); assert.equal(parsedBody.total_rows, 2); - assert.response(app, createRequest('SELECT * FROM "foo.bar"'), responseOk, - function(res, err) { + assert.response(server, createRequest('SELECT * FROM "foo.bar"'), responseOk, + function(err, res) { if (err) { return done(err); } - assert.equal(res.headers['x-cache-channel'], 'cartodb_test_user_1_db:public."foo.bar"'); + // table should not get a cache channel as it won't get invalidated + assert.ok(!res.headers.hasOwnProperty('x-cache-channel')); var parsedBody = JSON.parse(res.body); assert.equal(parsedBody.total_rows, 2); assert.deepEqual(parsedBody.rows, [{ a: 1 }, { a: 2 }]); - done(); + + // delete table + assert.response(server, createRequest('DROP TABLE "foo.bar"'), responseOk, done); } ); } diff --git a/test/acceptance/stream-responses.js b/test/acceptance/stream-responses.js index c309e487e..237cc9de4 100644 --- a/test/acceptance/stream-responses.js +++ b/test/acceptance/stream-responses.js @@ -1,6 +1,6 @@ require('../helper'); -var app = require(global.settings.app_root + '/app/app')(); +var server = require('../../app/server')(); var assert = require('../support/assert'); var querystring = require('querystring'); @@ -32,10 +32,10 @@ describe('stream-responses', function() { it('should close on error and error message must be part of the response', function(done) { assert.response( - app, + server, createFailingQueryRequest(), okResponse, - function(res) { + function(err, res) { var parsedBody = JSON.parse(res.body); assert.equal(parsedBody.rows.length, 2); assert.deepEqual(parsedBody.fields, { @@ -54,10 +54,10 @@ describe('stream-responses', function() { it('should close on error and error message must be part of the response', function(done) { assert.response( - app, + server, createFailingQueryRequest('geojson'), okResponse, - function(res) { + function(err, res) { var parsedBody = JSON.parse(res.body); assert.equal(parsedBody.features.length, 2); assert.deepEqual(parsedBody.error, ["division by zero"]); diff --git a/test/acceptance/surrogate-key.js b/test/acceptance/surrogate-key.js index eba5aba73..f460f5568 100644 --- a/test/acceptance/surrogate-key.js +++ b/test/acceptance/surrogate-key.js @@ -1,14 +1,11 @@ require('../helper'); -var app = require(global.settings.app_root + '/app/app')(); +var server = require('../../app/server')(); var assert = require('../support/assert'); var querystring = require('querystring'); var QueryTables = require('cartodb-query-tables'); var _ = require('underscore'); -// allow lots of emitters to be set to silence warning -app.setMaxListeners(0); - describe('Surrogate-Key header', function() { function createGetRequest(sqlQuery) { @@ -43,7 +40,7 @@ describe('Surrogate-Key header', function() { function tableNamesInSurrogateKeyHeader(expectedTableNames, done) { - return function(res) { + return function(err, res) { surrogateKeyHasTables(res.headers['surrogate-key'], expectedTableNames); done(); }; @@ -53,7 +50,7 @@ describe('Surrogate-Key header', function() { var sql = "SELECT a.name as an, b.name as bn FROM untitle_table_4 a " + "left join private_table b ON (a.cartodb_id = b.cartodb_id)"; - assert.response(app, createGetRequest(sql), RESPONSE_OK, tableNamesInSurrogateKeyHeader([ + assert.response(server, createGetRequest(sql), RESPONSE_OK, tableNamesInSurrogateKeyHeader([ {dbname: 'cartodb_test_user_1_db', schema_name: 'public', table_name: 'private_table'}, {dbname: 'cartodb_test_user_1_db', schema_name: 'public', table_name: 'untitle_table_4'} ], done)); @@ -62,7 +59,7 @@ describe('Surrogate-Key header', function() { it('supports multistatements', function(done) { var sql = "SELECT * FROM untitle_table_4; SELECT * FROM private_table"; - assert.response(app, createGetRequest(sql), RESPONSE_OK, tableNamesInSurrogateKeyHeader([ + assert.response(server, createGetRequest(sql), RESPONSE_OK, tableNamesInSurrogateKeyHeader([ {dbname: 'cartodb_test_user_1_db', schema_name: 'public', table_name: 'private_table'}, {dbname: 'cartodb_test_user_1_db', schema_name: 'public', table_name: 'untitle_table_4'} ], done)); @@ -71,7 +68,7 @@ describe('Surrogate-Key header', function() { it('supports explicit transactions', function(done) { var sql = "BEGIN; SELECT * FROM untitle_table_4; COMMIT; BEGIN; SELECT * FROM private_table; COMMIT;"; - assert.response(app, createGetRequest(sql), RESPONSE_OK, tableNamesInSurrogateKeyHeader([ + assert.response(server, createGetRequest(sql), RESPONSE_OK, tableNamesInSurrogateKeyHeader([ {dbname: 'cartodb_test_user_1_db', schema_name: 'public', table_name: 'private_table'}, {dbname: 'cartodb_test_user_1_db', schema_name: 'public', table_name: 'untitle_table_4'} ], done)); @@ -80,14 +77,14 @@ describe('Surrogate-Key header', function() { it('survives partial transactions', function(done) { var sql = "BEGIN; SELECT * FROM untitle_table_4"; - assert.response(app, createGetRequest(sql), RESPONSE_OK, tableNamesInSurrogateKeyHeader([ + assert.response(server, createGetRequest(sql), RESPONSE_OK, tableNamesInSurrogateKeyHeader([ {dbname: 'cartodb_test_user_1_db', schema_name: 'public', table_name: 'untitle_table_4'} ], done)); }); it('should not add header for functions', function(done) { var sql = "SELECT format('%s', 'wadus')"; - assert.response(app, createGetRequest(sql), RESPONSE_OK, function(res) { + assert.response(server, createGetRequest(sql), RESPONSE_OK, function(err, res) { assert.ok(!res.headers.hasOwnProperty('surrogate-key'), res.headers['surrogate-key']); done(); }); @@ -95,7 +92,7 @@ describe('Surrogate-Key header', function() { it('should not add header for CDB_QueryTables', function(done) { var sql = "SELECT CDB_QueryTablesText('select * from untitle_table_4')"; - assert.response(app, createGetRequest(sql), RESPONSE_OK, function(res) { + assert.response(server, createGetRequest(sql), RESPONSE_OK, function(err, res) { assert.ok(!res.headers.hasOwnProperty('surrogate-key'), res.headers['surrogate-key']); done(); }); @@ -103,7 +100,7 @@ describe('Surrogate-Key header', function() { it('should not add header for non table results', function(done) { var sql = "SELECT 'wadus'::text"; - assert.response(app, createGetRequest(sql), RESPONSE_OK, function(res) { + assert.response(server, createGetRequest(sql), RESPONSE_OK, function(err, res) { assert.ok(!res.headers.hasOwnProperty('surrogate-key'), res.headers['surrogate-key']); done(); }); diff --git a/test/acceptance/timeout.js b/test/acceptance/timeout.js index 7e64e97c4..24464cf1b 100644 --- a/test/acceptance/timeout.js +++ b/test/acceptance/timeout.js @@ -26,17 +26,14 @@ it('after configured milliseconds', function(done){ //console.log("settings:"); console.dir(global.settings); var timeoutBackup = global.settings.node_socket_timeout; global.settings.node_socket_timeout = testTimeout; - var app = require(global.settings.app_root + '/app/app')(); + var server = require('../../app/server')(); step( function sendLongQuery() { - var next = this; - assert.response(app, { + assert.response(server, { url: '/api/v1/sql?q=SELECT+count(*)+FROM+generate_series(1,100000)', method: 'GET', headers: {host: 'vizzuality.localhost' } - },{}, function(res, err) { - next(err, res); - }); + },{}, this); }, function checkResponse(err/*, res*/) { assert.ok(err); diff --git a/test/acceptance/transaction.js b/test/acceptance/transaction.js index 33a59c1ea..8a831853f 100644 --- a/test/acceptance/transaction.js +++ b/test/acceptance/transaction.js @@ -10,12 +10,13 @@ describe('transaction', function() { var server; before(function(done) { - server = require(global.settings.app_root + '/app/app')(); - server.listen(SERVER_PORT, '127.0.0.1', done); + server = require('../../app/server')(); + this.listener = server.listen(SERVER_PORT, '127.0.0.1'); + this.listener.on('listening', done); }); after(function(done) { - server.close(done); + this.listener.close(done); }); var sqlRequest = request.defaults({ diff --git a/test/acceptance/x-cache-channel.js b/test/acceptance/x-cache-channel.js index b63390320..d6effc9ed 100644 --- a/test/acceptance/x-cache-channel.js +++ b/test/acceptance/x-cache-channel.js @@ -1,13 +1,10 @@ require('../helper'); -var app = require(global.settings.app_root + '/app/app')(); +var server = require('../../app/server')(); var assert = require('../support/assert'); var querystring = require('querystring'); var _ = require('underscore'); -// allow lots of emitters to be set to silence warning -app.setMaxListeners(0); - describe('X-Cache-Channel header', function() { function createGetRequest(sqlQuery) { @@ -42,7 +39,7 @@ describe('X-Cache-Channel header', function() { } function tableNamesInCacheChannelHeader(expectedTableNames, done) { - return function(res) { + return function(err, res) { xCacheChannelHeaderHasTables(res.headers['x-cache-channel'], expectedTableNames); done(); }; @@ -52,7 +49,7 @@ describe('X-Cache-Channel header', function() { var sql = "SELECT a.name as an, b.name as bn FROM untitle_table_4 a " + "left join private_table b ON (a.cartodb_id = b.cartodb_id)"; - assert.response(app, createGetRequest(sql), RESPONSE_OK, tableNamesInCacheChannelHeader([ + assert.response(server, createGetRequest(sql), RESPONSE_OK, tableNamesInCacheChannelHeader([ 'public.private_table', 'public.untitle_table_4' ], done)); @@ -61,7 +58,7 @@ describe('X-Cache-Channel header', function() { it('supports multistatements', function(done) { var sql = "SELECT * FROM untitle_table_4; SELECT * FROM private_table"; - assert.response(app, createGetRequest(sql), RESPONSE_OK, tableNamesInCacheChannelHeader([ + assert.response(server, createGetRequest(sql), RESPONSE_OK, tableNamesInCacheChannelHeader([ 'public.private_table', 'public.untitle_table_4' ], done)); @@ -70,7 +67,7 @@ describe('X-Cache-Channel header', function() { it('supports explicit transactions', function(done) { var sql = "BEGIN; SELECT * FROM untitle_table_4; COMMIT; BEGIN; SELECT * FROM private_table; COMMIT;"; - assert.response(app, createGetRequest(sql), RESPONSE_OK, tableNamesInCacheChannelHeader([ + assert.response(server, createGetRequest(sql), RESPONSE_OK, tableNamesInCacheChannelHeader([ 'public.private_table', 'public.untitle_table_4' ], done)); @@ -79,14 +76,14 @@ describe('X-Cache-Channel header', function() { it('survives partial transactions', function(done) { var sql = "BEGIN; SELECT * FROM untitle_table_4"; - assert.response(app, createGetRequest(sql), RESPONSE_OK, tableNamesInCacheChannelHeader([ + assert.response(server, createGetRequest(sql), RESPONSE_OK, tableNamesInCacheChannelHeader([ 'public.untitle_table_4' ], done)); }); it('should not add header for functions', function(done) { var sql = "SELECT format('%s', 'wadus')"; - assert.response(app, createGetRequest(sql), RESPONSE_OK, function(res) { + assert.response(server, createGetRequest(sql), RESPONSE_OK, function(err, res) { assert.ok(!res.headers.hasOwnProperty('x-cache-channel'), res.headers['x-cache-channel']); done(); }); @@ -94,7 +91,7 @@ describe('X-Cache-Channel header', function() { it('should not add header for CDB_QueryTables', function(done) { var sql = "SELECT CDB_QueryTablesText('select * from untitle_table_4')"; - assert.response(app, createGetRequest(sql), RESPONSE_OK, function(res) { + assert.response(server, createGetRequest(sql), RESPONSE_OK, function(err, res) { assert.ok(!res.headers.hasOwnProperty('x-cache-channel'), res.headers['x-cache-channel']); done(); }); @@ -102,7 +99,7 @@ describe('X-Cache-Channel header', function() { it('should not add header for non table results', function(done) { var sql = "SELECT 'wadus'::text"; - assert.response(app, createGetRequest(sql), RESPONSE_OK, function(res) { + assert.response(server, createGetRequest(sql), RESPONSE_OK, function(err, res) { assert.ok(!res.headers.hasOwnProperty('x-cache-channel'), res.headers['x-cache-channel']); done(); }); diff --git a/test/helper.js b/test/helper.js index 7c3b132fa..1b2e6b6df 100644 --- a/test/helper.js +++ b/test/helper.js @@ -1,5 +1,2 @@ -var _ = require('underscore'); - -global.settings = require(__dirname + '/../config/settings'); -var env = require(__dirname + '/../config/environments/test'); -_.extend(global.settings, env); +global.settings = require('../config/environments/test'); +process.env.NODE_ENV = 'test'; diff --git a/test/integration/batch/job_backend.test.js b/test/integration/batch/job_backend.test.js new file mode 100644 index 000000000..a1a72c58b --- /dev/null +++ b/test/integration/batch/job_backend.test.js @@ -0,0 +1,159 @@ +'use strict'; + +require('../../helper'); + +var BATCH_SOURCE = '../../../batch/'; + +var assert = require('../../support/assert'); +var redisUtils = require('../../support/redis_utils'); + +var JobQueue = require(BATCH_SOURCE + 'job_queue'); +var JobBackend = require(BATCH_SOURCE + 'job_backend'); +var JobPublisher = require(BATCH_SOURCE + 'pubsub/job-publisher'); +var JobFactory = require(BATCH_SOURCE + 'models/job_factory'); +var jobStatus = require(BATCH_SOURCE + 'job_status'); + +var metadataBackend = require('cartodb-redis')({ pool: redisUtils.getPool() }); +var jobPublisher = new JobPublisher(redisUtils.getPool()); +var jobQueue = new JobQueue(metadataBackend, jobPublisher); + +var queue = require('queue-async'); + +var USER = 'vizzuality'; +var QUERY = 'select pg_sleep(0)'; +var HOST = 'localhost'; +var JOB = { + user: USER, + query: QUERY, + host: HOST +}; + +function createWadusJob() { + return JobFactory.create(JSON.parse(JSON.stringify(JOB))); +} + +describe('job backend', function() { + var jobBackend = new JobBackend(metadataBackend, jobQueue); + + after(function (done) { + redisUtils.clean('batch:*', done); + }); + + it('.create() should persist a job', function (done) { + var job = createWadusJob(); + + jobBackend.create(job.data, function (err, jobCreated) { + if (err) { + return done(err); + } + + assert.ok(jobCreated.job_id); + assert.equal(jobCreated.status, jobStatus.PENDING); + done(); + }); + }); + + it('.create() should return error', function (done) { + var job = createWadusJob(); + + delete job.data.job_id; + + jobBackend.create(job, function (err) { + assert.ok(err); + assert.equal(err.name, 'NotFoundError'); + assert.equal(err.message, 'Job with id undefined not found'); + done(); + }); + }); + + it('.update() should update an existent job', function (done) { + var job = createWadusJob(); + + jobBackend.create(job.data, function (err, jobCreated) { + if (err) { + return done(err); + } + + jobCreated.query = 'select pg_sleep(1)'; + + var job = JobFactory.create(jobCreated); + + jobBackend.update(job.data, function (err, jobUpdated) { + if (err) { + return done(err); + } + + assert.equal(jobUpdated.query, 'select pg_sleep(1)'); + done(); + }); + }); + }); + + it('.update() should return error when updates a nonexistent job', function (done) { + var job = createWadusJob(); + + jobBackend.update(job.data, function (err) { + assert.ok(err, err); + assert.equal(err.name, 'NotFoundError'); + assert.equal(err.message, 'Job with id ' + job.data.job_id + ' not found'); + done(); + }); + }); + + it('.addWorkInProgressJob() should add current job to user and host lists', function (done) { + var job = createWadusJob(); + + jobBackend.addWorkInProgressJob(job.data.user, job.data.job_id, function (err) { + if (err) { + return done(err); + } + done(); + }); + }); + + it('.listWorkInProgressJobByUser() should retrieve WIP jobs of given user', function (done) { + var testStepsQueue = queue(1); + + testStepsQueue.defer(redisUtils.clean, 'batch:wip:user:*'); + testStepsQueue.defer(jobBackend.addWorkInProgressJob.bind(jobBackend), 'vizzuality', 'wadus'); + testStepsQueue.defer(jobBackend.listWorkInProgressJobByUser.bind(jobBackend), 'vizzuality'); + + testStepsQueue.awaitAll(function (err, results) { + if (err) { + return done(err); + } + assert.deepEqual(results[2], ['wadus']); + done(); + }); + }); + + it('.listWorkInProgressJobs() should retrieve WIP users', function (done) { + var jobs = [{ user: 'userA', id: 'jobId1' }, { user: 'userA', id: 'jobId2' }, { user: 'userB', id: 'jobId3' }]; + + var testStepsQueue = queue(1); + + jobs.forEach(function (job) { + testStepsQueue.defer(jobBackend.addWorkInProgressJob.bind(jobBackend), job.user, job.id); + }); + + testStepsQueue.awaitAll(function (err) { + if (err) { + done(err); + } + + jobBackend.listWorkInProgressJobs(function (err, users) { + if (err) { + return done(err); + } + + assert.ok(users.userA); + assert.deepEqual(users.userA, [ 'jobId1', 'jobId2' ]); + assert.ok(users.userB); + assert.deepEqual(users.userB, [ 'jobId3' ]); + done(); + }); + + }); + }); + +}); diff --git a/test/integration/batch/job_canceller.test.js b/test/integration/batch/job_canceller.test.js new file mode 100644 index 000000000..ae640280b --- /dev/null +++ b/test/integration/batch/job_canceller.test.js @@ -0,0 +1,116 @@ +'use strict'; + +require('../../helper'); + +var BATCH_SOURCE = '../../../batch/'; + +var assert = require('../../support/assert'); +var redisUtils = require('../../support/redis_utils'); + +var JobQueue = require(BATCH_SOURCE + 'job_queue'); +var JobBackend = require(BATCH_SOURCE + 'job_backend'); +var JobPublisher = require(BATCH_SOURCE + 'pubsub/job-publisher'); +var jobStatus = require(BATCH_SOURCE + 'job_status'); +var UserDatabaseMetadataService = require(BATCH_SOURCE + 'user_database_metadata_service'); +var JobCanceller = require(BATCH_SOURCE + 'job_canceller'); +var PSQL = require('cartodb-psql'); + +var metadataBackend = require('cartodb-redis')({ pool: redisUtils.getPool() }); +var jobPublisher = new JobPublisher(redisUtils.getPool()); +var jobQueue = new JobQueue(metadataBackend, jobPublisher); +var jobBackend = new JobBackend(metadataBackend, jobQueue); +var userDatabaseMetadataService = new UserDatabaseMetadataService(metadataBackend); +var JobFactory = require(BATCH_SOURCE + 'models/job_factory'); + +var USER = 'vizzuality'; +var QUERY = 'select pg_sleep(0)'; +var HOST = 'localhost'; + +// sets job to running, run its query and returns inmediatly (don't wait for query finishes) +// in order to test query cancelation/draining +function runQueryHelper(job, callback) { + var job_id = job.job_id; + var user = job.user; + var sql = job.query; + + job.status = jobStatus.RUNNING; + + jobBackend.update(job, function (err) { + if (err) { + return callback(err); + } + + userDatabaseMetadataService.getUserMetadata(user, function (err, userDatabaseMetadata) { + if (err) { + return callback(err); + } + + var pg = new PSQL(userDatabaseMetadata, {}, { destroyOnError: true }); + + sql = '/* ' + job_id + ' */ ' + sql; + + pg.eventedQuery(sql, function (err, query) { + if (err) { + return callback(err); + } + + callback(null, query); + }); + }); + }); +} + +function createWadusJob(query) { + query = query || QUERY; + return JobFactory.create(JSON.parse(JSON.stringify({ + user: USER, + query: query, + host: HOST + }))); +} + +describe('job canceller', function() { + var jobCanceller = new JobCanceller(userDatabaseMetadataService); + + after(function (done) { + redisUtils.clean('batch:*', done); + }); + + it('.cancel() should cancel a job', function (done) { + var job = createWadusJob('select pg_sleep(1)'); + + jobBackend.create(job.data, function (err, jobCreated) { + if (err) { + return done(err); + } + + assert.equal(job.data.job_id, jobCreated.job_id); + + runQueryHelper(job.data, function (err) { + if (err) { + return done(err); + } + + jobCanceller.cancel(job, function (err) { + if (err) { + return done(err); + } + + done(); + }); + }); + }); + }); + + it('.cancel() a non running job should not return an error', function (done) { + var job = createWadusJob(); + + jobCanceller.cancel(job, function (err) { + if (err) { + return done(err); + } + + done(); + }); + }); +}); diff --git a/test/integration/batch/job_publisher.test.js b/test/integration/batch/job_publisher.test.js new file mode 100644 index 000000000..72c81f97c --- /dev/null +++ b/test/integration/batch/job_publisher.test.js @@ -0,0 +1,38 @@ +'use strict'; + +require('../../helper'); + +var BATCH_SOURCE = '../../../batch/'; + +var assert = require('../../support/assert'); + +var redisUtils = require('../../support/redis_utils'); + + +var Channel = require(BATCH_SOURCE + 'pubsub/channel'); +var JobPublisher = require(BATCH_SOURCE + 'pubsub/job-publisher'); + +var HOST = 'wadus'; + +describe('job publisher', function() { + var jobPublisher = new JobPublisher(redisUtils.getPool()); + + it('.publish() should publish in job channel', function (done) { + redisUtils.getPool().acquire(Channel.DB, function (err, client) { + if (err) { + return done(err); + } + + client.subscribe(Channel.NAME); + + client.on('message', function (channel, host) { + assert.equal(host, HOST); + assert.equal(channel, Channel.NAME) ; + done(); + }); + + jobPublisher.publish(HOST); + }); + }); + +}); diff --git a/test/integration/batch/job_runner.test.js b/test/integration/batch/job_runner.test.js new file mode 100644 index 000000000..6f923e325 --- /dev/null +++ b/test/integration/batch/job_runner.test.js @@ -0,0 +1,76 @@ +'use strict'; + +require('../../helper'); + +var BATCH_SOURCE = '../../../batch/'; + +var assert = require('../../support/assert'); +var redisUtils = require('../../support/redis_utils'); + +var JobQueue = require(BATCH_SOURCE + 'job_queue'); +var JobBackend = require(BATCH_SOURCE + 'job_backend'); +var JobPublisher = require(BATCH_SOURCE + 'pubsub/job-publisher'); +var jobStatus = require(BATCH_SOURCE + 'job_status'); +var UserDatabaseMetadataService = require(BATCH_SOURCE + 'user_database_metadata_service'); +var JobCanceller = require(BATCH_SOURCE + 'job_canceller'); +var JobService = require(BATCH_SOURCE + 'job_service'); +var JobRunner = require(BATCH_SOURCE + 'job_runner'); +var QueryRunner = require(BATCH_SOURCE + 'query_runner'); + + +var metadataBackend = require('cartodb-redis')({ pool: redisUtils.getPool() }); +var jobPublisher = new JobPublisher(redisUtils.getPool()); +var jobQueue = new JobQueue(metadataBackend, jobPublisher); +var jobBackend = new JobBackend(metadataBackend, jobQueue); +var userDatabaseMetadataService = new UserDatabaseMetadataService(metadataBackend); +var jobCanceller = new JobCanceller(userDatabaseMetadataService); +var jobService = new JobService(jobBackend, jobCanceller); +var queryRunner = new QueryRunner(userDatabaseMetadataService); +var StatsD = require('node-statsd').StatsD; +var statsdClient = new StatsD(global.settings.statsd); + +var USER = 'vizzuality'; +var QUERY = 'select pg_sleep(0)'; +var HOST = 'localhost'; +var JOB = { + user: USER, + query: QUERY, + host: HOST +}; + +describe('job runner', function() { + var jobRunner = new JobRunner(jobService, jobQueue, queryRunner, metadataBackend, statsdClient); + + after(function (done) { + redisUtils.clean('batch:*', function() { + redisUtils.clean('limits:batch:*', done); + }); + }); + + it('.run() should run a job', function (done) { + jobService.create(JOB, function (err, job) { + if (err) { + return done(err); + } + + jobRunner.run(job.data.job_id, function (err, job) { + if (err) { + return done(err); + } + + assert.equal(job.data.status, jobStatus.DONE); + done(); + }); + }); + }); + + it('.run() should return a job not found error', function (done) { + jobRunner.run('wadus_job_id', function (err) { + assert.ok(err, err); + assert.equal(err.name, 'NotFoundError'); + assert.equal(err.message, 'Job with id wadus_job_id not found'); + done(); + }); + }); + +}); diff --git a/test/integration/batch/job_service.test.js b/test/integration/batch/job_service.test.js new file mode 100644 index 000000000..811e8b237 --- /dev/null +++ b/test/integration/batch/job_service.test.js @@ -0,0 +1,201 @@ +'use strict'; + +require('../../helper'); + +var BATCH_SOURCE = '../../../batch/'; + +var assert = require('../../support/assert'); +var redisUtils = require('../../support/redis_utils'); + +var JobQueue = require(BATCH_SOURCE + 'job_queue'); +var JobBackend = require(BATCH_SOURCE + 'job_backend'); +var JobPublisher = require(BATCH_SOURCE + 'pubsub/job-publisher'); +var jobStatus = require(BATCH_SOURCE + 'job_status'); +var UserDatabaseMetadataService = require(BATCH_SOURCE + 'user_database_metadata_service'); +var JobCanceller = require(BATCH_SOURCE + 'job_canceller'); +var JobService = require(BATCH_SOURCE + 'job_service'); +var PSQL = require('cartodb-psql'); + +var metadataBackend = require('cartodb-redis')({ pool: redisUtils.getPool() }); +var jobPublisher = new JobPublisher(redisUtils.getPool()); +var jobQueue = new JobQueue(metadataBackend, jobPublisher); +var jobBackend = new JobBackend(metadataBackend, jobQueue); +var userDatabaseMetadataService = new UserDatabaseMetadataService(metadataBackend); +var jobCanceller = new JobCanceller(userDatabaseMetadataService); + +var USER = 'vizzuality'; +var QUERY = 'select pg_sleep(0)'; +var HOST = 'localhost'; +var JOB = { + user: USER, + query: QUERY, + host: HOST +}; + +function createWadusDataJob() { + return JSON.parse(JSON.stringify(JOB)); +} + +// sets job to running, run its query and returns inmediatly (don't wait for query finishes) +// in order to test query cancelation/draining +function runQueryHelper(job, callback) { + var job_id = job.job_id; + var user = job.user; + var sql = job.query; + + job.status = jobStatus.RUNNING; + + jobBackend.update(job, function (err) { + if (err) { + return callback(err); + } + + userDatabaseMetadataService.getUserMetadata(user, function (err, userDatabaseMetadata) { + if (err) { + return callback(err); + } + + var pg = new PSQL(userDatabaseMetadata, {}, { destroyOnError: true }); + + sql = '/* ' + job_id + ' */ ' + sql; + + pg.eventedQuery(sql, function (err, query) { + if (err) { + return callback(err); + } + + callback(null, query); + }); + }); + }); +} + +describe('job service', function() { + var jobService = new JobService(jobBackend, jobCanceller); + + after(function (done) { + redisUtils.clean('batch:*', done); + }); + + it('.get() should return a job', function (done) { + jobService.create(createWadusDataJob(), function (err, jobCreated) { + if (err) { + return done(err); + } + + jobService.get(jobCreated.data.job_id, function (err, job) { + if (err) { + return done(err); + } + + assert.equal(job.data.job_id, jobCreated.data.job_id); + done(); + }); + }); + }); + + it('.get() should return a not found error', function (done) { + jobService.get('wadus_job_id', function (err) { + assert.ok(err); + assert.equal(err.message, 'Job with id wadus_job_id not found'); + done(); + }); + }); + + it('.create() should persist a job', function (done) { + jobService.create(createWadusDataJob(), function (err, jobCreated) { + if (err) { + return done(err); + } + + assert.ok(jobCreated.data.job_id); + assert.equal(jobCreated.data.status, jobStatus.PENDING); + done(); + }); + }); + + it('.create() should return error with invalid job data', function (done) { + var job = createWadusDataJob(); + + delete job.query; + + jobService.create(job, function (err) { + assert.ok(err); + assert.equal(err.message, 'You must indicate a valid SQL'); + done(); + }); + }); + + it('.cancel() should cancel a running job', function (done) { + var job = createWadusDataJob(); + job.query = 'select pg_sleep(3)'; + + jobService.create(job, function (err, job) { + if (err) { + return done(err); + } + + runQueryHelper(job.data, function (err) { + if (err) { + return done(err); + } + + jobService.cancel(job.data.job_id, function (err, jobCancelled) { + if (err) { + return done(err); + } + + assert.equal(jobCancelled.data.job_id, job.data.job_id); + assert.equal(jobCancelled.data.status, jobStatus.CANCELLED); + done(); + }); + }); + }); + }); + + it('.cancel() should return a job not found error', function (done) { + jobService.cancel('wadus_job_id', function (err) { + assert.ok(err, err); + assert.equal(err.name, 'NotFoundError'); + assert.equal(err.message, 'Job with id wadus_job_id not found'); + done(); + }); + }); + + it('.drain() should draing a running job', function (done) { + var job = createWadusDataJob(); + job.query = 'select pg_sleep(3)'; + + jobService.create(job, function (err, job) { + if (err) { + return done(err); + } + + runQueryHelper(job.data, function (err) { + if (err) { + return done(err); + } + + jobService.drain(job.data.job_id, function (err, jobDrained) { + if (err) { + return done(err); + } + + assert.equal(jobDrained.job_id, job.data.job_id); + assert.equal(jobDrained.status, jobStatus.PENDING); + done(); + }); + }); + }); + }); + + it('.drain() should return a job not found error', function (done) { + jobService.drain('wadus_job_id', function (err) { + assert.ok(err, err); + assert.equal(err.name, 'NotFoundError'); + assert.equal(err.message, 'Job with id wadus_job_id not found'); + done(); + }); + }); + +}); diff --git a/test/integration/batch/locker.js b/test/integration/batch/locker.js new file mode 100644 index 000000000..f1f2116f8 --- /dev/null +++ b/test/integration/batch/locker.js @@ -0,0 +1,60 @@ +'use strict'; + +require('../../helper'); + +var assert = require('../../support/assert'); +var redisUtils = require('../../support/redis_utils'); +var Locker = require('../../../batch/leader/locker'); + +describe('locker', function() { + var host = 'localhost'; + + var TTL = 500; + + var config = { ttl: TTL, pool: redisUtils.getPool() }; + + it('should lock and unlock', function (done) { + var lockerA = Locker.create('redis-distlock', config); + var lockerB = Locker.create('redis-distlock', config); + lockerA.lock(host, function(err, lock) { + if (err) { + return done(err); + } + assert.ok(lock); + + // others can't lock on same host + lockerB.lock(host, function(err) { + assert.ok(err); + assert.equal(err.name, 'LockError'); + + lockerA.unlock(host, function(err) { + assert.ok(!err); + // others can lock after unlock + lockerB.lock(host, function(err, lock2) { + assert.ok(!err); + assert.ok(lock2); + lockerB.unlock(host, done); + }); + }); + }); + }); + }); + + it('should lock and keep locking until unlock', function (done) { + var lockerA = Locker.create('redis-distlock', config); + var lockerB = Locker.create('redis-distlock', config); + lockerA.lock(host, function(err, lock) { + if (err) { + return done(err); + } + setTimeout(function() { + lockerB.lock(host, function(err) { + assert.ok(err); + + assert.ok(lock); + lockerA.unlock(host, done); + }); + }, 2 * TTL); + }); + }); +}); diff --git a/test/integration/batch/queue-seeker.js b/test/integration/batch/queue-seeker.js new file mode 100644 index 000000000..e5ade0e28 --- /dev/null +++ b/test/integration/batch/queue-seeker.js @@ -0,0 +1,65 @@ +'use strict'; + +require('../../helper'); +var assert = require('../../support/assert'); +var redisUtils = require('../../support/redis_utils'); + +var metadataBackend = require('cartodb-redis')({ pool: redisUtils.getPool() }); +var JobPublisher = require('../../../batch/pubsub/job-publisher'); +var QueueSeeker = require('../../../batch/pubsub/queue-seeker'); +var JobQueue = require('../../../batch/job_queue'); + +var jobPublisher = new JobPublisher(redisUtils.getPool()); + + +describe('queue seeker', function() { + var userA = 'userA'; + var userB = 'userB'; + + beforeEach(function () { + this.jobQueue = new JobQueue(metadataBackend, jobPublisher); + }); + + afterEach(function (done) { + redisUtils.clean('batch:*', done); + }); + + it('should find queues for one user', function (done) { + var seeker = new QueueSeeker(redisUtils.getPool()); + this.jobQueue.enqueue(userA, 'wadus-wadus-wadus-wadus', function(err) { + if (err) { + return done(err); + } + seeker.seek(function(err, users) { + assert.ok(!err); + assert.equal(users.length, 1); + assert.equal(users[0], userA); + + return done(); + }); + }); + }); + + it('should find queues for more than one user', function (done) { + var self = this; + var seeker = new QueueSeeker(redisUtils.getPool()); + this.jobQueue.enqueue(userA, 'wadus-wadus-wadus-wadus', function(err) { + if (err) { + return done(err); + } + self.jobQueue.enqueue(userB, 'wadus-wadus-wadus-wadus', function(err) { + if (err) { + return done(err); + } + seeker.seek(function(err, users) { + assert.ok(!err); + assert.equal(users.length, 2); + assert.ok(users[0] === userA || users[0] === userB); + assert.ok(users[1] === userA || users[1] === userB); + + return done(); + }); + }); + }); + }); +}); diff --git a/test/integration/batch/scheduler.js b/test/integration/batch/scheduler.js new file mode 100644 index 000000000..512ef2881 --- /dev/null +++ b/test/integration/batch/scheduler.js @@ -0,0 +1,204 @@ +'use strict'; + +require('../../helper'); +var debug = require('../../../batch/util/debug')('scheduler-test'); +var assert = require('../../support/assert'); +var Scheduler = require('../../../batch/scheduler/scheduler'); +var FixedCapacity = require('../../../batch/scheduler/capacity/fixed'); + +describe('scheduler', function() { + + var USER_FINISHED = true; + + var USER_A = 'userA'; + var USER_B = 'userB'; + var USER_C = 'userC'; + + function TaskRunner(userTasks) { + this.results = []; + this.userTasks = userTasks; + } + + TaskRunner.prototype.run = function(user, callback) { + this.results.push(user); + this.userTasks[user]--; + setTimeout(function() { + return callback(null, this.userTasks[user] === 0); + }.bind(this), 50); + }; + + function ManualTaskRunner() { + this.userTasks = {}; + } + + ManualTaskRunner.prototype.run = function(user, callback) { + if (!this.userTasks.hasOwnProperty(user)) { + this.userTasks[user] = []; + } + this.userTasks[user].push(callback); + }; + + ManualTaskRunner.prototype.dispatch = function(user, isDone) { + if (this.userTasks.hasOwnProperty(user)) { + var cb = this.userTasks[user].shift(); + if (cb) { + return cb(null, isDone); + } + } + }; + + + // simulate one by one or infinity capacity + var capacities = [new FixedCapacity(1), new FixedCapacity(2), new FixedCapacity(Infinity)]; + + capacities.forEach(function(capacity) { + + it('regression #1', function (done) { + var taskRunner = new TaskRunner({ + userA: 2, + userB: 2 + }); + var scheduler = new Scheduler(capacity, taskRunner); + scheduler.add(USER_A); + scheduler.add(USER_B); + + scheduler.on('done', function() { + var results = taskRunner.results; + + assert.equal(results.length, 4); + + assert.equal(results[0], USER_A); + assert.equal(results[1], USER_B); + assert.equal(results[2], USER_A); + assert.equal(results[3], USER_B); + + return done(); + }); + + scheduler.schedule(); + }); + + it('regression #2: it should restart task after it was done but got re-scheduled', function (done) { + var taskRunner = new ManualTaskRunner(); + var scheduler = new Scheduler(capacity, taskRunner); + debug('Adding users A and B'); + scheduler.add(USER_A); + scheduler.add(USER_B); + + var acquiredUsers = []; + + scheduler.on('done', function() { + debug('Users %j', acquiredUsers); + assert.equal(acquiredUsers[0], USER_A); + assert.equal(acquiredUsers[1], USER_B); + assert.equal(acquiredUsers[2], USER_A); + assert.equal(acquiredUsers[3], USER_B); + + assert.equal(acquiredUsers.length, 4); + + return done(); + }); + + scheduler.on('acquired', function(user) { + debug('Acquired user %s', user); + acquiredUsers.push(user); + }); + + scheduler.schedule(); + + debug('User A will be mark as DONE'); + taskRunner.dispatch(USER_A, USER_FINISHED); + + debug('User B should be running'); + debug('User A submit a new task'); + scheduler.add(USER_A); + + debug('User B will get another task to run'); + taskRunner.dispatch(USER_B); + + debug('User A should start working on this new task'); + taskRunner.dispatch(USER_A, USER_FINISHED); + taskRunner.dispatch(USER_B, USER_FINISHED); + }); + + it('should run tasks', function (done) { + var taskRunner = new TaskRunner({ + userA: 1 + }); + var scheduler = new Scheduler(capacity, taskRunner); + scheduler.add(USER_A); + + scheduler.on('done', function() { + var results = taskRunner.results; + + assert.equal(results.length, 1); + + assert.equal(results[0], USER_A); + + return done(); + }); + + scheduler.schedule(); + }); + + + it('should run tasks for different users', function (done) { + var taskRunner = new TaskRunner({ + userA: 1, + userB: 1, + userC: 1 + }); + var scheduler = new Scheduler(capacity, taskRunner); + scheduler.add(USER_A); + scheduler.add(USER_B); + scheduler.add(USER_C); + + scheduler.on('done', function() { + var results = taskRunner.results; + + assert.equal(results.length, 3); + + assert.equal(results[0], USER_A); + assert.equal(results[1], USER_B); + assert.equal(results[2], USER_C); + + return done(); + }); + + scheduler.schedule(); + }); + + it('should be fair when scheduling tasks', function (done) { + var taskRunner = new TaskRunner({ + userA: 3, + userB: 2, + userC: 1 + }); + + var scheduler = new Scheduler(capacity, taskRunner); + scheduler.add(USER_A); + scheduler.add(USER_A); + scheduler.add(USER_A); + scheduler.add(USER_B); + scheduler.add(USER_B); + scheduler.add(USER_C); + + scheduler.on('done', function() { + var results = taskRunner.results; + + assert.equal(results.length, 6); + + assert.equal(results[0], USER_A); + assert.equal(results[1], USER_B); + assert.equal(results[2], USER_C); + assert.equal(results[3], USER_A); + assert.equal(results[4], USER_B); + assert.equal(results[5], USER_A); + + return done(); + }); + + scheduler.schedule(); + }); + }); +}); diff --git a/test/prepare_db.sh b/test/prepare_db.sh index 7563885f7..b86d96cf8 100755 --- a/test/prepare_db.sh +++ b/test/prepare_db.sh @@ -7,6 +7,7 @@ PREPARE_REDIS=yes PREPARE_PGSQL=yes +OFFLINE=no while [ -n "$1" ]; do if test "$1" = "--skip-pg"; then @@ -15,6 +16,9 @@ while [ -n "$1" ]; do elif test "$1" = "--skip-redis"; then PREPARE_REDIS=no shift; continue + elif test "$1" = "--offline"; then + OFFLINE=yes + shift; continue fi done @@ -65,31 +69,36 @@ export PGHOST PGPORT if test x"$PREPARE_PGSQL" = xyes; then echo "preparing postgres..." + echo "PostgreSQL server version: `psql -A -t -c 'select version()'`" dropdb ${TEST_DB} # 2> /dev/null # error expected if doesn't exist, but not otherwise createdb -Ttemplate_postgis -EUTF8 ${TEST_DB} || die "Could not create test database" - psql -c 'CREATE EXTENSION "uuid-ossp";' ${TEST_DB} - cat test.sql | - sed "s/:PUBLICUSER/${PUBLICUSER}/" | - sed "s/:PUBLICPASS/${PUBLICPASS}/" | - sed "s/:TESTUSER/${TESTUSER}/" | - sed "s/:TESTPASS/${TESTPASS}/" | - psql -v ON_ERROR_STOP=1 ${TEST_DB} || exit 1 - - echo "Populating windshaft_test database with reduced populated places data" - cat ./fixtures/populated_places_simple_reduced.sql | - sed "s/:PUBLICUSER/${PUBLICUSER}/" | - sed "s/:PUBLICPASS/${PUBLICPASS}/" | - sed "s/:TESTUSER/${TESTUSER}/" | - sed "s/:TESTPASS/${TESTPASS}/" | - psql -v ON_ERROR_STOP=1 ${TEST_DB} || exit 1 - - # TODO: send in a single run, togheter with test.sql - psql -c "CREATE EXTENSION plpythonu;" ${TEST_DB} - for i in CDB_QueryStatements CDB_QueryTables CDB_CartodbfyTable CDB_TableMetadata CDB_ForeignTable CDB_UserTables CDB_ColumnNames CDB_ZoomFromScale CDB_Overviews + psql -c 'CREATE EXTENSION IF NOT EXISTS "uuid-ossp";' ${TEST_DB} + psql -c "CREATE EXTENSION IF NOT EXISTS plpythonu;" ${TEST_DB} + + LOCAL_SQL_SCRIPTS='test populated_places_simple_reduced' + REMOTE_SQL_SCRIPTS='CDB_QueryStatements CDB_QueryTables CDB_CartodbfyTable CDB_TableMetadata CDB_ForeignTable CDB_UserTables CDB_ColumnNames CDB_ZoomFromScale CDB_OverviewsSupport CDB_Overviews' + + if test x"$OFFLINE" = xno; then + CURL_ARGS="" + for i in ${REMOTE_SQL_SCRIPTS} + do + CURL_ARGS="${CURL_ARGS}\"https://github.com/CartoDB/cartodb-postgresql/raw/master/scripts-available/$i.sql\" -o support/sql/$i.sql " + done + echo "Downloading and updating: ${REMOTE_SQL_SCRIPTS}" + echo ${CURL_ARGS} | xargs curl -L -s + fi + + psql -c "CREATE EXTENSION IF NOT EXISTS plpythonu;" ${TEST_DB} + ALL_SQL_SCRIPTS="${REMOTE_SQL_SCRIPTS} ${LOCAL_SQL_SCRIPTS}" + for i in ${ALL_SQL_SCRIPTS} do - curl -L -s https://github.com/CartoDB/cartodb-postgresql/raw/master/scripts-available/$i.sql -o support/$i.sql - cat support/$i.sql | sed -e 's/cartodb\./public./g' -e "s/''cartodb''/''public''/g" \ - | psql -v ON_ERROR_STOP=1 ${TEST_DB} || exit 1 + cat support/sql/${i}.sql | + sed -e 's/cartodb\./public./g' -e "s/''cartodb''/''public''/g" | + sed "s/:PUBLICUSER/${PUBLICUSER}/" | + sed "s/:PUBLICPASS/${PUBLICPASS}/" | + sed "s/:TESTUSER/${TESTUSER}/" | + sed "s/:TESTPASS/${TESTPASS}/" | + PGOPTIONS='--client-min-messages=WARNING' psql -q -v ON_ERROR_STOP=1 ${TEST_DB} > /dev/null || exit 1 done fi diff --git a/test/run_tests.sh b/test/run_tests.sh index 8741199cf..b91216f86 100755 --- a/test/run_tests.sh +++ b/test/run_tests.sh @@ -17,6 +17,7 @@ OPT_CREATE_REDIS=yes # create/prepare the redis test databases OPT_DROP_PGSQL=yes # drop the postgreql test environment OPT_DROP_REDIS=yes # drop the redis test environment OPT_COVERAGE=no # run tests with coverage +OPT_OFFLINE=no # do not donwload scripts cd $(dirname $0) BASEDIR=$(pwd) @@ -85,6 +86,10 @@ while [ -n "$1" ]; do OPT_COVERAGE=yes shift continue + elif test "$1" = "--offline"; then + OPT_OFFLINE=yes + shift + continue else break fi @@ -119,6 +124,9 @@ fi if test x"$OPT_CREATE_REDIS" != xyes; then PREPARE_DB_OPTS="$PREPARE_DB_OPTS --skip-redis" fi +if test x"$OPT_OFFLINE" == xyes; then + PREPARE_DB_OPTS="$PREPARE_DB_OPTS --offline" +fi echo "Preparing the environment" cd ${BASEDIR} diff --git a/test/support/assert.js b/test/support/assert.js index 64774ef6b..c069b6dd2 100644 --- a/test/support/assert.js +++ b/test/support/assert.js @@ -1,145 +1,74 @@ -var http = require('http'); - var assert = module.exports = exports = require('assert'); +var request = require('request'); -/** - * Assert response from `server` with - * the given `req` object and `res` assertions object. - * - * @param {Server} server - * @param {Object} req - * @param {Object|Function} res - * @param {String|Function} msg - */ -assert.response = function(server, req, res, msg){ - var port = 5555; - function check(){ - try { - server.__port = server.address().port; - server.__listening = true; - } catch (err) { - process.nextTick(check); - return; - } - if (server.__deferred) { - server.__deferred.forEach(function(args){ - assert.response.apply(assert, args); - }); - server.__deferred = null; - } - } - - // Check that the server is ready or defer - if (!server.fd) { - server.__deferred = server.__deferred || []; - server.listen(server.__port = port++, '127.0.0.1', check); - } else if (!server.__port) { - server.__deferred = server.__deferred || []; - process.nextTick(check); - } - - // The socket was created but is not yet listening, so keep deferring - if (!server.__listening) { - server.__deferred.push(arguments); - return; - } - - // Callback as third or fourth arg - var callback = typeof res === 'function' - ? res - : typeof msg === 'function' - ? msg - : function(){}; - - // Default messate to test title - if (typeof msg === 'function') msg = null; - msg = msg || assert.testTitle; - msg += '. '; - - // Pending responses - server.__pending = server.__pending || 0; - server.__pending++; - - // Create client - if (!server.fd) { - server.listen(server.__port = port++, '127.0.0.1', issue); - } else { - issue(); +assert.response = function(server, req, res, callback) { + if (!callback) { + callback = res; + res = {}; } - function issue(){ - - // Issue request - var timer, - method = req.method || 'GET', - status = res.status || res.statusCode, - data = req.data || req.body, - requestTimeout = req.timeout || 0, - encoding = req.encoding || 'utf8'; + var port = 5555, + host = '127.0.0.1'; - var request = http.request({ - host: '127.0.0.1', - port: server.__port, - path: req.url, - method: method, - headers: req.headers, - agent: false + var listeningAttempts = 0; + var listener; + function listen() { + if (listeningAttempts > 25) { + return callback(new Error('Tried too many ports')); + } + listener = server.listen(port, host); + listener.on('error', function() { + port++; + listeningAttempts++; + listen(); }); + listener.on('listening', onServerListening); + } - var check = function() { - if (--server.__pending === 0) { - server.close(); - server.__listening = false; - } + listen(); + + // jshint maxcomplexity:10 + function onServerListening() { + var status = res.status || res.statusCode; + var requestParams = { + url: 'http://' + host + ':' + port + req.url, + method: req.method || 'GET', + headers: req.headers || {}, + timeout: req.timeout || 0, + encoding: req.encoding || 'utf8' }; - // Timeout - if (requestTimeout) { - timer = setTimeout(function(){ - check(); - delete req.timeout; - request.destroy(); // will trigger 'error' event - }, requestTimeout); + if (req.body || req.data) { + requestParams.body = req.body || req.data; } - if (data) request.write(data); - - request.on('error', function(err){ - check(); - callback(null, err); - }); - - request.on('response', function(response){ - response.body = ''; - response.setEncoding(encoding); - response.on('data', function(chunk){ response.body += chunk; }); - response.on('end', function(){ - if (timer) clearTimeout(timer); + request(requestParams, function assert$response$requestHandler(error, response, body) { + listener.close(function() { + if (error) { + return callback(error); + } - check(); + response = response || {}; + response.body = response.body || body; // Assert response body - if (res.body !== undefined) { - var eql = res.body instanceof RegExp - ? res.body.test(response.body) - : res.body === response.body; + if (res.body) { + var eql = res.body instanceof RegExp ? res.body.test(response.body) : res.body === response.body; assert.ok( eql, - msg + 'Invalid response body.\n' - + ' Expected: ' + res.body + '\n' - + ' Got: ' + response.body + colorize('[red]{Invalid response body.}\n' + + ' Expected: [green]{' + res.body + '}\n' + + ' Got: [red]{' + response.body + '}') ); } // Assert response status if (typeof status === 'number') { - assert.equal( - response.statusCode, - status, - msg + colorize('Invalid response status code.\n' - + ' Expected: [green]{' + status + '}\n' - + ' Got: [red]{' + response.statusCode + '}\n' - + ' Response body: ' + response.body) + assert.equal(response.statusCode, status, + colorize('[red]{Invalid response status code.}\n' + + ' Expected: [green]{' + status + '}\n' + + ' Got: [red]{' + response.statusCode + '}\n' + + ' Body: ' + response.body) ); } @@ -150,25 +79,21 @@ assert.response = function(server, req, res, msg){ var name = keys[i], actual = response.headers[name.toLowerCase()], expected = res.headers[name], - eql = expected instanceof RegExp - ? expected.test(actual) - : expected == actual; - assert.ok( - eql, - msg + colorize('Invalid response header [bold]{' + name + '}.\n' - + ' Expected: [green]{' + expected + '}\n' - + ' Got: [red]{' + actual + '}\n' - + ' Response body: ' + response.body) + headerEql = expected instanceof RegExp ? expected.test(actual) : expected === actual; + assert.ok(headerEql, + colorize('Invalid response header [bold]{' + name + '}.\n' + + ' Expected: [green]{' + expected + '}\n' + + ' Got: [red]{' + actual + '}') ); } } - callback(response); + // Callback + callback(null, response); }); }); - request.end(); - } + } }; /** diff --git a/test/support/batch-test-client.js b/test/support/batch-test-client.js new file mode 100644 index 000000000..1b5c20053 --- /dev/null +++ b/test/support/batch-test-client.js @@ -0,0 +1,198 @@ +'use strict'; + +require('../helper'); +var assert = require('assert'); +var appServer = require('../../app/server'); +var redisUtils = require('./redis_utils'); +var debug = require('debug')('batch-test-client'); + +var JobStatus = require('../../batch/job_status'); +var metadataBackend = require('cartodb-redis')({ pool: redisUtils.getPool() }); +var batchFactory = require('../../batch/index'); + +function response(code) { + return { + status: code + }; +} + +var RESPONSE = { + OK: response(200), + CREATED: response(201) +}; + + +function BatchTestClient(config) { + this.config = config || {}; + this.server = appServer(); + + this.batch = batchFactory(metadataBackend, redisUtils.getPool(), this.config.name); + this.batch.start(); + + this.pendingJobs = []; + this.ready = false; + this.batch.on('ready', function() { + this.ready = true; + this.pendingJobs.forEach(function(pendingJob) { + this.createJob(pendingJob.job, pendingJob.callback); + }.bind(this)); + }.bind(this)); +} + +module.exports = BatchTestClient; + +BatchTestClient.prototype.isReady = function() { + return this.ready; +}; + +BatchTestClient.prototype.createJob = function(job, override, callback) { + if (!callback) { + callback = override; + override = {}; + } + if (!this.isReady()) { + this.pendingJobs.push({ + job: job, + callback: callback + }); + return debug('Waiting for Batch service to be ready'); + } + assert.response( + this.server, + { + url: this.getUrl(override), + headers: { + host: this.getHost(override), + 'Content-Type': 'application/json' + }, + method: 'POST', + data: JSON.stringify(job) + }, + RESPONSE.CREATED, + function (err, res) { + if (err) { + return callback(err); + } + return callback(null, new JobResult(JSON.parse(res.body), this, override)); + }.bind(this) + ); +}; + +BatchTestClient.prototype.getJobStatus = function(jobId, override, callback) { + assert.response( + this.server, + { + url: this.getUrl(override, jobId), + headers: { + host: this.getHost(override) + }, + method: 'GET' + }, + RESPONSE.OK, + function (err, res) { + if (err) { + return callback(err); + } + return callback(null, JSON.parse(res.body)); + } + ); +}; + +BatchTestClient.prototype.getWorkInProgressJobs = function(override, callback) { + if (!callback) { + callback = override; + override = {}; + } + + assert.response( + this.server, + { + url: this.getUrl(override, 'wip'), + headers: { + host: this.getHost(override) + }, + method: 'GET' + }, + RESPONSE.OK, + function (err, res) { + if (err) { + return callback(err); + } + return callback(null, JSON.parse(res.body)); + } + ); +}; + +BatchTestClient.prototype.cancelJob = function(jobId, override, callback) { + assert.response( + this.server, + { + url: this.getUrl(jobId), + headers: { + host: this.getHost(override) + }, + method: 'DELETE' + }, + RESPONSE.OK, + function (err, res) { + if (err) { + return callback(err); + } + return callback(null, JSON.parse(res.body)); + } + ); +}; + +BatchTestClient.prototype.drain = function(callback) { + this.batch.stop(function() { + return redisUtils.clean('batch:*', callback); + }); +}; + +BatchTestClient.prototype.getHost = function(override) { + return override.host || this.config.host || 'vizzuality.cartodb.com'; +}; + +BatchTestClient.prototype.getUrl = function(override, jobId) { + var urlParts = ['/api/v2/sql/job']; + if (jobId) { + urlParts.push(jobId); + } + return urlParts.join('/') + '?api_key=' + this.getApiKey(override); +}; + +BatchTestClient.prototype.getApiKey = function(override) { + return override.apiKey || this.config.apiKey || '1234'; +}; + +/****************** JobResult ******************/ + + +function JobResult(job, batchTestClient, override) { + this.job = job; + this.batchTestClient = batchTestClient; + this.override = override; +} + +JobResult.prototype.getStatus = function(callback) { + var self = this; + var interval = setInterval(function () { + self.batchTestClient.getJobStatus(self.job.job_id, self.override, function (err, job) { + if (err) { + clearInterval(interval); + return callback(err); + } + + if (JobStatus.isFinal(job.status)) { + clearInterval(interval); + return callback(null, job); + } else { + debug('Job %s [status=%s] waiting to be done', self.job.job_id, job.status); + } + }); + }, 50); +}; + +JobResult.prototype.cancel = function(callback) { + this.batchTestClient.cancelJob(this.job.job_id, this.override, callback); +}; diff --git a/test/support/redis_utils.js b/test/support/redis_utils.js new file mode 100644 index 000000000..be48945a3 --- /dev/null +++ b/test/support/redis_utils.js @@ -0,0 +1,35 @@ +'use strict'; + +var RedisPool = require('redis-mpool'); + +var redisConfig = { + host: global.settings.redis_host, + port: global.settings.redis_port, + max: global.settings.redisPool, + idleTimeoutMillis: global.settings.redisIdleTimeoutMillis, + reapIntervalMillis: global.settings.redisReapIntervalMillis +}; +var metadataBackend = require('cartodb-redis')(redisConfig); + +module.exports.clean = function clean(pattern, callback) { + metadataBackend.redisCmd(5, 'KEYS', [ pattern ], function (err, keys) { + if (err) { + return callback(err); + } + + if (!keys || !keys.length) { + return callback(); + } + + metadataBackend.redisCmd(5, 'DEL', keys, callback); + }); +}; + +module.exports.getConfig = function getConfig() { + return redisConfig; +}; + +var pool = new RedisPool(redisConfig); +module.exports.getPool = function getPool() { + return pool; +}; diff --git a/test/support/server_utils.js b/test/support/server_utils.js deleted file mode 100644 index 42f229e8a..000000000 --- a/test/support/server_utils.js +++ /dev/null @@ -1,17 +0,0 @@ -var utils = {}; - -utils.startOnNextPort = function(server, issue, start_port) { - - var port = start_port || 5555; - - server.on('error', function(e) { - console.log("Port " + port + " already in use, retrying"); - utils.startOnNextPort(server, issue, port+1); - }); - - server.listen(port, '127.0.0.1', issue); -} - -module.exports = utils; - - diff --git a/test/fixtures/populated_places_simple_reduced.sql b/test/support/sql/populated_places_simple_reduced.sql similarity index 100% rename from test/fixtures/populated_places_simple_reduced.sql rename to test/support/sql/populated_places_simple_reduced.sql diff --git a/test/test.sql b/test/support/sql/test.sql similarity index 100% rename from test/test.sql rename to test/support/sql/test.sql diff --git a/test/support/test-client.js b/test/support/test-client.js new file mode 100644 index 000000000..c1a74ae08 --- /dev/null +++ b/test/support/test-client.js @@ -0,0 +1,63 @@ +'use strict'; + +require('../helper'); +var assert = require('assert'); +var appServer = require('../../app/server'); + +function response(code) { + return { + status: code + }; +} + +var RESPONSE = { + OK: response(200), + CREATED: response(201) +}; + + +function TestClient(config) { + this.config = config || {}; + this.server = appServer(); +} + +module.exports = TestClient; + + +TestClient.prototype.getResult = function(query, override, callback) { + if (!callback) { + callback = override; + override = {}; + } + assert.response( + this.server, + { + url: this.getUrl(override), + headers: { + host: this.getHost(override), + 'Content-Type': 'application/json' + }, + method: 'POST', + data: JSON.stringify({ + q: query + }) + }, + RESPONSE.OK, + function (err, res) { + if (err) { + return callback(err); + } + var result = JSON.parse(res.body); + + return callback(null, result.rows || []); + } + ); +}; + +TestClient.prototype.getHost = function(override) { + return override.host || this.config.host || 'vizzuality.cartodb.com'; +}; + +TestClient.prototype.getUrl = function(override) { + return '/api/v2/sql?api_key=' + (override.apiKey || this.config.apiKey || '1234'); +}; diff --git a/test/unit/batch/job_publisher.js b/test/unit/batch/job_publisher.js index 57e02c56c..17eb1919f 100644 --- a/test/unit/batch/job_publisher.js +++ b/test/unit/batch/job_publisher.js @@ -1,4 +1,5 @@ -var JobPublisher = require('../../../batch/job_publisher'); +var Channel = require('../../../batch/pubsub/channel'); +var JobPublisher = require('../../../batch/pubsub/job-publisher'); var assert = require('assert'); describe('batch API job publisher', function () { @@ -10,13 +11,22 @@ describe('batch API job publisher', function () { return this; }, publish: function () { - var isValidFirstArg = arguments[0] === 'batch:hosts'; + var isValidFirstArg = arguments[0] === Channel.NAME; var isValidSecondArg = arguments[1] === self.host; self.redis.publishIsCalledWithValidArgs = isValidFirstArg && isValidSecondArg; + }, + on: function () {}, + ping: function (cb) { + cb(); + } + }; + this.pool = { + acquire: function (db, cb) { + cb(null, self.redis); } }; - this.jobPublisher = new JobPublisher(this.redis); + this.jobPublisher = new JobPublisher(this.pool); }); it('.publish() should publish new messages', function () { diff --git a/test/unit/batch/job_queue.js b/test/unit/batch/job_queue.js index 9b54371c1..c3f376e3b 100644 --- a/test/unit/batch/job_queue.js +++ b/test/unit/batch/job_queue.js @@ -11,29 +11,29 @@ describe('batch API job queue', function () { }); } }; - this.jobQueue = new JobQueue(this.metadataBackend); + this.jobPublisher = { + publish: function () {} + }; + this.jobQueue = new JobQueue(this.metadataBackend, this.jobPublisher); }); it('.enqueue() should enqueue the provided job', function (done) { - this.jobQueue.enqueue('irrelevantJob', 'irrelevantHost', function (err, username) { + this.jobQueue.enqueue('irrelevantJob', 'irrelevantHost', function (err) { assert.ok(!err); - assert.equal(username, 'irrelevantJob'); done(); }); }); it('.dequeue() should dequeue the next job', function (done) { - this.jobQueue.dequeue('irrelevantHost', function (err, username) { + this.jobQueue.dequeue('irrelevantHost', function (err) { assert.ok(!err); - assert.equal(username, 'irrelevantJob'); done(); }); }); it('.enqueueFirst() should dequeue the next job', function (done) { - this.jobQueue.enqueueFirst('irrelevantJob', 'irrelevantHost', function (err, username) { + this.jobQueue.enqueueFirst('irrelevantJob', 'irrelevantHost', function (err) { assert.ok(!err); - assert.equal(username, 'irrelevantJob'); done(); }); }); diff --git a/test/unit/batch/job_subscriber.js b/test/unit/batch/job_subscriber.js index d4b607d56..70caa1a95 100644 --- a/test/unit/batch/job_subscriber.js +++ b/test/unit/batch/job_subscriber.js @@ -1,29 +1,43 @@ -var JobSubscriber = require('../../../batch/job_subscriber'); +var Channel = require('../../../batch/pubsub/channel'); +var JobSubscriber = require('../../../batch/pubsub/job-subscriber'); var assert = require('assert'); describe('batch API job subscriber', function () { beforeEach(function () { var self = this; + this.onMessageListener = function () {}; this.redis = { createClient: function () { return this; }, subscribe: function () { - var isValidFirstArg = arguments[0] === 'batch:hosts'; + var isValidFirstArg = arguments[0] === Channel.NAME; self.redis.subscribeIsCalledWithValidArgs = isValidFirstArg; }, on: function () { - var isValidFirstArg = arguments[0] === 'message'; - var isValidSecondArg = arguments[1] === self.onMessageListener; - self.redis.onIsCalledWithValidArgs = isValidFirstArg && isValidSecondArg; + if (arguments[0] === 'message') { + self.redis.onIsCalledWithValidArgs = true; + } }, unsubscribe: function () { - var isValidFirstArg = arguments[0] === 'batch:hosts'; + var isValidFirstArg = arguments[0] === Channel.NAME; self.redis.unsubscribeIsCalledWithValidArgs = isValidFirstArg; }, + scan: function(params, callback) { + return callback(null, ['0']); + }, removeAllListeners: function () { return this; + }, + connected: true + }; + this.pool = { + acquire: function (db, cb) { + cb(null, self.redis); + }, + release: function(/*db, client*/) { + } }; this.queueSeeker = { @@ -34,7 +48,7 @@ describe('batch API job subscriber', function () { } }; - this.jobSubscriber = new JobSubscriber(this.redis, this.queueSeeker); + this.jobSubscriber = new JobSubscriber(this.pool, this.queueSeeker); }); it('.subscribe() should listen for incoming messages', function () { @@ -44,6 +58,7 @@ describe('batch API job subscriber', function () { }); it('.unsubscribe() should stop listening for incoming messages', function () { + this.jobSubscriber.subscribe(this.onMessageListener); this.jobSubscriber.unsubscribe(); assert.ok(this.redis.unsubscribeIsCalledWithValidArgs); }); diff --git a/test/unit/batch/user_indexer.js b/test/unit/batch/user_indexer.js deleted file mode 100644 index 38185d37c..000000000 --- a/test/unit/batch/user_indexer.js +++ /dev/null @@ -1,80 +0,0 @@ -var UserIndexer = require('../../../batch/user_indexer'); -var assert = require('assert'); - -describe('batch API user indexer', function () { - describe('backend works well', function () { - beforeEach(function () { - this.metadataBackend = { - redisCmd: function () { - var callback = arguments[arguments.length -1]; - process.nextTick(function () { - callback(null, 'irrelevantJob'); - }); - } - }; - this.userIndexer = new UserIndexer(this.metadataBackend); - }); - - it('.add() should save the given job into the given username list', function (done) { - this.userIndexer.add('irrelevantUsername', 'irrelevantJobId', function (err) { - assert.ok(!err); - done(); - }); - }); - - it('.list() should list jobs of the given username', function (done) { - this.userIndexer.list('irrelevantUsername', function (err) { - assert.ok(!err); - done(); - }); - }); - - it('.remove() should remove the job id from the given username list', function (done) { - this.userIndexer.remove('irrelevantUsername', 'irrelevantJobId', function (err) { - assert.ok(!err); - done(); - }); - }); - }); - - - describe('backend fails', function () { - beforeEach(function () { - this.metadataBackend = { - redisCmd: function () { - var callback = arguments[arguments.length -1]; - process.nextTick(function () { - callback(new Error('Something went wrong')); - }); - } - }; - this.userIndexer = new UserIndexer(this.metadataBackend); - }); - - it('.add() should save the given job into the given username list', function (done) { - this.userIndexer.add('irrelevantUsername', 'irrelevantJobId', function (err) { - assert.ok(err); - assert.ok(err.message, 'Something went wrong'); - done(); - }); - }); - - it('.list() should list jobs of the given username', function (done) { - this.userIndexer.list('irrelevantUsername', function (err) { - assert.ok(err); - assert.ok(err.message, 'Something went wrong'); - done(); - }); - }); - - it('.remove() should remove the job id from the given username list', function (done) { - this.userIndexer.remove('irrelevantUsername', 'irrelevantJobId', function (err) { - assert.ok(err); - assert.ok(err.message, 'Something went wrong'); - done(); - }); - }); - - }); - -}); diff --git a/test/unit/oauth.test.js b/test/unit/oauth.test.js index 9571cc50d..c3795326c 100644 --- a/test/unit/oauth.test.js +++ b/test/unit/oauth.test.js @@ -78,7 +78,7 @@ it('test can access oauth hash for a user based on access token (oauth_token)', }); it('test non existant oauth hash for a user based on oauth_token returns empty hash', function(done){ - var req = {query:{}, headers:{authorization:full_oauth_header}}; + var req = {query:{}, params: { user: 'vizzuality' }, headers:{authorization:full_oauth_header}}; var tokens = oAuth.parseTokens(req); oAuth.getOAuthHash(metadataBackend, tokens.oauth_token, function(err, data){ @@ -91,12 +91,34 @@ it('test non existant oauth hash for a user based on oauth_token returns empty h it('can return user for verified signature', function(done){ var req = {query:{}, headers:{authorization:real_oauth_header, host: 'vizzuality.testhost.lan' }, + params: { user: 'vizzuality' }, + protocol: 'http', + method: 'GET', + path: '/api/v1/tables' + }; + + oAuth.verifyRequest(req, metadataBackend, function(err, data){ + assert.ok(!err, err); + assert.equal(data, 1); + done(); + }); +}); + +it('can return user for verified signature (for other allowed domains)', function(done){ + var oAuthGetAllowedHostsFn = oAuth.getAllowedHosts; + oAuth.getAllowedHosts = function() { + return ['testhost.lan', 'testhostdb.lan']; + }; + var req = {query:{}, + headers:{authorization:real_oauth_header, host: 'vizzuality.testhostdb.lan' }, + params: { user: 'vizzuality' }, protocol: 'http', method: 'GET', path: '/api/v1/tables' }; oAuth.verifyRequest(req, metadataBackend, function(err, data){ + oAuth.getAllowedHosts = oAuthGetAllowedHostsFn; assert.ok(!err, err); assert.equal(data, 1); done(); @@ -106,6 +128,7 @@ it('can return user for verified signature', function(done){ it('returns null user for unverified signatures', function(done){ var req = {query:{}, headers:{authorization:real_oauth_header, host: 'vizzuality.testyhost.lan' }, + params: { user: 'vizzuality' }, protocol: 'http', method: 'GET', path: '/api/v1/tables' @@ -121,6 +144,7 @@ it('returns null user for no oauth', function(done){ var req = { query:{}, headers:{}, + params: { user: 'vizzuality' }, protocol: 'http', method: 'GET', path: '/api/v1/tables'