Merge pull request #215 from idralyuk/0.4

Emit a warning if the number of nested requests is being pruned.

modules/engine/lib/engine/http.request.js

@@ -34,6 +34,8 @@ var strTemplate = require('./peg/str-template.js'),
     uuid = require('node-uuid'),
     os = require('os');
 
+var maxResponseLength;
+
 exports.exec = function(args) {
     var request, context, resource, statement, params, resourceUri, template, cb, holder, tasks,
         logEmitter;
@@ -46,6 +48,8 @@ exports.exec = function(args) {
     logEmitter = args.logEmitter;
     holder = {};
 
+    maxResponseLength = maxResponseLength || getMaxResponseLength(args.config, logEmitter);
+
     // Prepare params (former ones override the later ones)
     params = prepareParams(context,
         args.params,
@@ -323,11 +327,27 @@ function sendMessage(client, emitter, logEmitter, statement, httpReqTx, options,
         emitter.emit(packet.type, packet);
     }
 
+
     clientRequest = client.request(options, function(res) {
         setEncoding(res);
         respData = '';
+        var responseLength = 0;
         res.on('data', function (chunk) {
+            responseLength += chunk.length;
+
+            if (responseLength > maxResponseLength) {
+                var err = new Error('Response length exceeds limit');
+                err.uri = resourceUri;
+                err.status = 502;
+
+                logEmitter.emitError(httpReqTx.event, 'error with uri - ' + resourceUri + ' - ' +
+                    'response length ' + responseLength + ' exceeds config.maxResponseLength of ' + maxResponseLength +
+                    ' ' + (Date.now() - start) + 'msec');
+                res.socket.destroy();
+                return httpReqTx.cb(err);
+            }
             respData += chunk;
+
         });
         res.on('end', function() {
 
@@ -730,4 +750,17 @@ function toISO(d) {
         + pad(d.getUTCHours()) + ':'
         + pad(d.getUTCMinutes()) + ':'
         + pad(d.getUTCSeconds()) + 'Z';
+}
+
+function getMaxResponseLength(config, logEmitter) {
+    if (config && config.maxResponseLength) {
+        maxResponseLength = config.maxResponseLength;
+    }
+
+    if (typeof maxResponseLength == 'undefined') {
+        maxResponseLength = 10000000; // default to 10,000,000
+        logEmitter.emitWarning('config.maxResponseLength is undefined! Defaulting to ' + maxResponseLength);
+    }
+
+    return maxResponseLength;
 }

modules/engine/lib/engine/select.js

@@ -54,6 +54,12 @@ exports.exec = function(opts, statement, cb, parentEvent) {
                 // Set the join field
                 cloned.whereCriteria[0].rhs.value = (_.isArray(row) || _.isObject(row)) ? row[joiningColumn] : row;
 
+                // Determine whether the number of funcs is within the limit, otherwise break out of the loop
+                if (funcs.length >= (maxNestedRequests || getMaxNestedRequests(opts))) {
+                    opts.logEmitter.emitWarning('Pruning the number of nested requests to config.maxNestedRequests = ' + maxNestedRequests + '.');
+                    return;
+                }
+
                 funcs.push(function(s) {
                     return function(callback) {
                         execInternal(opts, s, function(e, r) {
@@ -68,9 +74,6 @@ exports.exec = function(opts, statement, cb, parentEvent) {
                 }(cloned));
             });
 
-            // Determine whether the number of funcs is within the limit and prune the funcs array
-            funcs = funcs.slice(0, maxNestedRequests || getMaxNestedRequests(opts));
-
             // Execute joins
             async.parallel(funcs, function(err, more) {
                 // If there is nothing to loop throough, leave the body undefined.
@@ -169,7 +172,10 @@ function execInternal(opts, statement, cb, parentEvent) {
                         ret[name] = [];
 
                         // Determine whether the number of values is within the limit and prune the values array
-                        cond.rhs.value = cond.rhs.value.slice(0, maxNestedRequests || getMaxNestedRequests(opts));
+                        if (cond.rhs.value.length > (maxNestedRequests || getMaxNestedRequests(opts))) {
+                            opts.logEmitter.emitWarning('Pruning the number of nested requests in in-clause to config.maxNestedRequests = ' + maxNestedRequests + '.');
+                            cond.rhs.value = cond.rhs.value.slice(0, maxNestedRequests);
+                        }
 
                         // Expand variables from context
                         _.each(cond.rhs.value, function(key) {

modules/engine/test/config/dev.json

@@ -3,6 +3,8 @@
         "apikey": "Qlio1a92e-fea5-485d-bcdb-1140ee96527"
 
     },
-    "maxNestedCalls" : 50
+    "maxNestedCalls" : 50,
+    "maxRequestLength" : 10000000,
+    "maxResponseLength" : 10000000
 }
 

modules/engine/test/max-reqresp-size-test.js

@@ -0,0 +1,58 @@
+/*
+ * Copyright 2011 eBay Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+"use strict";
+
+var _ = require('underscore'),
+    Engine = require('../lib/engine'),
+    logger = require('winston');
+
+logger.remove(logger.transports.Console);
+logger.add(logger.transports.Console, {level: 'error'});
+
+var engine = new Engine({
+    tables : __dirname + '/tables',
+    config: __dirname + '/config/dev.json',
+    connection: 'close'
+});
+
+module.exports = {
+    'response-from-server': function(test) {
+        // temporarily lower the max in order to run this test
+        var defaultMaxResponseLength = engine.config.maxResponseLength;
+        engine.config.maxResponseLength = 5000;
+
+        var script = "create table maxtable \
+                    on select get from 'http://svcs.ebay.com:80/services/search/FindingService/v1?OPERATION-NAME=findItemsByKeywords&SERVICE-VERSION=1.8.0&GLOBAL-ID={globalid}&SECURITY-APPNAME={apikey}&RESPONSE-DATA-FORMAT={format}&REST-PAYLOAD&keywords={^keywords}'\
+                     with aliases format = 'RESPONSE-DATA-FORMAT', json = 'JSON', xml = 'XML'\
+                     using defaults format = 'JSON', globalid = 'EBAY-US', sortorder ='BestMatch',\
+                           apikey = '{config.ebay.apikey}', limit = 1000, pageNumber = 1\
+                     resultset 'findItemsByKeywordsResponse.searchResult.item';\n\
+        select * from maxtable where keywords = 'ipad'";
+        engine.exec(script, function(err, list) {
+            logger.log('engine.config.maxResponseLength' + engine.config.maxResponseLength);
+            if (!err) {
+                test.fail('did not get expected error');
+                test.done();
+            } else {
+                test.equals(err.status, 502, '502 status code expected');
+                test.equals(err.message, 'Response length exceeds limit', 'Error explanation expected');
+                test.done();
+            }
+        });
+        engine.config.maxResponseLength = defaultMaxResponseLength;
+    }
+}