Skip to content

Added follow redirects feature. #363

Merged
merged 8 commits into from Mar 20, 2012
View
56 modules/engine/lib/engine/http/request.js
@@ -81,8 +81,58 @@ function putInCache(key, cache, result, res, timeout) {
}
}
-function sendHttpRequest(client, options, args, start, timings, reqStart, key, cache, timeout, uniqueId, status, retry) {
+var followRedirects = true, maxRedirects = 10;
+
+function sendHttpRequest(client, options, args, start, timings, reqStart, key, cache, timeout, uniqueId, status, retry, redirects) {
var clientRequest = client.request(options, function (res) {
+ if (followRedirects && (res.statusCode >= 301 && res.statusCode <= 307) &&
+ (options.method.toUpperCase() === 'GET' || options.method.toUpperCase() === 'HEAD')) {
+ res.socket.destroy();
+ if (res.statusCode === 305) { // Log but don't follow
+ args.logEmitter.emitWarning(args.httpReqTx.event, 'Warning with uri - ' + args.uri + ' - ' +
+ 'Received status code 305 ' + (Date.now() - start) + 'msec');
+ var err = new Error('Received status code 305 from downstream server');
+ err.uri = args.uri;
+ err.status = 502;
+ return args.httpReqTx.cb(err);
+ } else if (res.statusCode !== 304 && res.statusCode !== 306) { // Only follow 301, 302, 303, 307
+ if (res.headers.location) {
+ if (redirects++ >= maxRedirects) {
+ args.logEmitter.emitError(args.httpReqTx.event, 'Error with uri - ' + args.uri + ' - ' +
+ 'Exceeded max redirects (' + maxRedirects + '). In a loop? ' + (Date.now() - start) + 'msec');
+ var err = new Error('Exceeded max redirects');
+ err.uri = args.uri;
+ err.status = 502;
+ return args.httpReqTx.cb(err);
+ }
+
+ var location = new URI(res.headers.location);
+
+ if (location.isAbsolute()) {
+ options.host = location.heirpart().authority().host();
+ options.port = location.heirpart().authority().port();
+ } else {
+ location = new URI(args.uri);
+ location = location.resolveReference(res.headers.location);
+ }
+ options.path = location.heirpart().path();
+
+ args.logEmitter.emitEvent(args.httpReqTx.event, 'being redirected for the ' + redirects + ' time, ' +
+ 'going to ' + options.host + ':' + options.port + options.path + ' - ' + args.uri + ' - ' + (Date.now() - start) + 'msec');
+ sendHttpRequest(client, options, args, start, timings, reqStart, key, cache, timeout, uniqueId, status, retry, redirects);
+ return;
+ } else {
+ args.logEmitter.emitError(args.httpReqTx.event, 'Error with uri - ' + args.uri + ' - ' +
+ 'Received status code ' + res.statusCode + ', but Location header was not provided' +
+ ' ' + (Date.now() - start) + 'msec');
+ var err = new Error('Missing Location header in redirect');
+ err.uri = args.uri;
+ err.status = 502;
+ return args.httpReqTx.cb(err);
+ }
+ }
+ }
+
var bufs = []; // array for bufs for each chunk
var responseLength = 0;
var contentEncoding = res.headers['content-encoding'];
@@ -228,15 +278,15 @@ function sendMessage(args, client, options, retry) {
cache.get(key,function(err,result){
if(err || !result.data){
sendHttpRequest(client, options, args, start, timings, reqStart,
- key, cache, timeout, uniqueId, status, retry);
+ key, cache, timeout, uniqueId, status, retry, 0);
}
else {
response.exec(timings, reqStart, args, uniqueId, res, result.start, result.result, options, status);
}
});
}
else {
- sendHttpRequest(client, options, args, start, timings, reqStart, key, cache, timeout, uniqueId, status, retry);
+ sendHttpRequest(client, options, args, start, timings, reqStart, key, cache, timeout, uniqueId, status, retry, 0);
}
}
View
2 modules/engine/package.json
@@ -1,7 +1,7 @@
{
"author": "ql.io",
"name": "ql.io-engine",
- "version": "0.4.19",
+ "version": "0.4.20",
"repository": {
"type": "git",
"url": "https://github.com/ql-io/ql.io"
View
5 modules/engine/test/mock/redirect-rel.ql
@@ -0,0 +1,5 @@
+create table redirect
+ on select get from "http://127.0.0.1:8300/rel/redirect-response.json"
+response = select * from redirect;
+
+return response;
View
4 modules/engine/test/mock/redirect-response.json
@@ -0,0 +1,4 @@
+{
+ "id": "42",
+ "title":"Redirect Response"
+}
View
6 modules/engine/test/mock/redirect.ql
@@ -0,0 +1,6 @@
+create table redirect
+ on select get from "http://127.0.0.1:8300/redirect-response.json"
+
+response = select * from redirect;
+
+return response;
View
356 modules/engine/test/redirect-test.js
@@ -0,0 +1,356 @@
+/**
+ /*
+ * Copyright 2011 eBay Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+"use strict"
+
+var Engine = require('../lib/engine'),
+ http = require('http'),
+ fs = require('fs'),
+ util = require('util'),
+ logger = require('winston');
+
+logger.remove(logger.transports.Console);
+logger.add(logger.transports.Console, {level:'error'});
+
+var localhost = "127.0.0.1", protocol = "http";
+
+// http/request.js will follow up to 10 redirects
+
+var servers_positive = [
+ { 'host' : localhost, 'port' : '8300', 'status' : 301 },
+ { 'host' : localhost, 'port' : '8301', 'status' : 302 },
+ { 'host' : localhost, 'port' : '8302', 'status' : 303 },
+ { 'host' : localhost, 'port' : '8303', 'status' : 307 },
+ { 'host' : localhost, 'port' : '8304', 'status' : 301 },
+ { 'host' : localhost, 'port' : '8305', 'status' : 302 },
+ { 'host' : localhost, 'port' : '8306', 'status' : 303 },
+ { 'host' : localhost, 'port' : '8307', 'status' : 307 },
+ { 'host' : localhost, 'port' : '8308', 'status' : 301 },
+ { 'host' : localhost, 'port' : '8309', 'status' : 302 },
+ { 'host' : localhost, 'port' : '8310' }
+];
+
+var servers_negative = [
+ { 'host' : localhost, 'port' : '8300', 'status' : 301 },
+ { 'host' : localhost, 'port' : '8301', 'status' : 302 },
+ { 'host' : localhost, 'port' : '8302', 'status' : 303 },
+ { 'host' : localhost, 'port' : '8303', 'status' : 307 },
+ { 'host' : localhost, 'port' : '8304', 'status' : 301 },
+ { 'host' : localhost, 'port' : '8305', 'status' : 302 },
+ { 'host' : localhost, 'port' : '8306', 'status' : 303 },
+ { 'host' : localhost, 'port' : '8307', 'status' : 307 },
+ { 'host' : localhost, 'port' : '8308', 'status' : 301 },
+ { 'host' : localhost, 'port' : '8309', 'status' : 302 },
+ { 'host' : localhost, 'port' : '8310', 'status' : 303 },
+ { 'host' : localhost, 'port' : '8311', 'status' : 307 },
+ { 'host' : localhost, 'port' : '8312' }
+];
+
+var servers_rel_location = [
+ { 'host' : localhost, 'port' : '8300', 'status' : 301 }
+];
+
+var servers_no_location = [
+ { 'host' : localhost, 'port' : '8300', 'status' : 302 },
+ { 'host' : localhost, 'port' : '8301' }
+];
+
+var servers_305 = [
+ { 'host' : localhost, 'port' : '8300', 'status' : 305 },
+ { 'host' : localhost, 'port' : '8301' }
+];
+
+
+function setupServers(servers) {
+
+ //create redirect servers that return response codes in the 300 range (from 300 to 309), and provide Location header
+ for (var i = 0; i < servers.length - 1; i++) {
+ servers[i].instance = http.createServer((function(j) {
+ return function (req, res) {
+ var location = protocol + '://' + servers[j + 1].host + ':' + servers[j + 1].port + req.url;
+ res.writeHead(servers[j].status, { 'Location': location });
+ res.end();
+ }
+ })(i)).listen(servers[i].port, servers[i].host);
+ }
+
+ // create the final server, the one that's going to return data
+ servers[i].instance = http.createServer(function (req, res) {
+ var file = __dirname + '/mock' + req.url;
+ var stat = fs.statSync(file);
+ res.writeHead(200, {
+ 'Content-Type':file.indexOf('.xml') >= 0 ? 'application/xml' : 'application/json',
+ 'Content-Length':stat.size
+ });
+ var readStream = fs.createReadStream(file);
+ util.pump(readStream, res, function (e) {
+ if (e) {
+ console.log(e.stack || e);
+ }
+ res.end();
+ });
+ }).listen(servers[i].port, servers[i].host);
+}
+
+module.exports = {
+ 'positive':function (test) {
+ var servers = servers_positive;
+
+ setupServers(servers);
+
+ var engine = new Engine({
+ config:__dirname + '/config/dev.json'
+ });
+
+ var script = fs.readFileSync(__dirname + '/mock/redirect.ql', 'UTF-8');
+
+ engine.exec({
+ script:script,
+ cb:function (err, result) {
+ try {
+ if (err) {
+ console.log(err.stack || err);
+ test.ok(false);
+ } else {
+ test.ok(result && result.body.id === "42");
+ }
+ test.done();
+ }
+ finally {
+ for (var i = 0; i < servers.length; i++) {
+ servers[i].instance.close();
+ }
+ }
+ }
+ });
+ },
+ 'negative':function (test) {
+ var servers = servers_negative;
+
+ setupServers(servers);
+
+ var engine = new Engine({
+ config:__dirname + '/config/dev.json'
+ });
+
+ var script = fs.readFileSync(__dirname + '/mock/redirect.ql', 'UTF-8');
+
+ engine.exec({
+ script:script,
+ cb:function (err, result) {
+ try {
+ if (!err) {
+ test.ok(false, "Error expected.");
+ } else {
+ test.ok(err.message === "Exceeded max redirects");
+ }
+ test.done();
+ }
+ finally {
+ for (var i = 0; i < servers.length; i++) {
+ servers[i].instance.close();
+ }
+ }
+ }
+ });
+
+ },
+ 'rel-location-header':function (test) {
+ var servers = servers_rel_location;
+
+ // Special case: need to create custom server that will redirect if given prefix '/rel' in the path, but return data otherwise
+ servers[0].instance = http.createServer(function (req, res) {
+ if (req.url.indexOf('/rel') === 0) {
+ res.writeHead(servers[0].status, { 'Location': req.url.substring('/rel'.length) });
+ res.end();
+ return;
+ }
+
+ var file = __dirname + '/mock' + req.url;
+ var stat = fs.statSync(file);
+ res.writeHead(200, {
+ 'Content-Type':file.indexOf('.xml') >= 0 ? 'application/xml' : 'application/json',
+ 'Content-Length':stat.size
+ });
+ var readStream = fs.createReadStream(file);
+ util.pump(readStream, res, function (e) {
+ if (e) {
+ console.log(e.stack || e);
+ }
+ res.end();
+ });
+ }).listen(servers[0].port, servers[0].host);
+
+ var engine = new Engine({
+ config:__dirname + '/config/dev.json'
+ });
+
+ var script = fs.readFileSync(__dirname + '/mock/redirect-rel.ql', 'UTF-8');
+
+ engine.exec({
+ script:script,
+ cb:function (err, result) {
+ try {
+ if (err) {
+ console.log(err.stack || err);
+ test.ok(false);
+ } else {
+ test.ok(result && result.body.id === "42");
+ }
+ test.done();
+ }
+ finally {
+ for (var i = 0; i < servers.length; i++) {
+ servers[i].instance.close();
+ }
+ }
+ }
+ });
+ },
+ 'bad-location-header':function (test) {
+ var servers = servers_no_location;
+
+ // Special case: need to create bad server
+ servers[0].instance = http.createServer(function (req, res) {
+ res.writeHead(servers[0].status, { 'Location': 'BadLocationURI' });
+ res.end();
+ }).listen(servers[0].port, servers[0].host);
+
+ // create the final server, the one that's going to return data
+ servers[1].instance = http.createServer(function (req, res) {
+ var file = __dirname + '/mock' + req.url;
+ var stat = fs.statSync(file);
+ res.writeHead(200, {
+ 'Content-Type':file.indexOf('.xml') >= 0 ? 'application/xml' : 'application/json',
+ 'Content-Length':stat.size
+ });
+ var readStream = fs.createReadStream(file);
+ util.pump(readStream, res, function (e) {
+ if (e) {
+ console.log(e.stack || e);
+ }
+ res.end();
+ });
+ }).listen(servers[1].port, servers[1].host);
+
+ var engine = new Engine({
+ config:__dirname + '/config/dev.json'
+ });
+
+ var script = fs.readFileSync(__dirname + '/mock/redirect.ql', 'UTF-8');
+
+ engine.exec({
+ script:script,
+ cb:function (err, result) {
+ try {
+ if (!err) {
+ test.ok(false, "Error expected.");
+ } else {
+ test.ok(err.status === 502);
+ }
+ test.done();
+ }
+ finally {
+ for (var i = 0; i < servers.length; i++) {
+ servers[i].instance.close();
+ }
+ }
+ }
+ });
+ },
+ 'no-location-header':function (test) {
+ var servers = servers_no_location;
+
+ // Special case: need to create bad server
+ servers[0].instance = http.createServer(function (req, res) {
+ var location = protocol + '://' + servers[1].host + ':' + servers[1].port + req.url;
+ res.writeHead(servers[0].status, { 'WrongLocationHeader': location });
+ res.end();
+ }).listen(servers[0].port, servers[0].host);
+
+ // create the final server, the one that's going to return data
+ servers[1].instance = http.createServer(function (req, res) {
+ var file = __dirname + '/mock' + req.url;
+ var stat = fs.statSync(file);
+ res.writeHead(200, {
+ 'Content-Type':file.indexOf('.xml') >= 0 ? 'application/xml' : 'application/json',
+ 'Content-Length':stat.size
+ });
+ var readStream = fs.createReadStream(file);
+ util.pump(readStream, res, function (e) {
+ if (e) {
+ console.log(e.stack || e);
+ }
+ res.end();
+ });
+ }).listen(servers[1].port, servers[1].host);
+
+ var engine = new Engine({
+ config:__dirname + '/config/dev.json'
+ });
+
+ var script = fs.readFileSync(__dirname + '/mock/redirect.ql', 'UTF-8');
+
+ engine.exec({
+ script:script,
+ cb:function (err, result) {
+ try {
+ if (!err) {
+ test.ok(false, "Error expected.");
+ } else {
+ test.ok(err.message === "Missing Location header in redirect");
+ }
+ test.done();
+ }
+ finally {
+ for (var i = 0; i < servers.length; i++) {
+ servers[i].instance.close();
+ }
+ }
+ }
+ });
+ },
+ 'status-305':function (test) {
+ var servers = servers_305;
+
+ setupServers(servers);
+
+ var engine = new Engine({
+ config:__dirname + '/config/dev.json'
+ });
+
+ var script = fs.readFileSync(__dirname + '/mock/redirect.ql', 'UTF-8');
+
+ engine.exec({
+ script:script,
+ cb:function (err, result) {
+ try {
+ if (!err) {
+ test.ok(false, "Error expected.");
+ } else {
+ test.ok(err.message === "Received status code 305 from downstream server");
+ }
+ test.done();
+ }
+ finally {
+ for (var i = 0; i < servers.length; i++) {
+ servers[i].instance.close();
+ }
+ }
+ }
+ });
+ }
+}
Something went wrong with that request. Please try again.