From bdafb4d6084e1df0a97899bcf12561f9db6d9ef5 Mon Sep 17 00:00:00 2001 From: Neeraj Shukla Date: Thu, 8 Oct 2020 21:50:43 +0530 Subject: [PATCH 01/44] reverse proxy server --- config/constants.js | 3 + src/requestsDebugger.js | 16 +++- src/server.js | 199 +++++++++++++++++++++++++++++++++++++++- test/server.test.js | 10 +- 4 files changed, 219 insertions(+), 9 deletions(-) diff --git a/config/constants.js b/config/constants.js index 346bd9c..5c10fce 100644 --- a/config/constants.js +++ b/config/constants.js @@ -1,4 +1,5 @@ module.exports.VERSION = '1.0.0'; +module.exports.HUB_HOST = 'hub-cloud.browserstack.com'; module.exports.HUB_STATUS_URL = 'http://hub-cloud.browserstack.com/wd/hub/status'; module.exports.RAILS_AUTOMATE = 'http://automate.browserstack.com'; module.exports.CONNECTIVITY_REQ_TIMEOUT = 30000; @@ -24,6 +25,7 @@ module.exports.LOGS = Object.freeze({ module.exports.RdGlobalConfig = { RETRY_DELAY: 1000, // in ms RD_HANDLER_PORT: process.env.NODE_ENV === 'test' ? 8787 : 9687, + RD_HANDLER_REVERSE_PROXY_PORT: process.env.NODE_ENV === 'test' ? 8788 : 9688, CLIENT_REQ_TIMEOUT: 260000, // in ms }; @@ -80,6 +82,7 @@ module.exports.STATIC_MESSAGES = Object.freeze({ CHECK_CONNECTIVITY: 'Checks : Checking Connectivity With BrowserStack', ERR_STARTING_TOOL: 'Error in starting Requests Debugger Tool Proxy: ', TOOL_STARTED_ON_PORT: 'Requests Debugger Tool Proxy Started on Port: ', + TOOL_REVERSE_PROXY_STARTED_ON_PORT: 'Requests Debugger Tool Reverse Proxy Started on Port: ', CPU_STATS_COLLECTED: 'Stats : Initial CPU Stats Collected', NETWORK_STATS_COLLECTED: 'Stats : Initial Network Stats Collected', MEMORY_STATS_COLLECTED: 'Stats : Initial Memory Stats Collected', diff --git a/src/requestsDebugger.js b/src/requestsDebugger.js index 765e76e..15271df 100644 --- a/src/requestsDebugger.js +++ b/src/requestsDebugger.js @@ -11,7 +11,7 @@ var RdGlobalConfig = constants.RdGlobalConfig; var STATIC_MESSAGES = constants.STATIC_MESSAGES; var CommandLineManager = require('./commandLine'); var ConnectivityChecker = require('./connectivity'); -var RdHandler = require('./server'); +var server = require('./server'); var StatsFactory = require('./stats/statsFactory'); var LogManager = require('./logger'); var fs = require('fs'); @@ -139,14 +139,24 @@ var RdTool = { console.log(Utils.formatAndBeautifyLine(STATIC_MESSAGES.CONNECTIVITY_CHECKS_DONE, '', '-', 60, true)); }); - RdHandler.startProxy(RdGlobalConfig.RD_HANDLER_PORT, function (err, result) { + server.RdHandler.startProxy(RdGlobalConfig.RD_HANDLER_PORT, function (err, result) { if (err) { console.log(STATIC_MESSAGES.ERR_STARTING_TOOL, err); - console.log('Exiting the Tool...'); + console.log('Exiting the Proxy...'); process.exit(1); } console.log(Utils.formatAndBeautifyLine(STATIC_MESSAGES.TOOL_STARTED_ON_PORT + result, '', '-', 60, true)); }); + + server.RdReverseProxyHandler.startServer(RdGlobalConfig.RD_HANDLER_REVERSE_PROXY_PORT, function (err, result) { + if (err) { + console.log(STATIC_MESSAGES.ERR_STARTING_TOOL, err); + console.log('Exiting the Reverse Proxy...'); + process.exit(1); + } + console.log(Utils.formatAndBeautifyLine(STATIC_MESSAGES.TOOL_REVERSE_PROXY_STARTED_ON_PORT + result, '', '-', 60, true)); + }); + } }; diff --git a/src/server.js b/src/server.js index 451f326..1be825c 100644 --- a/src/server.js +++ b/src/server.js @@ -208,4 +208,201 @@ var RdHandler = { } }; -module.exports = RdHandler; + +var RdReverseProxyHandler = { + + _requestCounter: 0, + + /** + * Generates the request options template for firing requests based on + * whether the user had provided any proxy input or not. + */ + generatorForRequestOptionsObject: function () { + RdReverseProxyHandler._reqObjTemplate = { + method: null, + headers: {}, + host: null, + port: null, + path: null + }; + + if (RdGlobalConfig.proxy) { + RdReverseProxyHandler._reqObjTemplate.host = RdGlobalConfig.proxy.host; + RdReverseProxyHandler._reqObjTemplate.port = RdGlobalConfig.proxy.port; + + if (RdGlobalConfig.proxy.username && RdGlobalConfig.proxy.password) { + RdReverseProxyHandler._reqObjTemplate.headers['Proxy-Authorization'] = Utils.proxyAuthToBase64(RdGlobalConfig.proxy); + } + + /** + * Sets the internal method to generate request options if external/upstream + * proxy exists + * @param {http.IncomingMessage} clientRequest + * @returns {Object} + */ + RdReverseProxyHandler._generateRequestOptions = function (clientRequest) { + var parsedClientUrl = url.parse(clientRequest.url); + var headersCopy = Object.assign({}, clientRequest.headers, RdReverseProxyHandler._reqObjTemplate.headers); + var requestOptions = Object.assign({}, RdReverseProxyHandler._reqObjTemplate); + requestOptions.path = parsedClientUrl.href; + requestOptions.method = clientRequest.method; + requestOptions.headers = headersCopy; + return requestOptions; + }; + } else { + + /** + * Sets the internal method to generate request options if external/upstream proxy + * doesn't exists + * @param {http.IncomingMessage} clientRequest + * @returns {Object} + */ + RdReverseProxyHandler._generateRequestOptions = function (clientRequest) { + var parsedClientUrl = url.parse(clientRequest.url); + var requestOptions = Object.assign({}, RdReverseProxyHandler._reqObjTemplate); + requestOptions.host = constants.HUB_HOST; + requestOptions.port = 80; + requestOptions.path = parsedClientUrl.path; + requestOptions.method = clientRequest.method; + requestOptions.headers = clientRequest.headers; + if (parsedClientUrl.auth) { + requestOptions.headers['authorization'] = Utils.proxyAuthToBase64(parsedClientUrl.auth); + } + return requestOptions; + }; + } + }, + + /** + * Frames the error response based on the type of request. + * i.e., if its a request originating for Hub, the response + * is in the format which the client binding would understand. + * @param {Object} parsedRequest + * @param {String} errorMessage + */ + _frameErrorResponse: function (parsedRequest, errorMessage) { + errorMessage += '. ' + constants.STATIC_MESSAGES.REQ_FAILED_MSG; + var parseSessionId = parsedRequest.path.match(/\/wd\/hub\/session\/([a-z0-9]+)\/*/); + if (parseSessionId) { + var sessionId = parseSessionId[1]; + return { + data: { + sessionId: sessionId, + status: 13, + value: { + message: errorMessage, + error: constants.STATIC_MESSAGES.REQ_FAILED_MSG + }, + state: 'error' + }, + statusCode: constants.CUSTOM_ERROR_RESPONSE_CODE + }; + } else { + return { + data: { + message: errorMessage, + error: constants.STATIC_MESSAGES.REQ_FAILED_MSG + }, + statusCode: constants.CUSTOM_ERROR_RESPONSE_CODE + }; + } + }, + + /** + * Handler for incoming requests to Requests Debugger Tool proxy server. + * @param {http.IncomingMessage} clientRequest + * @param {http.ServerResponse} clientResponse + */ + requestHandler: function (clientRequest, clientResponse) { + clientRequest.id = ++RdReverseProxyHandler._requestCounter + '::' + uuidv4(); + + var request = { + method: clientRequest.method, + url: clientRequest.url, + headers: clientRequest.headers, + data: [] + }; + + RdGlobalConfig.reqLogger.info(constants.TOPICS.CLIENT_REQUEST_START, request.method + ' ' + request.url, + false, { + headers: request.headers + }, + clientRequest.id); + + var furtherRequestOptions = RdReverseProxyHandler._generateRequestOptions(clientRequest); + + var paramsForRequest = { + request: request, + furtherRequestOptions: furtherRequestOptions + }; + + ReqLib.call(paramsForRequest, clientRequest) + .then(function (response) { + RdGlobalConfig.reqLogger.info(constants.TOPICS.CLIENT_RESPONSE_END, clientRequest.method + ' ' + clientRequest.url + ', Status Code: ' + response.statusCode, + false, { + data: response.data, + headers: response.headers, + }, + clientRequest.id); + + clientResponse.writeHead(response.statusCode, response.headers); + clientResponse.end(response.data); + }) + .catch(function (err) { + RdGlobalConfig.reqLogger.error(err.customTopic || constants.TOPICS.UNEXPECTED_ERROR, clientRequest.method + ' ' + clientRequest.url, + false, { + errorMessage: err.message.toString() + }, + clientRequest.id); + + var errorResponse = RdReverseProxyHandler._frameErrorResponse(furtherRequestOptions, err.message.toString()); + RdGlobalConfig.reqLogger.error(constants.TOPICS.CLIENT_RESPONSE_END, clientRequest.method + ' ' + clientRequest.url + ', Status Code: ' + errorResponse.statusCode, + false, + errorResponse.data, + clientRequest.id); + + clientResponse.writeHead(errorResponse.statusCode); + clientResponse.end(JSON.stringify(errorResponse.data)); + }); + }, + + /** + * Starts the reverse proxy server on the given port + * @param {String|Number} port + * @param {Function} callback + */ + startServer: function (port, callback) { + try { + RdReverseProxyHandler.generatorForRequestOptionsObject(); + RdReverseProxyHandler.reverseProxyServer = http.createServer(RdReverseProxyHandler.requestHandler); + RdReverseProxyHandler.reverseProxyServer.listen(port); + RdReverseProxyHandler.reverseProxyServer.on('listening', function () { + callback(null, port); + }); + RdReverseProxyHandler.reverseProxyServer.on('error', function (err) { + callback(err.toString(), null); + }); + } catch (e) { + callback(e.toString(), null); + } + }, + + /** + * Stops the currently running reverse proxy server + * @param {Function} callback + */ + stopServer: function (callback) { + try { + if (RdReverseProxyHandler.reverseProxyServer) { + RdReverseProxyHandler.reverseProxyServer.close(); + RdReverseProxyHandler.reverseProxyServer = null; + } + callback(null, true); + } catch (e) { + callback(e.toString(), null); + } + } +}; + +module.exports.RdHandler = RdHandler; +module.exports.RdReverseProxyHandler = RdReverseProxyHandler; diff --git a/test/server.test.js b/test/server.test.js index 5307bd7..56960d8 100644 --- a/test/server.test.js +++ b/test/server.test.js @@ -1,7 +1,7 @@ var constants = require('../config/constants'); var RdGlobalConfig = constants.RdGlobalConfig; var nock = require('nock'); -var RdHandler = require('../src/server'); +var server = require('../src/server'); var http = require('http'); var assert = require('chai').assert; var testHelper = require('./testHelper'); @@ -15,7 +15,7 @@ describe('RdHandler', function () { testHelper.initializeDummyLoggers(); testHelper.initializeDummyHandlers(); - RdHandler.startProxy(RdGlobalConfig.RD_HANDLER_PORT, function (port) { + server.RdHandler.startProxy(RdGlobalConfig.RD_HANDLER_PORT, function (port) { console.log('Test Network Utility Proxy Started on Port: ', port); done(); }); @@ -23,7 +23,7 @@ describe('RdHandler', function () { after(function (done) { this.timeout = 5000; - RdHandler.stopProxy(function () { + server.RdHandler.stopProxy(function () { done(); }); testHelper.deleteLoggers(); @@ -61,7 +61,7 @@ describe('RdHandler', function () { this.timeout = 5000; testHelper.initializeDummyProxy(); testHelper.nockProxyUrl(RdGlobalConfig.proxy, 'http', 'hub', null, 200); - RdHandler.generatorForRequestOptionsObject(); + server.RdHandler.generatorForRequestOptionsObject(); var reqOptions = { method: 'GET', host: 'localhost', @@ -92,7 +92,7 @@ describe('RdHandler', function () { for (var i = 0; i <= constants.MAX_RETRIES; i++) { testHelper.nockGetRequestWithError(constants.HUB_STATUS_URL, 'http'); } - RdHandler.generatorForRequestOptionsObject(); + server.RdHandler.generatorForRequestOptionsObject(); var reqOptions = { method: 'GET', host: 'localhost', From 3af63abe77522fc3a522e56eacee5b7ada514dd5 Mon Sep 17 00:00:00 2001 From: Neeraj Shukla Date: Fri, 9 Oct 2020 14:15:42 +0530 Subject: [PATCH 02/44] fix for proxy host --- src/server.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/server.js b/src/server.js index 1be825c..7f9e386 100644 --- a/src/server.js +++ b/src/server.js @@ -244,6 +244,8 @@ var RdReverseProxyHandler = { var parsedClientUrl = url.parse(clientRequest.url); var headersCopy = Object.assign({}, clientRequest.headers, RdReverseProxyHandler._reqObjTemplate.headers); var requestOptions = Object.assign({}, RdReverseProxyHandler._reqObjTemplate); + requestOptions.host = constants.HUB_HOST; + requestOptions.port = 80; requestOptions.path = parsedClientUrl.href; requestOptions.method = clientRequest.method; requestOptions.headers = headersCopy; From b5582c450cc4dbeb68be6394922fece1f9fcd680 Mon Sep 17 00:00:00 2001 From: Neeraj Shukla Date: Fri, 9 Oct 2020 14:49:17 +0530 Subject: [PATCH 03/44] refactor --- src/proxy.js | 211 +++++++++++++++++++++ src/requestsDebugger.js | 7 +- src/reverseProxy.js | 213 +++++++++++++++++++++ src/server.js | 410 ---------------------------------------- test/proxy.test.js | 120 ++++++++++++ 5 files changed, 548 insertions(+), 413 deletions(-) create mode 100644 src/proxy.js create mode 100644 src/reverseProxy.js delete mode 100644 src/server.js create mode 100644 test/proxy.test.js diff --git a/src/proxy.js b/src/proxy.js new file mode 100644 index 0000000..32bf95c --- /dev/null +++ b/src/proxy.js @@ -0,0 +1,211 @@ +/** + * Proxy Server to Intercept the client's requests and handle them on their behalf. + * Initiates stats and connectivity checks when a requests fails. + * It also responds in selenium understandable error when a request fails + * at tool. + */ + +var http = require('http'); +var url = require('url'); +var uuidv4 = require('uuid/v4'); +var Utils = require('./utils'); +var constants = require('../config/constants'); +var ReqLib = require('./requestLib'); +var RdGlobalConfig = constants.RdGlobalConfig; + +var RdHandler = { + + _requestCounter: 0, + + /** + * Generates the request options template for firing requests based on + * whether the user had provided any proxy input or not. + */ + generatorForRequestOptionsObject: function () { + RdHandler._reqObjTemplate = { + method: null, + headers: {}, + host: null, + port: null, + path: null + }; + + if (RdGlobalConfig.proxy) { + RdHandler._reqObjTemplate.host = RdGlobalConfig.proxy.host; + RdHandler._reqObjTemplate.port = RdGlobalConfig.proxy.port; + + if (RdGlobalConfig.proxy.username && RdGlobalConfig.proxy.password) { + RdHandler._reqObjTemplate.headers['Proxy-Authorization'] = Utils.proxyAuthToBase64(RdGlobalConfig.proxy); + } + + /** + * Sets the internal method to generate request options if external/upstream + * proxy exists + * @param {http.IncomingMessage} clientRequest + * @returns {Object} + */ + RdHandler._generateRequestOptions = function (clientRequest) { + var parsedClientUrl = url.parse(clientRequest.url); + var headersCopy = Object.assign({}, clientRequest.headers, RdHandler._reqObjTemplate.headers); + var requestOptions = Object.assign({}, RdHandler._reqObjTemplate); + requestOptions.path = parsedClientUrl.href; + requestOptions.method = clientRequest.method; + requestOptions.headers = headersCopy; + return requestOptions; + }; + } else { + + /** + * Sets the internal method to generate request options if external/upstream proxy + * doesn't exists + * @param {http.IncomingMessage} clientRequest + * @returns {Object} + */ + RdHandler._generateRequestOptions = function (clientRequest) { + var parsedClientUrl = url.parse(clientRequest.url); + var requestOptions = Object.assign({}, RdHandler._reqObjTemplate); + requestOptions.host = parsedClientUrl.hostname; + requestOptions.port = parsedClientUrl.port || 80; + requestOptions.path = parsedClientUrl.path; + requestOptions.method = clientRequest.method; + requestOptions.headers = clientRequest.headers; + if (parsedClientUrl.auth) { + requestOptions.headers['authorization'] = Utils.proxyAuthToBase64(parsedClientUrl.auth); + } + return requestOptions; + }; + } + }, + + /** + * Frames the error response based on the type of request. + * i.e., if its a request originating for Hub, the response + * is in the format which the client binding would understand. + * @param {Object} parsedRequest + * @param {String} errorMessage + */ + _frameErrorResponse: function (parsedRequest, errorMessage) { + errorMessage += '. ' + constants.STATIC_MESSAGES.REQ_FAILED_MSG; + var parseSessionId = parsedRequest.path.match(/\/wd\/hub\/session\/([a-z0-9]+)\/*/); + if (parseSessionId) { + var sessionId = parseSessionId[1]; + return { + data: { + sessionId: sessionId, + status: 13, + value: { + message: errorMessage, + error: constants.STATIC_MESSAGES.REQ_FAILED_MSG + }, + state: 'error' + }, + statusCode: constants.CUSTOM_ERROR_RESPONSE_CODE + }; + } else { + return { + data: { + message: errorMessage, + error: constants.STATIC_MESSAGES.REQ_FAILED_MSG + }, + statusCode: constants.CUSTOM_ERROR_RESPONSE_CODE + }; + } + }, + + /** + * Handler for incoming requests to Requests Debugger Tool proxy server. + * @param {http.IncomingMessage} clientRequest + * @param {http.ServerResponse} clientResponse + */ + requestHandler: function (clientRequest, clientResponse) { + clientRequest.id = ++RdHandler._requestCounter + '::' + uuidv4(); + + var request = { + method: clientRequest.method, + url: clientRequest.url, + headers: clientRequest.headers, + data: [] + }; + + RdGlobalConfig.reqLogger.info(constants.TOPICS.CLIENT_REQUEST_START, request.method + ' ' + request.url, + false, { + headers: request.headers + }, + clientRequest.id); + + var furtherRequestOptions = RdHandler._generateRequestOptions(clientRequest); + + var paramsForRequest = { + request: request, + furtherRequestOptions: furtherRequestOptions + }; + + ReqLib.call(paramsForRequest, clientRequest) + .then(function (response) { + RdGlobalConfig.reqLogger.info(constants.TOPICS.CLIENT_RESPONSE_END, clientRequest.method + ' ' + clientRequest.url + ', Status Code: ' + response.statusCode, + false, { + data: response.data, + headers: response.headers, + }, + clientRequest.id); + + clientResponse.writeHead(response.statusCode, response.headers); + clientResponse.end(response.data); + }) + .catch(function (err) { + RdGlobalConfig.reqLogger.error(err.customTopic || constants.TOPICS.UNEXPECTED_ERROR, clientRequest.method + ' ' + clientRequest.url, + false, { + errorMessage: err.message.toString() + }, + clientRequest.id); + + var errorResponse = RdHandler._frameErrorResponse(furtherRequestOptions, err.message.toString()); + RdGlobalConfig.reqLogger.error(constants.TOPICS.CLIENT_RESPONSE_END, clientRequest.method + ' ' + clientRequest.url + ', Status Code: ' + errorResponse.statusCode, + false, + errorResponse.data, + clientRequest.id); + + clientResponse.writeHead(errorResponse.statusCode); + clientResponse.end(JSON.stringify(errorResponse.data)); + }); + }, + + /** + * Starts the proxy server on the given port + * @param {String|Number} port + * @param {Function} callback + */ + startServer: function (port, callback) { + try { + RdHandler.generatorForRequestOptionsObject(); + RdHandler.server = http.createServer(RdHandler.requestHandler); + RdHandler.server.listen(port); + RdHandler.server.on('listening', function () { + callback(null, port); + }); + RdHandler.server.on('error', function (err) { + callback(err.toString(), null); + }); + } catch (e) { + callback(e.toString(), null); + } + }, + + /** + * Stops the currently running proxy server + * @param {Function} callback + */ + stopServer: function (callback) { + try { + if (RdHandler.server) { + RdHandler.server.close(); + RdHandler.server = null; + } + callback(null, true); + } catch (e) { + callback(e.toString(), null); + } + } +}; + +module.exports.RdHandler = RdHandler; diff --git a/src/requestsDebugger.js b/src/requestsDebugger.js index 15271df..eb62c00 100644 --- a/src/requestsDebugger.js +++ b/src/requestsDebugger.js @@ -11,7 +11,8 @@ var RdGlobalConfig = constants.RdGlobalConfig; var STATIC_MESSAGES = constants.STATIC_MESSAGES; var CommandLineManager = require('./commandLine'); var ConnectivityChecker = require('./connectivity'); -var server = require('./server'); +var proxy = require('./proxy'); +var reverseProxy = require('./reverseProxy'); var StatsFactory = require('./stats/statsFactory'); var LogManager = require('./logger'); var fs = require('fs'); @@ -139,7 +140,7 @@ var RdTool = { console.log(Utils.formatAndBeautifyLine(STATIC_MESSAGES.CONNECTIVITY_CHECKS_DONE, '', '-', 60, true)); }); - server.RdHandler.startProxy(RdGlobalConfig.RD_HANDLER_PORT, function (err, result) { + proxy.RdHandler.startServer(RdGlobalConfig.RD_HANDLER_PORT, function (err, result) { if (err) { console.log(STATIC_MESSAGES.ERR_STARTING_TOOL, err); console.log('Exiting the Proxy...'); @@ -148,7 +149,7 @@ var RdTool = { console.log(Utils.formatAndBeautifyLine(STATIC_MESSAGES.TOOL_STARTED_ON_PORT + result, '', '-', 60, true)); }); - server.RdReverseProxyHandler.startServer(RdGlobalConfig.RD_HANDLER_REVERSE_PROXY_PORT, function (err, result) { + reverseProxy.RdHandler.startServer(RdGlobalConfig.RD_HANDLER_REVERSE_PROXY_PORT, function (err, result) { if (err) { console.log(STATIC_MESSAGES.ERR_STARTING_TOOL, err); console.log('Exiting the Reverse Proxy...'); diff --git a/src/reverseProxy.js b/src/reverseProxy.js new file mode 100644 index 0000000..08bfe3a --- /dev/null +++ b/src/reverseProxy.js @@ -0,0 +1,213 @@ +/** + * Reverse Proxy Server to Intercept the client's requests and handle them on their behalf. + * Initiates stats and connectivity checks when a requests fails. + * It also responds in selenium understandable error when a request fails + * at tool. + */ + +var http = require('http'); +var url = require('url'); +var uuidv4 = require('uuid/v4'); +var Utils = require('./utils'); +var constants = require('../config/constants'); +var ReqLib = require('./requestLib'); +var RdGlobalConfig = constants.RdGlobalConfig; + +var RdHandler = { + + _requestCounter: 0, + + /** + * Generates the request options template for firing requests based on + * whether the user had provided any proxy input or not. + */ + generatorForRequestOptionsObject: function () { + RdHandler._reqObjTemplate = { + method: null, + headers: {}, + host: null, + port: null, + path: null + }; + + if (RdGlobalConfig.proxy) { + RdHandler._reqObjTemplate.host = RdGlobalConfig.proxy.host; + RdHandler._reqObjTemplate.port = RdGlobalConfig.proxy.port; + + if (RdGlobalConfig.proxy.username && RdGlobalConfig.proxy.password) { + RdHandler._reqObjTemplate.headers['Proxy-Authorization'] = Utils.proxyAuthToBase64(RdGlobalConfig.proxy); + } + + /** + * Sets the internal method to generate request options if external/upstream + * proxy exists + * @param {http.IncomingMessage} clientRequest + * @returns {Object} + */ + RdHandler._generateRequestOptions = function (clientRequest) { + var parsedClientUrl = url.parse(clientRequest.url); + var headersCopy = Object.assign({}, clientRequest.headers, RdHandler._reqObjTemplate.headers); + var requestOptions = Object.assign({}, RdHandler._reqObjTemplate); + requestOptions.host = constants.HUB_HOST; + requestOptions.port = 80; + requestOptions.path = parsedClientUrl.href; + requestOptions.method = clientRequest.method; + requestOptions.headers = headersCopy; + return requestOptions; + }; + } else { + + /** + * Sets the internal method to generate request options if external/upstream proxy + * doesn't exists + * @param {http.IncomingMessage} clientRequest + * @returns {Object} + */ + RdHandler._generateRequestOptions = function (clientRequest) { + var parsedClientUrl = url.parse(clientRequest.url); + var requestOptions = Object.assign({}, RdHandler._reqObjTemplate); + requestOptions.host = constants.HUB_HOST; + requestOptions.port = 80; + requestOptions.path = parsedClientUrl.path; + requestOptions.method = clientRequest.method; + requestOptions.headers = clientRequest.headers; + if (parsedClientUrl.auth) { + requestOptions.headers['authorization'] = Utils.proxyAuthToBase64(parsedClientUrl.auth); + } + return requestOptions; + }; + } + }, + + /** + * Frames the error response based on the type of request. + * i.e., if its a request originating for Hub, the response + * is in the format which the client binding would understand. + * @param {Object} parsedRequest + * @param {String} errorMessage + */ + _frameErrorResponse: function (parsedRequest, errorMessage) { + errorMessage += '. ' + constants.STATIC_MESSAGES.REQ_FAILED_MSG; + var parseSessionId = parsedRequest.path.match(/\/wd\/hub\/session\/([a-z0-9]+)\/*/); + if (parseSessionId) { + var sessionId = parseSessionId[1]; + return { + data: { + sessionId: sessionId, + status: 13, + value: { + message: errorMessage, + error: constants.STATIC_MESSAGES.REQ_FAILED_MSG + }, + state: 'error' + }, + statusCode: constants.CUSTOM_ERROR_RESPONSE_CODE + }; + } else { + return { + data: { + message: errorMessage, + error: constants.STATIC_MESSAGES.REQ_FAILED_MSG + }, + statusCode: constants.CUSTOM_ERROR_RESPONSE_CODE + }; + } + }, + + /** + * Handler for incoming requests to Requests Debugger Tool reverse proxy server. + * @param {http.IncomingMessage} clientRequest + * @param {http.ServerResponse} clientResponse + */ + requestHandler: function (clientRequest, clientResponse) { + clientRequest.id = ++RdHandler._requestCounter + '::' + uuidv4(); + + var request = { + method: clientRequest.method, + url: clientRequest.url, + headers: clientRequest.headers, + data: [] + }; + + RdGlobalConfig.reqLogger.info(constants.TOPICS.CLIENT_REQUEST_START, request.method + ' ' + request.url, + false, { + headers: request.headers + }, + clientRequest.id); + + var furtherRequestOptions = RdHandler._generateRequestOptions(clientRequest); + + var paramsForRequest = { + request: request, + furtherRequestOptions: furtherRequestOptions + }; + + ReqLib.call(paramsForRequest, clientRequest) + .then(function (response) { + RdGlobalConfig.reqLogger.info(constants.TOPICS.CLIENT_RESPONSE_END, clientRequest.method + ' ' + clientRequest.url + ', Status Code: ' + response.statusCode, + false, { + data: response.data, + headers: response.headers, + }, + clientRequest.id); + + clientResponse.writeHead(response.statusCode, response.headers); + clientResponse.end(response.data); + }) + .catch(function (err) { + RdGlobalConfig.reqLogger.error(err.customTopic || constants.TOPICS.UNEXPECTED_ERROR, clientRequest.method + ' ' + clientRequest.url, + false, { + errorMessage: err.message.toString() + }, + clientRequest.id); + + var errorResponse = RdHandler._frameErrorResponse(furtherRequestOptions, err.message.toString()); + RdGlobalConfig.reqLogger.error(constants.TOPICS.CLIENT_RESPONSE_END, clientRequest.method + ' ' + clientRequest.url + ', Status Code: ' + errorResponse.statusCode, + false, + errorResponse.data, + clientRequest.id); + + clientResponse.writeHead(errorResponse.statusCode); + clientResponse.end(JSON.stringify(errorResponse.data)); + }); + }, + + /** + * Starts the reverse proxy server on the given port + * @param {String|Number} port + * @param {Function} callback + */ + startServer: function (port, callback) { + try { + RdHandler.generatorForRequestOptionsObject(); + RdHandler.server = http.createServer(RdHandler.requestHandler); + RdHandler.server.listen(port); + RdHandler.server.on('listening', function () { + callback(null, port); + }); + RdHandler.server.on('error', function (err) { + callback(err.toString(), null); + }); + } catch (e) { + callback(e.toString(), null); + } + }, + + /** + * Stops the currently running reverse proxy server + * @param {Function} callback + */ + stopServer: function (callback) { + try { + if (RdHandler.server) { + RdHandler.server.close(); + RdHandler.server = null; + } + callback(null, true); + } catch (e) { + callback(e.toString(), null); + } + } +}; + +module.exports.RdHandler = RdHandler; diff --git a/src/server.js b/src/server.js deleted file mode 100644 index 7f9e386..0000000 --- a/src/server.js +++ /dev/null @@ -1,410 +0,0 @@ -/** - * Server to Intercept the client's requests and handle them on their behalf. - * Initiates stats and connectivity checks when a requests fails. - * It also responds in selenium understandable error when a request fails - * at tool. - */ - -var http = require('http'); -var url = require('url'); -var uuidv4 = require('uuid/v4'); -var Utils = require('./utils'); -var constants = require('../config/constants'); -var ReqLib = require('./requestLib'); -var RdGlobalConfig = constants.RdGlobalConfig; - -var RdHandler = { - - _requestCounter: 0, - - /** - * Generates the request options template for firing requests based on - * whether the user had provided any proxy input or not. - */ - generatorForRequestOptionsObject: function () { - RdHandler._reqObjTemplate = { - method: null, - headers: {}, - host: null, - port: null, - path: null - }; - - if (RdGlobalConfig.proxy) { - RdHandler._reqObjTemplate.host = RdGlobalConfig.proxy.host; - RdHandler._reqObjTemplate.port = RdGlobalConfig.proxy.port; - - if (RdGlobalConfig.proxy.username && RdGlobalConfig.proxy.password) { - RdHandler._reqObjTemplate.headers['Proxy-Authorization'] = Utils.proxyAuthToBase64(RdGlobalConfig.proxy); - } - - /** - * Sets the internal method to generate request options if external/upstream - * proxy exists - * @param {http.IncomingMessage} clientRequest - * @returns {Object} - */ - RdHandler._generateRequestOptions = function (clientRequest) { - var parsedClientUrl = url.parse(clientRequest.url); - var headersCopy = Object.assign({}, clientRequest.headers, RdHandler._reqObjTemplate.headers); - var requestOptions = Object.assign({}, RdHandler._reqObjTemplate); - requestOptions.path = parsedClientUrl.href; - requestOptions.method = clientRequest.method; - requestOptions.headers = headersCopy; - return requestOptions; - }; - } else { - - /** - * Sets the internal method to generate request options if external/upstream proxy - * doesn't exists - * @param {http.IncomingMessage} clientRequest - * @returns {Object} - */ - RdHandler._generateRequestOptions = function (clientRequest) { - var parsedClientUrl = url.parse(clientRequest.url); - var requestOptions = Object.assign({}, RdHandler._reqObjTemplate); - requestOptions.host = parsedClientUrl.hostname; - requestOptions.port = parsedClientUrl.port || 80; - requestOptions.path = parsedClientUrl.path; - requestOptions.method = clientRequest.method; - requestOptions.headers = clientRequest.headers; - if (parsedClientUrl.auth) { - requestOptions.headers['authorization'] = Utils.proxyAuthToBase64(parsedClientUrl.auth); - } - return requestOptions; - }; - } - }, - - /** - * Frames the error response based on the type of request. - * i.e., if its a request originating for Hub, the response - * is in the format which the client binding would understand. - * @param {Object} parsedRequest - * @param {String} errorMessage - */ - _frameErrorResponse: function (parsedRequest, errorMessage) { - errorMessage += '. ' + constants.STATIC_MESSAGES.REQ_FAILED_MSG; - var parseSessionId = parsedRequest.path.match(/\/wd\/hub\/session\/([a-z0-9]+)\/*/); - if (parseSessionId) { - var sessionId = parseSessionId[1]; - return { - data: { - sessionId: sessionId, - status: 13, - value: { - message: errorMessage, - error: constants.STATIC_MESSAGES.REQ_FAILED_MSG - }, - state: 'error' - }, - statusCode: constants.CUSTOM_ERROR_RESPONSE_CODE - }; - } else { - return { - data: { - message: errorMessage, - error: constants.STATIC_MESSAGES.REQ_FAILED_MSG - }, - statusCode: constants.CUSTOM_ERROR_RESPONSE_CODE - }; - } - }, - - /** - * Handler for incoming requests to Requests Debugger Tool proxy server. - * @param {http.IncomingMessage} clientRequest - * @param {http.ServerResponse} clientResponse - */ - requestHandler: function (clientRequest, clientResponse) { - clientRequest.id = ++RdHandler._requestCounter + '::' + uuidv4(); - - var request = { - method: clientRequest.method, - url: clientRequest.url, - headers: clientRequest.headers, - data: [] - }; - - RdGlobalConfig.reqLogger.info(constants.TOPICS.CLIENT_REQUEST_START, request.method + ' ' + request.url, - false, { - headers: request.headers - }, - clientRequest.id); - - var furtherRequestOptions = RdHandler._generateRequestOptions(clientRequest); - - var paramsForRequest = { - request: request, - furtherRequestOptions: furtherRequestOptions - }; - - ReqLib.call(paramsForRequest, clientRequest) - .then(function (response) { - RdGlobalConfig.reqLogger.info(constants.TOPICS.CLIENT_RESPONSE_END, clientRequest.method + ' ' + clientRequest.url + ', Status Code: ' + response.statusCode, - false, { - data: response.data, - headers: response.headers, - }, - clientRequest.id); - - clientResponse.writeHead(response.statusCode, response.headers); - clientResponse.end(response.data); - }) - .catch(function (err) { - RdGlobalConfig.reqLogger.error(err.customTopic || constants.TOPICS.UNEXPECTED_ERROR, clientRequest.method + ' ' + clientRequest.url, - false, { - errorMessage: err.message.toString() - }, - clientRequest.id); - - var errorResponse = RdHandler._frameErrorResponse(furtherRequestOptions, err.message.toString()); - RdGlobalConfig.reqLogger.error(constants.TOPICS.CLIENT_RESPONSE_END, clientRequest.method + ' ' + clientRequest.url + ', Status Code: ' + errorResponse.statusCode, - false, - errorResponse.data, - clientRequest.id); - - clientResponse.writeHead(errorResponse.statusCode); - clientResponse.end(JSON.stringify(errorResponse.data)); - }); - }, - - /** - * Starts the proxy server on the given port - * @param {String|Number} port - * @param {Function} callback - */ - startProxy: function (port, callback) { - try { - RdHandler.generatorForRequestOptionsObject(); - RdHandler.server = http.createServer(RdHandler.requestHandler); - RdHandler.server.listen(port); - RdHandler.server.on('listening', function () { - callback(null, port); - }); - RdHandler.server.on('error', function (err) { - callback(err.toString(), null); - }); - } catch (e) { - callback(e.toString(), null); - } - }, - - /** - * Stops the currently running proxy server - * @param {Function} callback - */ - stopProxy: function (callback) { - try { - if (RdHandler.server) { - RdHandler.server.close(); - RdHandler.server = null; - } - callback(null, true); - } catch (e) { - callback(e.toString(), null); - } - } -}; - - -var RdReverseProxyHandler = { - - _requestCounter: 0, - - /** - * Generates the request options template for firing requests based on - * whether the user had provided any proxy input or not. - */ - generatorForRequestOptionsObject: function () { - RdReverseProxyHandler._reqObjTemplate = { - method: null, - headers: {}, - host: null, - port: null, - path: null - }; - - if (RdGlobalConfig.proxy) { - RdReverseProxyHandler._reqObjTemplate.host = RdGlobalConfig.proxy.host; - RdReverseProxyHandler._reqObjTemplate.port = RdGlobalConfig.proxy.port; - - if (RdGlobalConfig.proxy.username && RdGlobalConfig.proxy.password) { - RdReverseProxyHandler._reqObjTemplate.headers['Proxy-Authorization'] = Utils.proxyAuthToBase64(RdGlobalConfig.proxy); - } - - /** - * Sets the internal method to generate request options if external/upstream - * proxy exists - * @param {http.IncomingMessage} clientRequest - * @returns {Object} - */ - RdReverseProxyHandler._generateRequestOptions = function (clientRequest) { - var parsedClientUrl = url.parse(clientRequest.url); - var headersCopy = Object.assign({}, clientRequest.headers, RdReverseProxyHandler._reqObjTemplate.headers); - var requestOptions = Object.assign({}, RdReverseProxyHandler._reqObjTemplate); - requestOptions.host = constants.HUB_HOST; - requestOptions.port = 80; - requestOptions.path = parsedClientUrl.href; - requestOptions.method = clientRequest.method; - requestOptions.headers = headersCopy; - return requestOptions; - }; - } else { - - /** - * Sets the internal method to generate request options if external/upstream proxy - * doesn't exists - * @param {http.IncomingMessage} clientRequest - * @returns {Object} - */ - RdReverseProxyHandler._generateRequestOptions = function (clientRequest) { - var parsedClientUrl = url.parse(clientRequest.url); - var requestOptions = Object.assign({}, RdReverseProxyHandler._reqObjTemplate); - requestOptions.host = constants.HUB_HOST; - requestOptions.port = 80; - requestOptions.path = parsedClientUrl.path; - requestOptions.method = clientRequest.method; - requestOptions.headers = clientRequest.headers; - if (parsedClientUrl.auth) { - requestOptions.headers['authorization'] = Utils.proxyAuthToBase64(parsedClientUrl.auth); - } - return requestOptions; - }; - } - }, - - /** - * Frames the error response based on the type of request. - * i.e., if its a request originating for Hub, the response - * is in the format which the client binding would understand. - * @param {Object} parsedRequest - * @param {String} errorMessage - */ - _frameErrorResponse: function (parsedRequest, errorMessage) { - errorMessage += '. ' + constants.STATIC_MESSAGES.REQ_FAILED_MSG; - var parseSessionId = parsedRequest.path.match(/\/wd\/hub\/session\/([a-z0-9]+)\/*/); - if (parseSessionId) { - var sessionId = parseSessionId[1]; - return { - data: { - sessionId: sessionId, - status: 13, - value: { - message: errorMessage, - error: constants.STATIC_MESSAGES.REQ_FAILED_MSG - }, - state: 'error' - }, - statusCode: constants.CUSTOM_ERROR_RESPONSE_CODE - }; - } else { - return { - data: { - message: errorMessage, - error: constants.STATIC_MESSAGES.REQ_FAILED_MSG - }, - statusCode: constants.CUSTOM_ERROR_RESPONSE_CODE - }; - } - }, - - /** - * Handler for incoming requests to Requests Debugger Tool proxy server. - * @param {http.IncomingMessage} clientRequest - * @param {http.ServerResponse} clientResponse - */ - requestHandler: function (clientRequest, clientResponse) { - clientRequest.id = ++RdReverseProxyHandler._requestCounter + '::' + uuidv4(); - - var request = { - method: clientRequest.method, - url: clientRequest.url, - headers: clientRequest.headers, - data: [] - }; - - RdGlobalConfig.reqLogger.info(constants.TOPICS.CLIENT_REQUEST_START, request.method + ' ' + request.url, - false, { - headers: request.headers - }, - clientRequest.id); - - var furtherRequestOptions = RdReverseProxyHandler._generateRequestOptions(clientRequest); - - var paramsForRequest = { - request: request, - furtherRequestOptions: furtherRequestOptions - }; - - ReqLib.call(paramsForRequest, clientRequest) - .then(function (response) { - RdGlobalConfig.reqLogger.info(constants.TOPICS.CLIENT_RESPONSE_END, clientRequest.method + ' ' + clientRequest.url + ', Status Code: ' + response.statusCode, - false, { - data: response.data, - headers: response.headers, - }, - clientRequest.id); - - clientResponse.writeHead(response.statusCode, response.headers); - clientResponse.end(response.data); - }) - .catch(function (err) { - RdGlobalConfig.reqLogger.error(err.customTopic || constants.TOPICS.UNEXPECTED_ERROR, clientRequest.method + ' ' + clientRequest.url, - false, { - errorMessage: err.message.toString() - }, - clientRequest.id); - - var errorResponse = RdReverseProxyHandler._frameErrorResponse(furtherRequestOptions, err.message.toString()); - RdGlobalConfig.reqLogger.error(constants.TOPICS.CLIENT_RESPONSE_END, clientRequest.method + ' ' + clientRequest.url + ', Status Code: ' + errorResponse.statusCode, - false, - errorResponse.data, - clientRequest.id); - - clientResponse.writeHead(errorResponse.statusCode); - clientResponse.end(JSON.stringify(errorResponse.data)); - }); - }, - - /** - * Starts the reverse proxy server on the given port - * @param {String|Number} port - * @param {Function} callback - */ - startServer: function (port, callback) { - try { - RdReverseProxyHandler.generatorForRequestOptionsObject(); - RdReverseProxyHandler.reverseProxyServer = http.createServer(RdReverseProxyHandler.requestHandler); - RdReverseProxyHandler.reverseProxyServer.listen(port); - RdReverseProxyHandler.reverseProxyServer.on('listening', function () { - callback(null, port); - }); - RdReverseProxyHandler.reverseProxyServer.on('error', function (err) { - callback(err.toString(), null); - }); - } catch (e) { - callback(e.toString(), null); - } - }, - - /** - * Stops the currently running reverse proxy server - * @param {Function} callback - */ - stopServer: function (callback) { - try { - if (RdReverseProxyHandler.reverseProxyServer) { - RdReverseProxyHandler.reverseProxyServer.close(); - RdReverseProxyHandler.reverseProxyServer = null; - } - callback(null, true); - } catch (e) { - callback(e.toString(), null); - } - } -}; - -module.exports.RdHandler = RdHandler; -module.exports.RdReverseProxyHandler = RdReverseProxyHandler; diff --git a/test/proxy.test.js b/test/proxy.test.js new file mode 100644 index 0000000..dd8803d --- /dev/null +++ b/test/proxy.test.js @@ -0,0 +1,120 @@ +var constants = require('../config/constants'); +var RdGlobalConfig = constants.RdGlobalConfig; +var nock = require('nock'); +var proxy = require('../src/proxy'); +var http = require('http'); +var assert = require('chai').assert; +var testHelper = require('./testHelper'); + +describe('RdHandler', function () { + context('Proxy Server', function () { + + before(function (done) { + this.timeout = 5000; + testHelper.nockGetRequest(constants.HUB_STATUS_URL, 'http', null, 200); + testHelper.initializeDummyLoggers(); + testHelper.initializeDummyHandlers(); + + proxy.RdHandler.startServer(RdGlobalConfig.RD_HANDLER_PORT, function (port) { + console.log('Test Network Utility Proxy Started on Port: ', port); + done(); + }); + }); + + after(function (done) { + this.timeout = 5000; + proxy.RdHandler.stopServer(function () { + done(); + }); + testHelper.deleteLoggers(); + testHelper.deleteHandlers(); + nock.cleanAll(); + }); + + it('Requests on behalf of the client and returns the response', function (done) { + this.timeout = 5000; + var reqOptions = { + method: 'GET', + host: 'localhost', + port: RdGlobalConfig.RD_HANDLER_PORT, + headers: {}, + path: constants.HUB_STATUS_URL + }; + + var responseData = []; + var request = http.request(reqOptions, function (response) { + + response.on('data', function (chunk) { + responseData.push(chunk); + }); + + response.on('end', function () { + assert(Buffer.concat(responseData).toString() === '{"data":"value"}'); + done(); + }); + }); + + request.end(); + }); + + it('Requests on behalf of the client via external proxy and returns the response', function (done) { + this.timeout = 5000; + testHelper.initializeDummyProxy(); + testHelper.nockProxyUrl(RdGlobalConfig.proxy, 'http', 'hub', null, 200); + proxy.RdHandler.generatorForRequestOptionsObject(); + var reqOptions = { + method: 'GET', + host: 'localhost', + port: RdGlobalConfig.RD_HANDLER_PORT, + headers: {}, + path: constants.HUB_STATUS_URL + }; + + var responseData = []; + var request = http.request(reqOptions, function (response) { + + response.on('data', function (chunk) { + responseData.push(chunk); + }); + + response.on('end', function () { + assert(Buffer.concat(responseData).toString() === '{"data":"value"}'); + done(); + }); + }); + + request.end(); + testHelper.deleteProxy(); + }); + + it('Requests on behalf of the client via external proxy and returns the response even if request by tool fails', function (done) { + this.timeout = 5000; + for (var i = 0; i <= constants.MAX_RETRIES; i++) { + testHelper.nockGetRequestWithError(constants.HUB_STATUS_URL, 'http'); + } + proxy.RdHandler.generatorForRequestOptionsObject(); + var reqOptions = { + method: 'GET', + host: 'localhost', + port: RdGlobalConfig.RD_HANDLER_PORT, + headers: {}, + path: constants.HUB_STATUS_URL + }; + + var responseData = []; + var request = http.request(reqOptions, function (response) { + + response.on('data', function (chunk) { + responseData.push(chunk); + }); + + response.on('end', function () { + assert(Buffer.concat(responseData).toString() === '{"message":"Error: something terrible. Request Failed At Requests Debugger","error":"Request Failed At Requests Debugger"}'); + done(); + }); + }); + + request.end(); + }); + }); +}); From d93a2e01455f92e1d4c5c554f5c59451fcdf6711 Mon Sep 17 00:00:00 2001 From: Neeraj Shukla Date: Mon, 12 Oct 2020 11:29:39 +0530 Subject: [PATCH 04/44] mremoved --- test/server.test.js | 120 -------------------------------------------- 1 file changed, 120 deletions(-) delete mode 100644 test/server.test.js diff --git a/test/server.test.js b/test/server.test.js deleted file mode 100644 index 56960d8..0000000 --- a/test/server.test.js +++ /dev/null @@ -1,120 +0,0 @@ -var constants = require('../config/constants'); -var RdGlobalConfig = constants.RdGlobalConfig; -var nock = require('nock'); -var server = require('../src/server'); -var http = require('http'); -var assert = require('chai').assert; -var testHelper = require('./testHelper'); - -describe('RdHandler', function () { - context('Proxy Server', function () { - - before(function (done) { - this.timeout = 5000; - testHelper.nockGetRequest(constants.HUB_STATUS_URL, 'http', null, 200); - testHelper.initializeDummyLoggers(); - testHelper.initializeDummyHandlers(); - - server.RdHandler.startProxy(RdGlobalConfig.RD_HANDLER_PORT, function (port) { - console.log('Test Network Utility Proxy Started on Port: ', port); - done(); - }); - }); - - after(function (done) { - this.timeout = 5000; - server.RdHandler.stopProxy(function () { - done(); - }); - testHelper.deleteLoggers(); - testHelper.deleteHandlers(); - nock.cleanAll(); - }); - - it('Requests on behalf of the client and returns the response', function (done) { - this.timeout = 5000; - var reqOptions = { - method: 'GET', - host: 'localhost', - port: RdGlobalConfig.RD_HANDLER_PORT, - headers: {}, - path: constants.HUB_STATUS_URL - }; - - var responseData = []; - var request = http.request(reqOptions, function (response) { - - response.on('data', function (chunk) { - responseData.push(chunk); - }); - - response.on('end', function () { - assert(Buffer.concat(responseData).toString() === '{"data":"value"}'); - done(); - }); - }); - - request.end(); - }); - - it('Requests on behalf of the client via external proxy and returns the response', function (done) { - this.timeout = 5000; - testHelper.initializeDummyProxy(); - testHelper.nockProxyUrl(RdGlobalConfig.proxy, 'http', 'hub', null, 200); - server.RdHandler.generatorForRequestOptionsObject(); - var reqOptions = { - method: 'GET', - host: 'localhost', - port: RdGlobalConfig.RD_HANDLER_PORT, - headers: {}, - path: constants.HUB_STATUS_URL - }; - - var responseData = []; - var request = http.request(reqOptions, function (response) { - - response.on('data', function (chunk) { - responseData.push(chunk); - }); - - response.on('end', function () { - assert(Buffer.concat(responseData).toString() === '{"data":"value"}'); - done(); - }); - }); - - request.end(); - testHelper.deleteProxy(); - }); - - it('Requests on behalf of the client via external proxy and returns the response even if request by tool fails', function (done) { - this.timeout = 5000; - for (var i = 0; i <= constants.MAX_RETRIES; i++) { - testHelper.nockGetRequestWithError(constants.HUB_STATUS_URL, 'http'); - } - server.RdHandler.generatorForRequestOptionsObject(); - var reqOptions = { - method: 'GET', - host: 'localhost', - port: RdGlobalConfig.RD_HANDLER_PORT, - headers: {}, - path: constants.HUB_STATUS_URL - }; - - var responseData = []; - var request = http.request(reqOptions, function (response) { - - response.on('data', function (chunk) { - responseData.push(chunk); - }); - - response.on('end', function () { - assert(Buffer.concat(responseData).toString() === '{"message":"Error: something terrible. Request Failed At Requests Debugger","error":"Request Failed At Requests Debugger"}'); - done(); - }); - }); - - request.end(); - }); - }); -}); From 853c14096967d5d2554980f2f4552277ec3963d5 Mon Sep 17 00:00:00 2001 From: Neeraj Shukla Date: Mon, 12 Oct 2020 11:52:44 +0530 Subject: [PATCH 05/44] reverse proxy tests --- test/reverseProxy.test.js | 95 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 95 insertions(+) create mode 100644 test/reverseProxy.test.js diff --git a/test/reverseProxy.test.js b/test/reverseProxy.test.js new file mode 100644 index 0000000..bb1b7e5 --- /dev/null +++ b/test/reverseProxy.test.js @@ -0,0 +1,95 @@ +var constants = require('../config/constants'); +var RdGlobalConfig = constants.RdGlobalConfig; +var nock = require('nock'); +var reverseProxy = require('../src/reverseProxy'); +var http = require('http'); +var assert = require('chai').assert; +var testHelper = require('./testHelper'); + +var REVERSE_PROXY_DOMAIN = `http://127.0.0.1:${RdGlobalConfig.RD_HANDLER_REVERSE_PROXY_PORT}`; +var HUB_STATUS_URI_PATH = "/wd/hub/status"; + +describe('RdHandler', function () { + context('Reverse Proxy Server', function () { + + before(function (done) { + this.timeout = 5000; + testHelper.nockGetRequest(REVERSE_PROXY_DOMAIN+HUB_STATUS_URI_PATH, 'http', null, 200); + testHelper.initializeDummyLoggers(); + testHelper.initializeDummyHandlers(); + + reverseProxy.RdHandler.startServer(RdGlobalConfig.RD_HANDLER_REVERSE_PROXY_PORT, function (port) { + console.log('Test Network Utility Proxy Started on Port: ', port); + done(); + }); + }); + + after(function (done) { + this.timeout = 5000; + reverseProxy.RdHandler.stopServer(function () { + done(); + }); + testHelper.deleteLoggers(); + testHelper.deleteHandlers(); + nock.cleanAll(); + }); + + it('Requests on behalf of the client and returns the response', function (done) { + this.timeout = 5000; + var reqOptions = { + method: 'GET', + host: '127.0.0.1', + port: RdGlobalConfig.RD_HANDLER_REVERSE_PROXY_PORT, + headers: {}, + path: HUB_STATUS_URI_PATH + }; + + var responseData = []; + var request = http.request(reqOptions, function (response) { + + response.on('data', function (chunk) { + responseData.push(chunk); + }); + + response.on('end', function () { + assert(Buffer.concat(responseData).toString() === '{"data":"value"}'); + done(); + }); + }); + + request.end(); + }); + + it('Requests on behalf of the client via external reverse proxy and returns the response', function (done) { + this.timeout = 5000; + testHelper.nockGetRequest(REVERSE_PROXY_DOMAIN+HUB_STATUS_URI_PATH, 'http', null, 200); + testHelper.initializeDummyProxy(); + testHelper.nockProxyUrl(RdGlobalConfig.proxy, 'http', 'hub', null, 200); + reverseProxy.RdHandler.generatorForRequestOptionsObject(); + var reqOptions = { + method: 'GET', + host: '127.0.0.1', + port: RdGlobalConfig.RD_HANDLER_REVERSE_PROXY_PORT, + headers: {}, + path: HUB_STATUS_URI_PATH + }; + + var responseData = []; + var request = http.request(reqOptions, function (response) { + + response.on('data', function (chunk) { + responseData.push(chunk); + }); + + response.on('end', function () { + assert(Buffer.concat(responseData).toString() === '{"data":"value"}'); + done(); + }); + }); + + request.end(); + testHelper.deleteProxy(); + }); + + }); +}); From e02d3ab14ec0430a65b393c27b14688819dd6160 Mon Sep 17 00:00:00 2001 From: Neeraj Shukla Date: Wed, 14 Oct 2020 11:01:03 +0530 Subject: [PATCH 06/44] https support --- .gitignore | 6 +++ config/constants.js | 16 ++++--- src/commandLine.js | 41 +++++++++++++++++ src/proxy.js | 2 +- src/requestLib.js | 99 +++++++++++++++++++++++++++++------------ src/requestsDebugger.js | 2 +- 6 files changed, 129 insertions(+), 37 deletions(-) diff --git a/.gitignore b/.gitignore index 3f8f6b0..b208277 100644 --- a/.gitignore +++ b/.gitignore @@ -108,3 +108,9 @@ dist RequestsDebugger-Mac RequestsDebugger.exe RequestsDebugger-Linux* + +# VS code settings +.vscode/ + +# Others +.DS_Store diff --git a/config/constants.js b/config/constants.js index 5c10fce..ea7a83d 100644 --- a/config/constants.js +++ b/config/constants.js @@ -1,7 +1,8 @@ module.exports.VERSION = '1.0.0'; -module.exports.HUB_HOST = 'hub-cloud.browserstack.com'; -module.exports.HUB_STATUS_URL = 'http://hub-cloud.browserstack.com/wd/hub/status'; -module.exports.RAILS_AUTOMATE = 'http://automate.browserstack.com'; +module.exports.BS_DOMAIN = 'browserstack.com'; +module.exports.HUB_HOST = `hub-cloud.${this.BS_DOMAIN}`; +module.exports.HUB_STATUS_URL = `http://${this.HUB_HOST}/wd/hub/status`; +module.exports.RAILS_AUTOMATE = `http://automate.${this.BS_DOMAIN}`; module.exports.CONNECTIVITY_REQ_TIMEOUT = 30000; module.exports.DEFAULT_PROXY_PORT = 3128; module.exports.CUSTOM_ERROR_RESPONSE_CODE = 502; @@ -27,11 +28,12 @@ module.exports.RdGlobalConfig = { RD_HANDLER_PORT: process.env.NODE_ENV === 'test' ? 8787 : 9687, RD_HANDLER_REVERSE_PROXY_PORT: process.env.NODE_ENV === 'test' ? 8788 : 9688, CLIENT_REQ_TIMEOUT: 260000, // in ms + SCHEME: 'https' }; module.exports.COMMON = Object.freeze({ - PING_HUB: 'ping -c 5 hub-cloud.browserstack.com', - PING_AUTOMATE: 'ping -c 5 automate.browserstack.com' + PING_HUB: `ping -c 5 ${this.HUB_HOST}`, + PING_AUTOMATE: `ping -c 5 ${this.RAILS_AUTOMATE}` }); module.exports.MAC = Object.freeze({ @@ -51,8 +53,8 @@ module.exports.WIN = Object.freeze({ NETSTAT_ROUTING_TABLE: 'netstat -r', IPCONFIG_ALL: 'ipconfig /all', SWAP_USAGE: 'pagefile get AllocatedBaseSize, CurrentUsage', // this is a WMIC command. Prefix with WMIC Path - PING_HUB: 'ping -n 5 hub-cloud.browserstack.com', - PING_AUTOMATE: 'ping -n 5 automate.browserstack.com', + PING_HUB: `ping -n 5 ${this.HUB_HOST}`, + PING_AUTOMATE: `ping -n 5 ${this.RAILS_AUTOMATE}`, LOAD_PERCENTAGE: 'cpu get loadpercentage', // prefix wmic path }); diff --git a/src/commandLine.js b/src/commandLine.js index cd1a665..09f0304 100644 --- a/src/commandLine.js +++ b/src/commandLine.js @@ -15,6 +15,8 @@ var CommandLineManager = { + "ARGUMENTS:\n" + " --port : Port on which the Requests Debugger Tool's Proxy will run\n" + " Default: " + RdGlobalConfig.RD_HANDLER_PORT + "\n" + + " --scheme : Scheme for requests to browserstack\n" + + " Default: " + constants.DEFAULT_SCHEME + "\n" + " --proxy-host : Hostname of the Upstream Proxy\n" + " --proxy-port : Port of the Upstream Proxy. Default: " + constants.DEFAULT_PROXY_PORT + " (if hostname is provided)\n" + " --proxy-user : Username for auth of the Upstream Proxy\n" @@ -79,6 +81,24 @@ var CommandLineManager = { } } + // port for Requests Debugger Reverse Proxy + index = argv.indexOf('--reverse-proxy-port'); + if (index !== -1) { + if (CommandLineManager.validArgValue(argv[index + 1])) { + var probablePort = parseInt(argv[index + 1]); + if (!isNaN(probablePort) && (probablePort <= constants.PORTS.MAX) && (probablePort >= constants.PORTS.MIN)) { + RdGlobalConfig.RD_HANDLER_REVERSE_PROXY_PORT = probablePort; + } else { + console.log("\nPort can only range from:", constants.PORTS.MIN, "to:", constants.PORTS.MAX); + invalidArgs.add('--reverse-proxy-port'); + } + argv.splice(index, 2); + } else { + invalidArgs.add('--reverse-proxy-port'); + argv.splice(index, 1); + } + } + // delay for retries in case of request failures index = argv.indexOf('--retry-delay'); if (index !== -1) { @@ -115,6 +135,27 @@ var CommandLineManager = { } } + // process proxy host + index = argv.indexOf('--scheme'); + if (index !== -1) { + if (CommandLineManager.validArgValue(argv[index + 1])) { + var scheme = argv[index + 1]; + if (!(scheme == 'http' || scheme == 'https')){ + console.log("\nScheme can only be http/https"); + invalidArgs.add('--scheme'); + } + else{ + RdGlobalConfig.SCHEME = scheme; + console.log(RdGlobalConfig.SCHEME); + argv.splice(index, 2); + } + } else { + invalidArgs.add('--scheme'); + argv.splice(index, 1); + } + } + + // process proxy host index = argv.indexOf('--proxy-host'); if (index !== -1) { diff --git a/src/proxy.js b/src/proxy.js index 32bf95c..c16e496 100644 --- a/src/proxy.js +++ b/src/proxy.js @@ -65,7 +65,7 @@ var RdHandler = { var parsedClientUrl = url.parse(clientRequest.url); var requestOptions = Object.assign({}, RdHandler._reqObjTemplate); requestOptions.host = parsedClientUrl.hostname; - requestOptions.port = parsedClientUrl.port || 80; + requestOptions.port = parsedClientUrl.port || 443; requestOptions.path = parsedClientUrl.path; requestOptions.method = clientRequest.method; requestOptions.headers = clientRequest.headers; diff --git a/src/requestLib.js b/src/requestLib.js index 4e5e318..ac20cdf 100644 --- a/src/requestLib.js +++ b/src/requestLib.js @@ -1,11 +1,18 @@ var http = require('http'); +var https = require('https'); var constants = require('../config/constants'); var Utils = require('./utils'); -var keepAliveAgent = new http.Agent({ - keepAlive: true -}); + var RdGlobalConfig = constants.RdGlobalConfig; +var keepAliveAgent = null; +if (RdGlobalConfig.SCHEME == "http"){ + keepAliveAgent = new http.Agent({keepAlive: true}); +} +else{ + keepAliveAgent = new https.Agent({keepAlive: true}); +} + var RequestLib = { /** * Method to perform the request on behalf of the client @@ -15,38 +22,74 @@ var RequestLib = { */ _makeRequest: function (params, clientRequest, retries) { return new Promise(function (resolve, reject) { - var requestOptions = Object.assign({}, params.furtherRequestOptions, { - agent: keepAliveAgent - }); - - // Adding a custom header for usage and debugging purpose at BrowserStack - requestOptions.headers['X-Requests-Debugger'] = clientRequest.id; - + var requestOptions = { + headers:{ + // Adding a custom header for usage and debugging purpose at BrowserStack + 'X-Requests-Debugger': clientRequest + } + }; + // Initialize the request to be fired on behalf of the client - var request = http.request(requestOptions, function (response) { - var responseToSend = { - statusCode: response.statusCode, - headers: response.headers, - data: [] - }; - - response.on('data', function (chunk) { - responseToSend.data.push(chunk); - }); + var request = null; - response.on('end', function () { - responseToSend.data = Buffer.concat(responseToSend.data).toString(); - resolve(responseToSend); + if(RdGlobalConfig.SCHEME == "http") { + keepAliveAgent = new http.Agent({keepAlive: true}); + requestOptions = Object.assign(requestOptions, params.furtherRequestOptions, { + agent: keepAliveAgent }); + request = http.request(requestOptions, function (response) { + var responseToSend = { + statusCode: response.statusCode, + headers: response.headers, + data: [] + }; - response.on('error', function (err) { - reject({ - message: err, - customTopic: constants.TOPICS.TOOL_RESPONSE_ERROR + response.on('data', function (chunk) { + responseToSend.data.push(chunk); + }); + + response.on('end', function () { + responseToSend.data = Buffer.concat(responseToSend.data).toString(); + resolve(responseToSend); + }); + + response.on('error', function (err) { + reject({ + message: err, + customTopic: constants.TOPICS.TOOL_RESPONSE_ERROR + }); }); }); - }); + } + else { + keepAliveAgent = new https.Agent({keepAlive: true}); + requestOptions = Object.assign(requestOptions, params.furtherRequestOptions, { + agent: keepAliveAgent + }); + request = https.request(requestOptions, function (response) { + var responseToSend = { + statusCode: response.statusCode, + headers: response.headers, + data: [] + }; + + response.on('data', function (chunk) { + responseToSend.data.push(chunk); + }); + response.on('end', function () { + responseToSend.data = Buffer.concat(responseToSend.data).toString(); + resolve(responseToSend); + }); + + response.on('error', function (err) { + reject({ + message: err, + customTopic: constants.TOPICS.TOOL_RESPONSE_ERROR + }); + }); + }); + } // Log the request that will be initiated on behalf of the client request.on('finish', function () { RdGlobalConfig.reqLogger.info(constants.TOPICS.TOOL_REQUEST_WITH_RETRIES + retries, clientRequest.method + ' ' + clientRequest.url, diff --git a/src/requestsDebugger.js b/src/requestsDebugger.js index eb62c00..4bbe86e 100644 --- a/src/requestsDebugger.js +++ b/src/requestsDebugger.js @@ -140,7 +140,7 @@ var RdTool = { console.log(Utils.formatAndBeautifyLine(STATIC_MESSAGES.CONNECTIVITY_CHECKS_DONE, '', '-', 60, true)); }); - proxy.RdHandler.startServer(RdGlobalConfig.RD_HANDLER_PORT, function (err, result) { + reverseProxy.RdHandler.startServer(RdGlobalConfig.RD_HANDLER_PORT, function (err, result) { if (err) { console.log(STATIC_MESSAGES.ERR_STARTING_TOOL, err); console.log('Exiting the Proxy...'); From d58a203178403810e1fa8e3fb0d8cf822420efcd Mon Sep 17 00:00:00 2001 From: Neeraj Shukla Date: Wed, 14 Oct 2020 13:09:22 +0530 Subject: [PATCH 07/44] refactor --- config/constants.js | 6 +- src/commandLine.js | 20 +--- src/requestsDebugger.js | 15 +-- src/reverseProxy.js | 213 ------------------------------------ src/{proxy.js => server.js} | 73 +++++------- 5 files changed, 33 insertions(+), 294 deletions(-) delete mode 100644 src/reverseProxy.js rename src/{proxy.js => server.js} (71%) diff --git a/config/constants.js b/config/constants.js index ea7a83d..732e85f 100644 --- a/config/constants.js +++ b/config/constants.js @@ -26,7 +26,6 @@ module.exports.LOGS = Object.freeze({ module.exports.RdGlobalConfig = { RETRY_DELAY: 1000, // in ms RD_HANDLER_PORT: process.env.NODE_ENV === 'test' ? 8787 : 9687, - RD_HANDLER_REVERSE_PROXY_PORT: process.env.NODE_ENV === 'test' ? 8788 : 9688, CLIENT_REQ_TIMEOUT: 260000, // in ms SCHEME: 'https' }; @@ -82,9 +81,8 @@ module.exports.STATIC_MESSAGES = Object.freeze({ CHECK_NETWORK_STATS: 'Stats : Checking Network Stats', CHECK_MEMORY_STATS: 'Stats : Checking Memory Stats', CHECK_CONNECTIVITY: 'Checks : Checking Connectivity With BrowserStack', - ERR_STARTING_TOOL: 'Error in starting Requests Debugger Tool Proxy: ', - TOOL_STARTED_ON_PORT: 'Requests Debugger Tool Proxy Started on Port: ', - TOOL_REVERSE_PROXY_STARTED_ON_PORT: 'Requests Debugger Tool Reverse Proxy Started on Port: ', + ERR_STARTING_TOOL: 'Error in starting Requests Debugger Tool: ', + TOOL_STARTED_ON_PORT: 'Requests Debugger Tool Started on Port: ', CPU_STATS_COLLECTED: 'Stats : Initial CPU Stats Collected', NETWORK_STATS_COLLECTED: 'Stats : Initial Network Stats Collected', MEMORY_STATS_COLLECTED: 'Stats : Initial Memory Stats Collected', diff --git a/src/commandLine.js b/src/commandLine.js index 09f0304..51d7f9a 100644 --- a/src/commandLine.js +++ b/src/commandLine.js @@ -15,7 +15,7 @@ var CommandLineManager = { + "ARGUMENTS:\n" + " --port : Port on which the Requests Debugger Tool's Proxy will run\n" + " Default: " + RdGlobalConfig.RD_HANDLER_PORT + "\n" - + " --scheme : Scheme for requests to browserstack\n" + + " --scheme : Scheme for requests to browserstack. Scheme is applicable to only\n" + " Default: " + constants.DEFAULT_SCHEME + "\n" + " --proxy-host : Hostname of the Upstream Proxy\n" + " --proxy-port : Port of the Upstream Proxy. Default: " + constants.DEFAULT_PROXY_PORT + " (if hostname is provided)\n" @@ -80,24 +80,6 @@ var CommandLineManager = { argv.splice(index, 1); } } - - // port for Requests Debugger Reverse Proxy - index = argv.indexOf('--reverse-proxy-port'); - if (index !== -1) { - if (CommandLineManager.validArgValue(argv[index + 1])) { - var probablePort = parseInt(argv[index + 1]); - if (!isNaN(probablePort) && (probablePort <= constants.PORTS.MAX) && (probablePort >= constants.PORTS.MIN)) { - RdGlobalConfig.RD_HANDLER_REVERSE_PROXY_PORT = probablePort; - } else { - console.log("\nPort can only range from:", constants.PORTS.MIN, "to:", constants.PORTS.MAX); - invalidArgs.add('--reverse-proxy-port'); - } - argv.splice(index, 2); - } else { - invalidArgs.add('--reverse-proxy-port'); - argv.splice(index, 1); - } - } // delay for retries in case of request failures index = argv.indexOf('--retry-delay'); diff --git a/src/requestsDebugger.js b/src/requestsDebugger.js index 4bbe86e..f53847a 100644 --- a/src/requestsDebugger.js +++ b/src/requestsDebugger.js @@ -11,8 +11,7 @@ var RdGlobalConfig = constants.RdGlobalConfig; var STATIC_MESSAGES = constants.STATIC_MESSAGES; var CommandLineManager = require('./commandLine'); var ConnectivityChecker = require('./connectivity'); -var proxy = require('./proxy'); -var reverseProxy = require('./reverseProxy'); +var RdHandler = require('./server'); var StatsFactory = require('./stats/statsFactory'); var LogManager = require('./logger'); var fs = require('fs'); @@ -140,7 +139,7 @@ var RdTool = { console.log(Utils.formatAndBeautifyLine(STATIC_MESSAGES.CONNECTIVITY_CHECKS_DONE, '', '-', 60, true)); }); - reverseProxy.RdHandler.startServer(RdGlobalConfig.RD_HANDLER_PORT, function (err, result) { + RdHandler.startServer(RdGlobalConfig.RD_HANDLER_PORT, function (err, result) { if (err) { console.log(STATIC_MESSAGES.ERR_STARTING_TOOL, err); console.log('Exiting the Proxy...'); @@ -148,16 +147,6 @@ var RdTool = { } console.log(Utils.formatAndBeautifyLine(STATIC_MESSAGES.TOOL_STARTED_ON_PORT + result, '', '-', 60, true)); }); - - reverseProxy.RdHandler.startServer(RdGlobalConfig.RD_HANDLER_REVERSE_PROXY_PORT, function (err, result) { - if (err) { - console.log(STATIC_MESSAGES.ERR_STARTING_TOOL, err); - console.log('Exiting the Reverse Proxy...'); - process.exit(1); - } - console.log(Utils.formatAndBeautifyLine(STATIC_MESSAGES.TOOL_REVERSE_PROXY_STARTED_ON_PORT + result, '', '-', 60, true)); - }); - } }; diff --git a/src/reverseProxy.js b/src/reverseProxy.js deleted file mode 100644 index 08bfe3a..0000000 --- a/src/reverseProxy.js +++ /dev/null @@ -1,213 +0,0 @@ -/** - * Reverse Proxy Server to Intercept the client's requests and handle them on their behalf. - * Initiates stats and connectivity checks when a requests fails. - * It also responds in selenium understandable error when a request fails - * at tool. - */ - -var http = require('http'); -var url = require('url'); -var uuidv4 = require('uuid/v4'); -var Utils = require('./utils'); -var constants = require('../config/constants'); -var ReqLib = require('./requestLib'); -var RdGlobalConfig = constants.RdGlobalConfig; - -var RdHandler = { - - _requestCounter: 0, - - /** - * Generates the request options template for firing requests based on - * whether the user had provided any proxy input or not. - */ - generatorForRequestOptionsObject: function () { - RdHandler._reqObjTemplate = { - method: null, - headers: {}, - host: null, - port: null, - path: null - }; - - if (RdGlobalConfig.proxy) { - RdHandler._reqObjTemplate.host = RdGlobalConfig.proxy.host; - RdHandler._reqObjTemplate.port = RdGlobalConfig.proxy.port; - - if (RdGlobalConfig.proxy.username && RdGlobalConfig.proxy.password) { - RdHandler._reqObjTemplate.headers['Proxy-Authorization'] = Utils.proxyAuthToBase64(RdGlobalConfig.proxy); - } - - /** - * Sets the internal method to generate request options if external/upstream - * proxy exists - * @param {http.IncomingMessage} clientRequest - * @returns {Object} - */ - RdHandler._generateRequestOptions = function (clientRequest) { - var parsedClientUrl = url.parse(clientRequest.url); - var headersCopy = Object.assign({}, clientRequest.headers, RdHandler._reqObjTemplate.headers); - var requestOptions = Object.assign({}, RdHandler._reqObjTemplate); - requestOptions.host = constants.HUB_HOST; - requestOptions.port = 80; - requestOptions.path = parsedClientUrl.href; - requestOptions.method = clientRequest.method; - requestOptions.headers = headersCopy; - return requestOptions; - }; - } else { - - /** - * Sets the internal method to generate request options if external/upstream proxy - * doesn't exists - * @param {http.IncomingMessage} clientRequest - * @returns {Object} - */ - RdHandler._generateRequestOptions = function (clientRequest) { - var parsedClientUrl = url.parse(clientRequest.url); - var requestOptions = Object.assign({}, RdHandler._reqObjTemplate); - requestOptions.host = constants.HUB_HOST; - requestOptions.port = 80; - requestOptions.path = parsedClientUrl.path; - requestOptions.method = clientRequest.method; - requestOptions.headers = clientRequest.headers; - if (parsedClientUrl.auth) { - requestOptions.headers['authorization'] = Utils.proxyAuthToBase64(parsedClientUrl.auth); - } - return requestOptions; - }; - } - }, - - /** - * Frames the error response based on the type of request. - * i.e., if its a request originating for Hub, the response - * is in the format which the client binding would understand. - * @param {Object} parsedRequest - * @param {String} errorMessage - */ - _frameErrorResponse: function (parsedRequest, errorMessage) { - errorMessage += '. ' + constants.STATIC_MESSAGES.REQ_FAILED_MSG; - var parseSessionId = parsedRequest.path.match(/\/wd\/hub\/session\/([a-z0-9]+)\/*/); - if (parseSessionId) { - var sessionId = parseSessionId[1]; - return { - data: { - sessionId: sessionId, - status: 13, - value: { - message: errorMessage, - error: constants.STATIC_MESSAGES.REQ_FAILED_MSG - }, - state: 'error' - }, - statusCode: constants.CUSTOM_ERROR_RESPONSE_CODE - }; - } else { - return { - data: { - message: errorMessage, - error: constants.STATIC_MESSAGES.REQ_FAILED_MSG - }, - statusCode: constants.CUSTOM_ERROR_RESPONSE_CODE - }; - } - }, - - /** - * Handler for incoming requests to Requests Debugger Tool reverse proxy server. - * @param {http.IncomingMessage} clientRequest - * @param {http.ServerResponse} clientResponse - */ - requestHandler: function (clientRequest, clientResponse) { - clientRequest.id = ++RdHandler._requestCounter + '::' + uuidv4(); - - var request = { - method: clientRequest.method, - url: clientRequest.url, - headers: clientRequest.headers, - data: [] - }; - - RdGlobalConfig.reqLogger.info(constants.TOPICS.CLIENT_REQUEST_START, request.method + ' ' + request.url, - false, { - headers: request.headers - }, - clientRequest.id); - - var furtherRequestOptions = RdHandler._generateRequestOptions(clientRequest); - - var paramsForRequest = { - request: request, - furtherRequestOptions: furtherRequestOptions - }; - - ReqLib.call(paramsForRequest, clientRequest) - .then(function (response) { - RdGlobalConfig.reqLogger.info(constants.TOPICS.CLIENT_RESPONSE_END, clientRequest.method + ' ' + clientRequest.url + ', Status Code: ' + response.statusCode, - false, { - data: response.data, - headers: response.headers, - }, - clientRequest.id); - - clientResponse.writeHead(response.statusCode, response.headers); - clientResponse.end(response.data); - }) - .catch(function (err) { - RdGlobalConfig.reqLogger.error(err.customTopic || constants.TOPICS.UNEXPECTED_ERROR, clientRequest.method + ' ' + clientRequest.url, - false, { - errorMessage: err.message.toString() - }, - clientRequest.id); - - var errorResponse = RdHandler._frameErrorResponse(furtherRequestOptions, err.message.toString()); - RdGlobalConfig.reqLogger.error(constants.TOPICS.CLIENT_RESPONSE_END, clientRequest.method + ' ' + clientRequest.url + ', Status Code: ' + errorResponse.statusCode, - false, - errorResponse.data, - clientRequest.id); - - clientResponse.writeHead(errorResponse.statusCode); - clientResponse.end(JSON.stringify(errorResponse.data)); - }); - }, - - /** - * Starts the reverse proxy server on the given port - * @param {String|Number} port - * @param {Function} callback - */ - startServer: function (port, callback) { - try { - RdHandler.generatorForRequestOptionsObject(); - RdHandler.server = http.createServer(RdHandler.requestHandler); - RdHandler.server.listen(port); - RdHandler.server.on('listening', function () { - callback(null, port); - }); - RdHandler.server.on('error', function (err) { - callback(err.toString(), null); - }); - } catch (e) { - callback(e.toString(), null); - } - }, - - /** - * Stops the currently running reverse proxy server - * @param {Function} callback - */ - stopServer: function (callback) { - try { - if (RdHandler.server) { - RdHandler.server.close(); - RdHandler.server = null; - } - callback(null, true); - } catch (e) { - callback(e.toString(), null); - } - } -}; - -module.exports.RdHandler = RdHandler; diff --git a/src/proxy.js b/src/server.js similarity index 71% rename from src/proxy.js rename to src/server.js index c16e496..61eb048 100644 --- a/src/proxy.js +++ b/src/server.js @@ -1,5 +1,5 @@ /** - * Proxy Server to Intercept the client's requests and handle them on their behalf. + * Server to Intercept the client's requests and handle them on their behalf. * Initiates stats and connectivity checks when a requests fails. * It also responds in selenium understandable error when a request fails * at tool. @@ -18,8 +18,7 @@ var RdHandler = { _requestCounter: 0, /** - * Generates the request options template for firing requests based on - * whether the user had provided any proxy input or not. + * Generates the request options template for firing requests */ generatorForRequestOptionsObject: function () { RdHandler._reqObjTemplate = { @@ -29,6 +28,7 @@ var RdHandler = { port: null, path: null }; + RdHandler._reqObjTemplate.headers['host'] = constants.BS_DOMAIN; if (RdGlobalConfig.proxy) { RdHandler._reqObjTemplate.host = RdGlobalConfig.proxy.host; @@ -37,44 +37,27 @@ var RdHandler = { if (RdGlobalConfig.proxy.username && RdGlobalConfig.proxy.password) { RdHandler._reqObjTemplate.headers['Proxy-Authorization'] = Utils.proxyAuthToBase64(RdGlobalConfig.proxy); } - - /** - * Sets the internal method to generate request options if external/upstream - * proxy exists - * @param {http.IncomingMessage} clientRequest - * @returns {Object} - */ - RdHandler._generateRequestOptions = function (clientRequest) { - var parsedClientUrl = url.parse(clientRequest.url); - var headersCopy = Object.assign({}, clientRequest.headers, RdHandler._reqObjTemplate.headers); - var requestOptions = Object.assign({}, RdHandler._reqObjTemplate); - requestOptions.path = parsedClientUrl.href; - requestOptions.method = clientRequest.method; - requestOptions.headers = headersCopy; - return requestOptions; - }; - } else { - - /** - * Sets the internal method to generate request options if external/upstream proxy - * doesn't exists - * @param {http.IncomingMessage} clientRequest - * @returns {Object} - */ - RdHandler._generateRequestOptions = function (clientRequest) { - var parsedClientUrl = url.parse(clientRequest.url); - var requestOptions = Object.assign({}, RdHandler._reqObjTemplate); - requestOptions.host = parsedClientUrl.hostname; - requestOptions.port = parsedClientUrl.port || 443; - requestOptions.path = parsedClientUrl.path; - requestOptions.method = clientRequest.method; - requestOptions.headers = clientRequest.headers; - if (parsedClientUrl.auth) { - requestOptions.headers['authorization'] = Utils.proxyAuthToBase64(parsedClientUrl.auth); - } - return requestOptions; - }; - } + } + var requestOptions = Object.assign({}, RdHandler._reqObjTemplate); + requestOptions.host = constants.HUB_HOST; + requestOptions.port = RdGlobalConfig.SCHEME == 'http' ? 80 : 443; + + /** + * Sets the internal method to generate request options + * doesn't exists + * @param {http.IncomingMessage} clientRequest + * @returns {Object} + */ + RdHandler._generateRequestOptions = function (clientRequest) { + var parsedClientUrl = url.parse(clientRequest.url); + requestOptions.path = RdGlobalConfig.SCHEME + "://" + constants.HUB_HOST + parsedClientUrl.path; + requestOptions.method = clientRequest.method; + requestOptions.headers = Object.assign({}, clientRequest.headers, RdHandler._reqObjTemplate.headers); + if (parsedClientUrl.auth) { + requestOptions.headers['authorization'] = Utils.proxyAuthToBase64(parsedClientUrl.auth); + } + return requestOptions; + }; }, /** @@ -113,7 +96,7 @@ var RdHandler = { }, /** - * Handler for incoming requests to Requests Debugger Tool proxy server. + * Handler for incoming requests to Requests Debugger Tool server. * @param {http.IncomingMessage} clientRequest * @param {http.ServerResponse} clientResponse */ @@ -171,7 +154,7 @@ var RdHandler = { }, /** - * Starts the proxy server on the given port + * Starts the server on the given port * @param {String|Number} port * @param {Function} callback */ @@ -192,7 +175,7 @@ var RdHandler = { }, /** - * Stops the currently running proxy server + * Stops the currently running server * @param {Function} callback */ stopServer: function (callback) { @@ -208,4 +191,4 @@ var RdHandler = { } }; -module.exports.RdHandler = RdHandler; +module.exports = RdHandler; From 00327badba01a7237a62b8fdd162a5f30f636dfc Mon Sep 17 00:00:00 2001 From: Neeraj Shukla Date: Wed, 14 Oct 2020 15:23:23 +0530 Subject: [PATCH 08/44] refactor and tests --- src/server.js | 7 +- test/reverseProxy.test.js | 95 -------------------------- test/{proxy.test.js => server.test.js} | 21 +++--- test/testHelper.js | 2 +- 4 files changed, 19 insertions(+), 106 deletions(-) delete mode 100644 test/reverseProxy.test.js rename test/{proxy.test.js => server.test.js} (84%) diff --git a/src/server.js b/src/server.js index 61eb048..1ebde74 100644 --- a/src/server.js +++ b/src/server.js @@ -50,7 +50,12 @@ var RdHandler = { */ RdHandler._generateRequestOptions = function (clientRequest) { var parsedClientUrl = url.parse(clientRequest.url); - requestOptions.path = RdGlobalConfig.SCHEME + "://" + constants.HUB_HOST + parsedClientUrl.path; + if (RdGlobalConfig.proxy) { + requestOptions.path = RdGlobalConfig.SCHEME + "://" + constants.HUB_HOST + parsedClientUrl.path; + } + else { + requestOptions.path = parsedClientUrl.path; + } requestOptions.method = clientRequest.method; requestOptions.headers = Object.assign({}, clientRequest.headers, RdHandler._reqObjTemplate.headers); if (parsedClientUrl.auth) { diff --git a/test/reverseProxy.test.js b/test/reverseProxy.test.js deleted file mode 100644 index bb1b7e5..0000000 --- a/test/reverseProxy.test.js +++ /dev/null @@ -1,95 +0,0 @@ -var constants = require('../config/constants'); -var RdGlobalConfig = constants.RdGlobalConfig; -var nock = require('nock'); -var reverseProxy = require('../src/reverseProxy'); -var http = require('http'); -var assert = require('chai').assert; -var testHelper = require('./testHelper'); - -var REVERSE_PROXY_DOMAIN = `http://127.0.0.1:${RdGlobalConfig.RD_HANDLER_REVERSE_PROXY_PORT}`; -var HUB_STATUS_URI_PATH = "/wd/hub/status"; - -describe('RdHandler', function () { - context('Reverse Proxy Server', function () { - - before(function (done) { - this.timeout = 5000; - testHelper.nockGetRequest(REVERSE_PROXY_DOMAIN+HUB_STATUS_URI_PATH, 'http', null, 200); - testHelper.initializeDummyLoggers(); - testHelper.initializeDummyHandlers(); - - reverseProxy.RdHandler.startServer(RdGlobalConfig.RD_HANDLER_REVERSE_PROXY_PORT, function (port) { - console.log('Test Network Utility Proxy Started on Port: ', port); - done(); - }); - }); - - after(function (done) { - this.timeout = 5000; - reverseProxy.RdHandler.stopServer(function () { - done(); - }); - testHelper.deleteLoggers(); - testHelper.deleteHandlers(); - nock.cleanAll(); - }); - - it('Requests on behalf of the client and returns the response', function (done) { - this.timeout = 5000; - var reqOptions = { - method: 'GET', - host: '127.0.0.1', - port: RdGlobalConfig.RD_HANDLER_REVERSE_PROXY_PORT, - headers: {}, - path: HUB_STATUS_URI_PATH - }; - - var responseData = []; - var request = http.request(reqOptions, function (response) { - - response.on('data', function (chunk) { - responseData.push(chunk); - }); - - response.on('end', function () { - assert(Buffer.concat(responseData).toString() === '{"data":"value"}'); - done(); - }); - }); - - request.end(); - }); - - it('Requests on behalf of the client via external reverse proxy and returns the response', function (done) { - this.timeout = 5000; - testHelper.nockGetRequest(REVERSE_PROXY_DOMAIN+HUB_STATUS_URI_PATH, 'http', null, 200); - testHelper.initializeDummyProxy(); - testHelper.nockProxyUrl(RdGlobalConfig.proxy, 'http', 'hub', null, 200); - reverseProxy.RdHandler.generatorForRequestOptionsObject(); - var reqOptions = { - method: 'GET', - host: '127.0.0.1', - port: RdGlobalConfig.RD_HANDLER_REVERSE_PROXY_PORT, - headers: {}, - path: HUB_STATUS_URI_PATH - }; - - var responseData = []; - var request = http.request(reqOptions, function (response) { - - response.on('data', function (chunk) { - responseData.push(chunk); - }); - - response.on('end', function () { - assert(Buffer.concat(responseData).toString() === '{"data":"value"}'); - done(); - }); - }); - - request.end(); - testHelper.deleteProxy(); - }); - - }); -}); diff --git a/test/proxy.test.js b/test/server.test.js similarity index 84% rename from test/proxy.test.js rename to test/server.test.js index dd8803d..aaf2fee 100644 --- a/test/proxy.test.js +++ b/test/server.test.js @@ -1,29 +1,30 @@ var constants = require('../config/constants'); var RdGlobalConfig = constants.RdGlobalConfig; var nock = require('nock'); -var proxy = require('../src/proxy'); +var RdHandler = require('../src/server'); var http = require('http'); var assert = require('chai').assert; var testHelper = require('./testHelper'); +STATUS_URL = `http://${constants.HUB_HOST}/wd/hub/status` + describe('RdHandler', function () { context('Proxy Server', function () { before(function (done) { this.timeout = 5000; - testHelper.nockGetRequest(constants.HUB_STATUS_URL, 'http', null, 200); testHelper.initializeDummyLoggers(); testHelper.initializeDummyHandlers(); - proxy.RdHandler.startServer(RdGlobalConfig.RD_HANDLER_PORT, function (port) { - console.log('Test Network Utility Proxy Started on Port: ', port); + RdHandler.startServer(RdGlobalConfig.RD_HANDLER_PORT, function (port) { + console.log('Test Network Utility Server Started on Port: ', port); done(); }); }); after(function (done) { this.timeout = 5000; - proxy.RdHandler.stopServer(function () { + RdHandler.stopServer(function () { done(); }); testHelper.deleteLoggers(); @@ -33,12 +34,13 @@ describe('RdHandler', function () { it('Requests on behalf of the client and returns the response', function (done) { this.timeout = 5000; + testHelper.nockGetRequest(STATUS_URL, 'https', null, 200); var reqOptions = { method: 'GET', host: 'localhost', port: RdGlobalConfig.RD_HANDLER_PORT, headers: {}, - path: constants.HUB_STATUS_URL + path: '/wd/hub/status' }; var responseData = []; @@ -59,9 +61,10 @@ describe('RdHandler', function () { it('Requests on behalf of the client via external proxy and returns the response', function (done) { this.timeout = 5000; + testHelper.nockGetRequest(STATUS_URL, 'https', null, 200); testHelper.initializeDummyProxy(); testHelper.nockProxyUrl(RdGlobalConfig.proxy, 'http', 'hub', null, 200); - proxy.RdHandler.generatorForRequestOptionsObject(); + RdHandler.generatorForRequestOptionsObject(); var reqOptions = { method: 'GET', host: 'localhost', @@ -90,9 +93,9 @@ describe('RdHandler', function () { it('Requests on behalf of the client via external proxy and returns the response even if request by tool fails', function (done) { this.timeout = 5000; for (var i = 0; i <= constants.MAX_RETRIES; i++) { - testHelper.nockGetRequestWithError(constants.HUB_STATUS_URL, 'http'); + testHelper.nockGetRequestWithError(constants.HUB_STATUS_URL, 'https'); } - proxy.RdHandler.generatorForRequestOptionsObject(); + RdHandler.generatorForRequestOptionsObject(); var reqOptions = { method: 'GET', host: 'localhost', diff --git a/test/testHelper.js b/test/testHelper.js index caa91d6..8504345 100644 --- a/test/testHelper.js +++ b/test/testHelper.js @@ -52,7 +52,7 @@ function nockGetRequestWithError(reqUrl, type) { function initializeDummyProxy() { constants.RdGlobalConfig.proxy = { host: "dummyhost12345.com", - port: "3128", + port: "3130", username: "user", password: "pass" }; From 678e4563dc4863681c7deaa3aab665736c4d8b66 Mon Sep 17 00:00:00 2001 From: Neeraj Shukla Date: Wed, 14 Oct 2020 15:30:20 +0530 Subject: [PATCH 09/44] removed redundant code --- src/commandLine.js | 2 +- src/requestLib.js | 8 -------- test/server.test.js | 5 ++--- 3 files changed, 3 insertions(+), 12 deletions(-) diff --git a/src/commandLine.js b/src/commandLine.js index 51d7f9a..b4a427b 100644 --- a/src/commandLine.js +++ b/src/commandLine.js @@ -15,7 +15,7 @@ var CommandLineManager = { + "ARGUMENTS:\n" + " --port : Port on which the Requests Debugger Tool's Proxy will run\n" + " Default: " + RdGlobalConfig.RD_HANDLER_PORT + "\n" - + " --scheme : Scheme for requests to browserstack. Scheme is applicable to only\n" + + " --scheme : Scheme for requests to browserstack.\n" + " Default: " + constants.DEFAULT_SCHEME + "\n" + " --proxy-host : Hostname of the Upstream Proxy\n" + " --proxy-port : Port of the Upstream Proxy. Default: " + constants.DEFAULT_PROXY_PORT + " (if hostname is provided)\n" diff --git a/src/requestLib.js b/src/requestLib.js index ac20cdf..a18c3e2 100644 --- a/src/requestLib.js +++ b/src/requestLib.js @@ -5,14 +5,6 @@ var Utils = require('./utils'); var RdGlobalConfig = constants.RdGlobalConfig; -var keepAliveAgent = null; -if (RdGlobalConfig.SCHEME == "http"){ - keepAliveAgent = new http.Agent({keepAlive: true}); -} -else{ - keepAliveAgent = new https.Agent({keepAlive: true}); -} - var RequestLib = { /** * Method to perform the request on behalf of the client diff --git a/test/server.test.js b/test/server.test.js index aaf2fee..46b3985 100644 --- a/test/server.test.js +++ b/test/server.test.js @@ -6,7 +6,6 @@ var http = require('http'); var assert = require('chai').assert; var testHelper = require('./testHelper'); -STATUS_URL = `http://${constants.HUB_HOST}/wd/hub/status` describe('RdHandler', function () { context('Proxy Server', function () { @@ -34,7 +33,7 @@ describe('RdHandler', function () { it('Requests on behalf of the client and returns the response', function (done) { this.timeout = 5000; - testHelper.nockGetRequest(STATUS_URL, 'https', null, 200); + testHelper.nockGetRequest(constants.HUB_STATUS_URL, 'https', null, 200); var reqOptions = { method: 'GET', host: 'localhost', @@ -61,7 +60,7 @@ describe('RdHandler', function () { it('Requests on behalf of the client via external proxy and returns the response', function (done) { this.timeout = 5000; - testHelper.nockGetRequest(STATUS_URL, 'https', null, 200); + testHelper.nockGetRequest(constants.HUB_STATUS_URL, 'https', null, 200); testHelper.initializeDummyProxy(); testHelper.nockProxyUrl(RdGlobalConfig.proxy, 'http', 'hub', null, 200); RdHandler.generatorForRequestOptionsObject(); From 94ce40ca7a778d6a6b2005851744295b75cd5c2d Mon Sep 17 00:00:00 2001 From: Neeraj Shukla Date: Wed, 14 Oct 2020 15:49:31 +0530 Subject: [PATCH 10/44] move common variable up --- src/requestLib.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/requestLib.js b/src/requestLib.js index a18c3e2..42b46ff 100644 --- a/src/requestLib.js +++ b/src/requestLib.js @@ -23,7 +23,7 @@ var RequestLib = { // Initialize the request to be fired on behalf of the client var request = null; - + var keepAliveAgent = null; if(RdGlobalConfig.SCHEME == "http") { keepAliveAgent = new http.Agent({keepAlive: true}); requestOptions = Object.assign(requestOptions, params.furtherRequestOptions, { From b77557d62d065f9c4ec58794b5d88d7853675702 Mon Sep 17 00:00:00 2001 From: Neeraj Shukla Date: Wed, 14 Oct 2020 19:05:53 +0530 Subject: [PATCH 11/44] refactored code --- src/requestLib.js | 84 +++++++++++++++-------------------------------- 1 file changed, 27 insertions(+), 57 deletions(-) diff --git a/src/requestLib.js b/src/requestLib.js index 42b46ff..d15ba31 100644 --- a/src/requestLib.js +++ b/src/requestLib.js @@ -12,7 +12,7 @@ var RequestLib = { * @param {http.IncomingMessage} clientRequest * @param {Number} retries */ - _makeRequest: function (params, clientRequest, retries) { + _makeRequest: function (schemeObj, params, clientRequest, retries) { return new Promise(function (resolve, reject) { var requestOptions = { headers:{ @@ -20,68 +20,37 @@ var RequestLib = { 'X-Requests-Debugger': clientRequest } }; - // Initialize the request to be fired on behalf of the client var request = null; var keepAliveAgent = null; - if(RdGlobalConfig.SCHEME == "http") { - keepAliveAgent = new http.Agent({keepAlive: true}); - requestOptions = Object.assign(requestOptions, params.furtherRequestOptions, { - agent: keepAliveAgent - }); - request = http.request(requestOptions, function (response) { - var responseToSend = { - statusCode: response.statusCode, - headers: response.headers, - data: [] - }; - - response.on('data', function (chunk) { - responseToSend.data.push(chunk); - }); - - response.on('end', function () { - responseToSend.data = Buffer.concat(responseToSend.data).toString(); - resolve(responseToSend); - }); - - response.on('error', function (err) { - reject({ - message: err, - customTopic: constants.TOPICS.TOOL_RESPONSE_ERROR - }); - }); + keepAliveAgent = new schemeObj.Agent({keepAlive: true}); + requestOptions = Object.assign(requestOptions, params.furtherRequestOptions, { + agent: keepAliveAgent + }); + request = schemeObj.request(requestOptions, function (response) { + var responseToSend = { + statusCode: response.statusCode, + headers: response.headers, + data: [] + }; + + response.on('data', function (chunk) { + responseToSend.data.push(chunk); }); - } - else { - keepAliveAgent = new https.Agent({keepAlive: true}); - requestOptions = Object.assign(requestOptions, params.furtherRequestOptions, { - agent: keepAliveAgent + + response.on('end', function () { + responseToSend.data = Buffer.concat(responseToSend.data).toString(); + resolve(responseToSend); }); - request = https.request(requestOptions, function (response) { - var responseToSend = { - statusCode: response.statusCode, - headers: response.headers, - data: [] - }; - - response.on('data', function (chunk) { - responseToSend.data.push(chunk); - }); - - response.on('end', function () { - responseToSend.data = Buffer.concat(responseToSend.data).toString(); - resolve(responseToSend); - }); - - response.on('error', function (err) { - reject({ - message: err, - customTopic: constants.TOPICS.TOOL_RESPONSE_ERROR - }); + + response.on('error', function (err) { + reject({ + message: err, + customTopic: constants.TOPICS.TOOL_RESPONSE_ERROR }); }); - } + }); + // Log the request that will be initiated on behalf of the client request.on('finish', function () { RdGlobalConfig.reqLogger.info(constants.TOPICS.TOOL_REQUEST_WITH_RETRIES + retries, clientRequest.method + ' ' + clientRequest.url, @@ -152,7 +121,8 @@ var RequestLib = { */ call: function (params, clientRequest, retries) { retries = (typeof retries === 'number') ? Math.min(constants.MAX_RETRIES, Math.max(retries, 0)) : constants.MAX_RETRIES; - return RequestLib._makeRequest(params, clientRequest, retries) + var schemeObj = RdGlobalConfig.SCHEME == "http" ? http : https; + return RequestLib._makeRequest(schemeObj, params, clientRequest, retries) .catch(function (err) { var errTopic = err.customTopic || constants.TOPICS.UNEXPECTED_ERROR; // Collect Network & Connectivity Logs whenever a request fails From 8fe5b2377d9c351b60e5ac43e58eb5a68c637f45 Mon Sep 17 00:00:00 2001 From: Neeraj Shukla Date: Wed, 14 Oct 2020 19:20:50 +0530 Subject: [PATCH 12/44] bug fix --- src/server.js | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/server.js b/src/server.js index 1ebde74..31c7d7c 100644 --- a/src/server.js +++ b/src/server.js @@ -39,8 +39,7 @@ var RdHandler = { } } var requestOptions = Object.assign({}, RdHandler._reqObjTemplate); - requestOptions.host = constants.HUB_HOST; - requestOptions.port = RdGlobalConfig.SCHEME == 'http' ? 80 : 443; + /** * Sets the internal method to generate request options @@ -51,9 +50,13 @@ var RdHandler = { RdHandler._generateRequestOptions = function (clientRequest) { var parsedClientUrl = url.parse(clientRequest.url); if (RdGlobalConfig.proxy) { + requestOptions.host = RdGlobalConfig.proxy.host; + requestOptions.port = RdGlobalConfig.proxy.port; requestOptions.path = RdGlobalConfig.SCHEME + "://" + constants.HUB_HOST + parsedClientUrl.path; } else { + requestOptions.host = constants.HUB_HOST; + requestOptions.port = RdGlobalConfig.SCHEME == 'http' ? 80 : 443; requestOptions.path = parsedClientUrl.path; } requestOptions.method = clientRequest.method; From a1e8281e54571a983d1b2ab5c880461de6fad6ad Mon Sep 17 00:00:00 2001 From: Neeraj Shukla Date: Thu, 15 Oct 2020 10:54:58 +0530 Subject: [PATCH 13/44] removed unnecessary console log --- src/commandLine.js | 1 - 1 file changed, 1 deletion(-) diff --git a/src/commandLine.js b/src/commandLine.js index b4a427b..10c3def 100644 --- a/src/commandLine.js +++ b/src/commandLine.js @@ -128,7 +128,6 @@ var CommandLineManager = { } else{ RdGlobalConfig.SCHEME = scheme; - console.log(RdGlobalConfig.SCHEME); argv.splice(index, 2); } } else { From c8fa1f954cba0ac822c6ad6af8145c065a85ab8e Mon Sep 17 00:00:00 2001 From: Neeraj Shukla Date: Thu, 15 Oct 2020 12:07:14 +0530 Subject: [PATCH 14/44] version bump --- config/constants.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/constants.js b/config/constants.js index 732e85f..99d7fe9 100644 --- a/config/constants.js +++ b/config/constants.js @@ -1,4 +1,4 @@ -module.exports.VERSION = '1.0.0'; +module.exports.VERSION = '1.1.0'; module.exports.BS_DOMAIN = 'browserstack.com'; module.exports.HUB_HOST = `hub-cloud.${this.BS_DOMAIN}`; module.exports.HUB_STATUS_URL = `http://${this.HUB_HOST}/wd/hub/status`; From d62fd57eb93bf05ff0964935d89854675b2e426f Mon Sep 17 00:00:00 2001 From: Neeraj Shukla Date: Fri, 16 Oct 2020 20:54:23 +0530 Subject: [PATCH 15/44] reverse proxy with https support --- README.md | 6 ++-- package.json | 2 ++ src/commandLine.js | 9 ++++-- src/requestLib.js | 28 ++++++++--------- src/server.js | 65 +++++++++++----------------------------- test/commandLine.test.js | 23 +++++++------- test/server.test.js | 2 -- 7 files changed, 55 insertions(+), 80 deletions(-) diff --git a/README.md b/README.md index 12d276f..4d52ba6 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ ### Tool for debugging client side failure of requests, leading to requests getting dropped or not reaching BrowserStack. ## Features -- Proxy Server to intercept requests fired by client bindings to keep track of their flow. +- Server to intercept requests fired by client bindings to keep track of their flow. - Connectivity Checker : To check for the reachability of BrowserStack components, i.e. Rails & Hub. - Multi-Platform Stats Compatibility : Ability to collect stats of CPU, Network & Memory Stats. - Retry Mechanism in case a request fails at the client side @@ -13,6 +13,7 @@ - Start Requests Debugger with the required arguments: `npm run start -- `. - Supported `args`: - `--port `: Port on which the Requests Debugger Tool's Proxy will run. Default: 9687 + - `--scheme `: Scheme for requests to browserstack. Default: https" - `--proxy-host `: Hostname of the Upstream Proxy - `--proxy-port `: Port of the Upstream Proxy. Default: 3128 (if hostname is provided) - `--proxy-user `: Username for auth of the Upstream Proxy @@ -30,7 +31,7 @@ - Windows: `RequestsDebugger.exe ` ## How to use -- Since the tool acts like a proxy, you will have to set the proxy to be used by your client binding to `localhost:9687`. i.e. +- To use tool as a proxy, you will have to set the proxy to be used by your client binding to `localhost:9687`. i.e. - For Java: - ``` System.getProperties().put("http.proxyHost", "localhost"); @@ -40,6 +41,7 @@ - Set your system's env variable `http_proxy=localhost:9687` and Ruby's Selenium Client Binding will pick the value. Or, - Run you test by giving the environment variable to your command itself, i.e. `http_proxy=localhost:9687 ruby ` - Similarly, you can also set proxy for other client bindings. +- To use tool as reverse proxy, you will have to replace hub-cloud.browserstack.com in hub url with `localhost:9687`. ## Steps to build the executables - Linux diff --git a/package.json b/package.json index 492ca4d..4fa720d 100644 --- a/package.json +++ b/package.json @@ -34,6 +34,8 @@ "sinon": "7.5.0" }, "dependencies": { + "http-proxy-agent": "^2.1.0", + "https-proxy-agent": "^2.1.0", "uuid": "3.4.0", "winston": "2.4.4" }, diff --git a/src/commandLine.js b/src/commandLine.js index 10c3def..b92c903 100644 --- a/src/commandLine.js +++ b/src/commandLine.js @@ -2,8 +2,9 @@ * Command Line Manager to parse the command line arguments * and set the necessary fields in RdGlobalConfig. */ - +var url = require('url'); var constants = require('../config/constants'); + var RdGlobalConfig = constants.RdGlobalConfig; var CommandLineManager = { @@ -16,7 +17,7 @@ var CommandLineManager = { + " --port : Port on which the Requests Debugger Tool's Proxy will run\n" + " Default: " + RdGlobalConfig.RD_HANDLER_PORT + "\n" + " --scheme : Scheme for requests to browserstack.\n" - + " Default: " + constants.DEFAULT_SCHEME + "\n" + + " Default: " + RdGlobalConfig.SCHEME + "\n" + " --proxy-host : Hostname of the Upstream Proxy\n" + " --proxy-port : Port of the Upstream Proxy. Default: " + constants.DEFAULT_PROXY_PORT + " (if hostname is provided)\n" + " --proxy-user : Username for auth of the Upstream Proxy\n" @@ -142,7 +143,9 @@ var CommandLineManager = { if (index !== -1) { if (CommandLineManager.validArgValue(argv[index + 1])) { var host = argv[index + 1]; - host = host.replace(constants.PROTOCOL_REGEX, ''); + if(host.lastIndexOf("http") != 0){ + host = `http://${host}`; + } RdGlobalConfig.proxy = RdGlobalConfig.proxy || {}; RdGlobalConfig.proxy.host = host; argv.splice(index, 2); diff --git a/src/requestLib.js b/src/requestLib.js index d15ba31..a4dbc5e 100644 --- a/src/requestLib.js +++ b/src/requestLib.js @@ -2,32 +2,31 @@ var http = require('http'); var https = require('https'); var constants = require('../config/constants'); var Utils = require('./utils'); +var url = require('url'); +var HttpProxyAgent = require('https-proxy-agent'); +var HttpsProxyAgent = require('https-proxy-agent'); var RdGlobalConfig = constants.RdGlobalConfig; var RequestLib = { /** * Method to perform the request on behalf of the client + * @param {schemeObj: Object} schemeObj * @param {{request: Object, furtherRequestOptions: Object}} params * @param {http.IncomingMessage} clientRequest * @param {Number} retries */ _makeRequest: function (schemeObj, params, clientRequest, retries) { return new Promise(function (resolve, reject) { - var requestOptions = { - headers:{ - // Adding a custom header for usage and debugging purpose at BrowserStack - 'X-Requests-Debugger': clientRequest - } - }; - // Initialize the request to be fired on behalf of the client - var request = null; - var keepAliveAgent = null; - keepAliveAgent = new schemeObj.Agent({keepAlive: true}); - requestOptions = Object.assign(requestOptions, params.furtherRequestOptions, { - agent: keepAliveAgent - }); - request = schemeObj.request(requestOptions, function (response) { + var requestOptions = Object.assign({}, params.furtherRequestOptions); + requestOptions.agent = new schemeObj.Agent({keepAlive: true}); + if(RdGlobalConfig.proxy) { + var proxyOpts = url.parse(`${RdGlobalConfig.proxy.host}:${RdGlobalConfig.proxy.port}`); + if(RdGlobalConfig.proxy.username && RdGlobalConfig.proxy.password); + proxyOpts.auth = `${RdGlobalConfig.proxy.username}:${RdGlobalConfig.proxy.password}`; + requestOptions.agent = RdGlobalConfig.SCHEME == 'http' ? new HttpProxyAgent(proxyOpts) : new HttpsProxyAgent(proxyOpts); + } + var request = schemeObj.request(requestOptions, function (response) { var responseToSend = { statusCode: response.statusCode, headers: response.headers, @@ -148,4 +147,3 @@ var RequestLib = { }; module.exports = RequestLib; - diff --git a/src/server.js b/src/server.js index 31c7d7c..ac1cac8 100644 --- a/src/server.js +++ b/src/server.js @@ -11,6 +11,7 @@ var uuidv4 = require('uuid/v4'); var Utils = require('./utils'); var constants = require('../config/constants'); var ReqLib = require('./requestLib'); + var RdGlobalConfig = constants.RdGlobalConfig; var RdHandler = { @@ -18,54 +19,25 @@ var RdHandler = { _requestCounter: 0, /** - * Generates the request options template for firing requests + * Generates the request options for firing requests + * @param {http.IncomingMessage} clientRequest + * @returns {Object} */ - generatorForRequestOptionsObject: function () { - RdHandler._reqObjTemplate = { - method: null, - headers: {}, - host: null, - port: null, - path: null - }; - RdHandler._reqObjTemplate.headers['host'] = constants.BS_DOMAIN; - - if (RdGlobalConfig.proxy) { - RdHandler._reqObjTemplate.host = RdGlobalConfig.proxy.host; - RdHandler._reqObjTemplate.port = RdGlobalConfig.proxy.port; - - if (RdGlobalConfig.proxy.username && RdGlobalConfig.proxy.password) { - RdHandler._reqObjTemplate.headers['Proxy-Authorization'] = Utils.proxyAuthToBase64(RdGlobalConfig.proxy); - } - } - var requestOptions = Object.assign({}, RdHandler._reqObjTemplate); - - - /** - * Sets the internal method to generate request options - * doesn't exists - * @param {http.IncomingMessage} clientRequest - * @returns {Object} - */ - RdHandler._generateRequestOptions = function (clientRequest) { - var parsedClientUrl = url.parse(clientRequest.url); - if (RdGlobalConfig.proxy) { - requestOptions.host = RdGlobalConfig.proxy.host; - requestOptions.port = RdGlobalConfig.proxy.port; - requestOptions.path = RdGlobalConfig.SCHEME + "://" + constants.HUB_HOST + parsedClientUrl.path; - } - else { - requestOptions.host = constants.HUB_HOST; - requestOptions.port = RdGlobalConfig.SCHEME == 'http' ? 80 : 443; - requestOptions.path = parsedClientUrl.path; - } - requestOptions.method = clientRequest.method; - requestOptions.headers = Object.assign({}, clientRequest.headers, RdHandler._reqObjTemplate.headers); - if (parsedClientUrl.auth) { - requestOptions.headers['authorization'] = Utils.proxyAuthToBase64(parsedClientUrl.auth); - } - return requestOptions; + _generateRequestOptions: function (clientRequest) { + var requestOptions = { + headers: {} }; + var parsedClientUrl = url.parse(clientRequest.url); + requestOptions.host = constants.HUB_HOST; + requestOptions.port = RdGlobalConfig.SCHEME == 'http' ? 80 : 443; + requestOptions.path = parsedClientUrl.path; + requestOptions.method = clientRequest.method; + requestOptions.headers = clientRequest.headers; + requestOptions.headers['X-Requests-Debugger'] = clientRequest.id; + if (parsedClientUrl.auth) { + requestOptions.headers['Authorization'] = Utils.proxyAuthToBase64(parsedClientUrl.auth); + } + return requestOptions; }, /** @@ -168,7 +140,6 @@ var RdHandler = { */ startServer: function (port, callback) { try { - RdHandler.generatorForRequestOptionsObject(); RdHandler.server = http.createServer(RdHandler.requestHandler); RdHandler.server.listen(port); RdHandler.server.on('listening', function () { diff --git a/test/commandLine.test.js b/test/commandLine.test.js index 6730d42..833b4d7 100644 --- a/test/commandLine.test.js +++ b/test/commandLine.test.js @@ -8,7 +8,8 @@ var testHelper = require('./testHelper'); describe('CommandLineManager', function () { var argv; - + var proxy_host_actual_value = "http://host"; + before(function () { console.log("NOTE: 'console.log' will be stubbed. In case any test fails, try removing the stub to see the logs"); }); @@ -34,22 +35,22 @@ describe('CommandLineManager', function () { argv = argv.concat(['--proxy-host', 'host', '--proxy-port', '9687']); CommandLineManager.processArgs(argv); console.log.restore(); - expect(RdGlobalConfig.proxy.host).to.eql('host'); + expect(RdGlobalConfig.proxy.host).to.eql(proxy_host_actual_value); expect(RdGlobalConfig.proxy.port).to.eql(9687); }); it('remove any protocol part from proxy-host', function () { sinon.stub(console, 'log'); - argv = argv.concat(['--proxy-host', 'http://host']); + argv = argv.concat(['--proxy-host', 'host']); CommandLineManager.processArgs(argv); - expect(RdGlobalConfig.proxy.host).to.eql('host'); + expect(RdGlobalConfig.proxy.host).to.eql(proxy_host_actual_value); }); it('remove any prefix slashes from proxy-host', function () { - argv = argv.concat(['--proxy-host', '//host']); + argv = argv.concat(['--proxy-host', 'host']); CommandLineManager.processArgs(argv); console.log.restore(); - expect(RdGlobalConfig.proxy.host).to.eql('host'); + expect(RdGlobalConfig.proxy.host).to.eql(proxy_host_actual_value); }); it('proxy-port is set to the default value when its not in the expected range', function () { @@ -57,7 +58,7 @@ describe('CommandLineManager', function () { argv = argv.concat(['--proxy-host', 'host', '--proxy-port', '99999']); CommandLineManager.processArgs(argv); console.log.restore(); - expect(RdGlobalConfig.proxy.host).to.eql('host'); + expect(RdGlobalConfig.proxy.host).to.eql(proxy_host_actual_value); expect(RdGlobalConfig.proxy.port).to.eql(constants.DEFAULT_PROXY_PORT); }); @@ -66,7 +67,7 @@ describe('CommandLineManager', function () { argv = argv.concat(['--proxy-host', 'host', '--proxy-port', '9687', '--proxy-user', 'user', '--proxy-pass', 'pass']); CommandLineManager.processArgs(argv); console.log.restore(); - expect(RdGlobalConfig.proxy.host).to.eql('host'); + expect(RdGlobalConfig.proxy.host).to.eql(proxy_host_actual_value); expect(RdGlobalConfig.proxy.port).to.eql(9687); expect(RdGlobalConfig.proxy.username).to.eql('user'); expect(RdGlobalConfig.proxy.password).to.eql('pass'); @@ -77,7 +78,7 @@ describe('CommandLineManager', function () { argv = argv.concat(['--proxy-host', 'host']); CommandLineManager.processArgs(argv); console.log.restore(); - expect(RdGlobalConfig.proxy.host).to.eql('host'); + expect(RdGlobalConfig.proxy.host).to.eql(proxy_host_actual_value); expect(RdGlobalConfig.proxy.port).to.eql(constants.DEFAULT_PROXY_PORT); }); @@ -86,7 +87,7 @@ describe('CommandLineManager', function () { argv = argv.concat(['--proxy-host', 'host', '--proxy-port', '9687', '--proxy-user', 'user']); CommandLineManager.processArgs(argv); console.log.restore(); - expect(RdGlobalConfig.proxy.host).to.eql('host'); + expect(RdGlobalConfig.proxy.host).to.eql(proxy_host_actual_value); expect(RdGlobalConfig.proxy.port).to.eql(9687); expect(RdGlobalConfig.proxy.username).to.eql('user'); expect(RdGlobalConfig.proxy.password).to.eql(''); @@ -106,7 +107,7 @@ describe('CommandLineManager', function () { argv = argv.concat(['--proxy-host', 'host', '--proxy-port', '9687', '--proxy-pass', 'pass']); CommandLineManager.processArgs(argv); console.log.restore(); - expect(RdGlobalConfig.proxy.host).to.eql('host'); + expect(RdGlobalConfig.proxy.host).to.eql(proxy_host_actual_value); expect(RdGlobalConfig.proxy.port).to.eql(9687); expect(RdGlobalConfig.proxy.username).to.eql(undefined); expect(RdGlobalConfig.proxy.password).to.eql(undefined); diff --git a/test/server.test.js b/test/server.test.js index 46b3985..f489c38 100644 --- a/test/server.test.js +++ b/test/server.test.js @@ -63,7 +63,6 @@ describe('RdHandler', function () { testHelper.nockGetRequest(constants.HUB_STATUS_URL, 'https', null, 200); testHelper.initializeDummyProxy(); testHelper.nockProxyUrl(RdGlobalConfig.proxy, 'http', 'hub', null, 200); - RdHandler.generatorForRequestOptionsObject(); var reqOptions = { method: 'GET', host: 'localhost', @@ -94,7 +93,6 @@ describe('RdHandler', function () { for (var i = 0; i <= constants.MAX_RETRIES; i++) { testHelper.nockGetRequestWithError(constants.HUB_STATUS_URL, 'https'); } - RdHandler.generatorForRequestOptionsObject(); var reqOptions = { method: 'GET', host: 'localhost', From b44e248abe3ea36599e38ab937e51ff77ade4c18 Mon Sep 17 00:00:00 2001 From: Neeraj Shukla Date: Fri, 16 Oct 2020 21:02:55 +0530 Subject: [PATCH 16/44] request log change --- src/requestLib.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/requestLib.js b/src/requestLib.js index a4dbc5e..b866b8b 100644 --- a/src/requestLib.js +++ b/src/requestLib.js @@ -52,7 +52,8 @@ var RequestLib = { // Log the request that will be initiated on behalf of the client request.on('finish', function () { - RdGlobalConfig.reqLogger.info(constants.TOPICS.TOOL_REQUEST_WITH_RETRIES + retries, clientRequest.method + ' ' + clientRequest.url, + RdGlobalConfig.reqLogger.info(constants.TOPICS.TOOL_REQUEST_WITH_RETRIES + retries, clientRequest.method + ' ' + + clientRequest.url.replace("http://", RdGlobalConfig.SCHEME + "://"), false, Object.assign({}, params.furtherRequestOptions, { data: Buffer.concat(params.request.data).toString() From 949c0c34101613f74d16da42cfc43afd803bba68 Mon Sep 17 00:00:00 2001 From: Neeraj Shukla Date: Fri, 16 Oct 2020 21:39:01 +0530 Subject: [PATCH 17/44] log url fix --- src/requestLib.js | 8 +++----- src/server.js | 10 +++++----- 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/src/requestLib.js b/src/requestLib.js index b866b8b..44461ce 100644 --- a/src/requestLib.js +++ b/src/requestLib.js @@ -52,10 +52,8 @@ var RequestLib = { // Log the request that will be initiated on behalf of the client request.on('finish', function () { - RdGlobalConfig.reqLogger.info(constants.TOPICS.TOOL_REQUEST_WITH_RETRIES + retries, clientRequest.method + ' ' + - clientRequest.url.replace("http://", RdGlobalConfig.SCHEME + "://"), - false, - Object.assign({}, params.furtherRequestOptions, { + RdGlobalConfig.reqLogger.info(constants.TOPICS.TOOL_REQUEST_WITH_RETRIES + retries, + clientRequest.method + ' ' + requestOptions.path, false, Object.assign({}, params.furtherRequestOptions, { data: Buffer.concat(params.request.data).toString() }), clientRequest.id); @@ -100,7 +98,7 @@ var RequestLib = { }); clientRequest.on('end', function () { - RdGlobalConfig.reqLogger.info(constants.TOPICS.CLIENT_REQUEST_END, params.request.method + ' ' + params.request.url, false, { + RdGlobalConfig.reqLogger.info(constants.TOPICS.CLIENT_REQUEST_END, params.request.method + ' ' + requestOptions.path, false, { data: Buffer.concat(params.request.data).toString() }, clientRequest.id); diff --git a/src/server.js b/src/server.js index ac1cac8..9302511 100644 --- a/src/server.js +++ b/src/server.js @@ -33,6 +33,7 @@ var RdHandler = { requestOptions.path = parsedClientUrl.path; requestOptions.method = clientRequest.method; requestOptions.headers = clientRequest.headers; + requestOptions.headers.host = constants.HUB_HOST; requestOptions.headers['X-Requests-Debugger'] = clientRequest.id; if (parsedClientUrl.auth) { requestOptions.headers['Authorization'] = Utils.proxyAuthToBase64(parsedClientUrl.auth); @@ -82,15 +83,14 @@ var RdHandler = { */ requestHandler: function (clientRequest, clientResponse) { clientRequest.id = ++RdHandler._requestCounter + '::' + uuidv4(); - + var uri = url.parse(clientRequest.url).path; var request = { method: clientRequest.method, url: clientRequest.url, headers: clientRequest.headers, data: [] }; - - RdGlobalConfig.reqLogger.info(constants.TOPICS.CLIENT_REQUEST_START, request.method + ' ' + request.url, + RdGlobalConfig.reqLogger.info(constants.TOPICS.CLIENT_REQUEST_START, request.method + ' ' + uri, false, { headers: request.headers }, @@ -105,7 +105,7 @@ var RdHandler = { ReqLib.call(paramsForRequest, clientRequest) .then(function (response) { - RdGlobalConfig.reqLogger.info(constants.TOPICS.CLIENT_RESPONSE_END, clientRequest.method + ' ' + clientRequest.url + ', Status Code: ' + response.statusCode, + RdGlobalConfig.reqLogger.info(constants.TOPICS.CLIENT_RESPONSE_END, clientRequest.method + ' ' + uri + ', Status Code: ' + response.statusCode, false, { data: response.data, headers: response.headers, @@ -123,7 +123,7 @@ var RdHandler = { clientRequest.id); var errorResponse = RdHandler._frameErrorResponse(furtherRequestOptions, err.message.toString()); - RdGlobalConfig.reqLogger.error(constants.TOPICS.CLIENT_RESPONSE_END, clientRequest.method + ' ' + clientRequest.url + ', Status Code: ' + errorResponse.statusCode, + RdGlobalConfig.reqLogger.error(constants.TOPICS.CLIENT_RESPONSE_END, clientRequest.method + ' ' + uri + ', Status Code: ' + errorResponse.statusCode, false, errorResponse.data, clientRequest.id); From 3eb7827c70052382d852b2cf38c8b8b74b5e8e3e Mon Sep 17 00:00:00 2001 From: Neeraj Shukla Date: Mon, 19 Oct 2020 11:19:37 +0530 Subject: [PATCH 18/44] refactored code --- src/commandLine.js | 8 ++++---- src/requestsDebugger.js | 2 +- src/server.js | 8 ++++---- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/commandLine.js b/src/commandLine.js index b92c903..642e293 100644 --- a/src/commandLine.js +++ b/src/commandLine.js @@ -81,7 +81,7 @@ var CommandLineManager = { argv.splice(index, 1); } } - + // delay for retries in case of request failures index = argv.indexOf('--retry-delay'); if (index !== -1) { @@ -123,11 +123,11 @@ var CommandLineManager = { if (index !== -1) { if (CommandLineManager.validArgValue(argv[index + 1])) { var scheme = argv[index + 1]; - if (!(scheme == 'http' || scheme == 'https')){ + if (!(scheme === 'http' || scheme === 'https')){ console.log("\nScheme can only be http/https"); invalidArgs.add('--scheme'); } - else{ + else { RdGlobalConfig.SCHEME = scheme; argv.splice(index, 2); } @@ -143,7 +143,7 @@ var CommandLineManager = { if (index !== -1) { if (CommandLineManager.validArgValue(argv[index + 1])) { var host = argv[index + 1]; - if(host.lastIndexOf("http") != 0){ + if(host.lastIndexOf("http") !== 0){ host = `http://${host}`; } RdGlobalConfig.proxy = RdGlobalConfig.proxy || {}; diff --git a/src/requestsDebugger.js b/src/requestsDebugger.js index f53847a..32b20ea 100644 --- a/src/requestsDebugger.js +++ b/src/requestsDebugger.js @@ -142,7 +142,7 @@ var RdTool = { RdHandler.startServer(RdGlobalConfig.RD_HANDLER_PORT, function (err, result) { if (err) { console.log(STATIC_MESSAGES.ERR_STARTING_TOOL, err); - console.log('Exiting the Proxy...'); + console.log('Exiting the Server...'); process.exit(1); } console.log(Utils.formatAndBeautifyLine(STATIC_MESSAGES.TOOL_STARTED_ON_PORT + result, '', '-', 60, true)); diff --git a/src/server.js b/src/server.js index 9302511..80a5b0b 100644 --- a/src/server.js +++ b/src/server.js @@ -83,14 +83,14 @@ var RdHandler = { */ requestHandler: function (clientRequest, clientResponse) { clientRequest.id = ++RdHandler._requestCounter + '::' + uuidv4(); - var uri = url.parse(clientRequest.url).path; + var path = url.parse(clientRequest.url).path; var request = { method: clientRequest.method, url: clientRequest.url, headers: clientRequest.headers, data: [] }; - RdGlobalConfig.reqLogger.info(constants.TOPICS.CLIENT_REQUEST_START, request.method + ' ' + uri, + RdGlobalConfig.reqLogger.info(constants.TOPICS.CLIENT_REQUEST_START, request.method + ' ' + path, false, { headers: request.headers }, @@ -105,7 +105,7 @@ var RdHandler = { ReqLib.call(paramsForRequest, clientRequest) .then(function (response) { - RdGlobalConfig.reqLogger.info(constants.TOPICS.CLIENT_RESPONSE_END, clientRequest.method + ' ' + uri + ', Status Code: ' + response.statusCode, + RdGlobalConfig.reqLogger.info(constants.TOPICS.CLIENT_RESPONSE_END, clientRequest.method + ' ' + path + ', Status Code: ' + response.statusCode, false, { data: response.data, headers: response.headers, @@ -123,7 +123,7 @@ var RdHandler = { clientRequest.id); var errorResponse = RdHandler._frameErrorResponse(furtherRequestOptions, err.message.toString()); - RdGlobalConfig.reqLogger.error(constants.TOPICS.CLIENT_RESPONSE_END, clientRequest.method + ' ' + uri + ', Status Code: ' + errorResponse.statusCode, + RdGlobalConfig.reqLogger.error(constants.TOPICS.CLIENT_RESPONSE_END, clientRequest.method + ' ' + path + ', Status Code: ' + errorResponse.statusCode, false, errorResponse.data, clientRequest.id); From 50744b32c9310784269116968238b937667ac830 Mon Sep 17 00:00:00 2001 From: Neeraj Shukla Date: Mon, 19 Oct 2020 13:26:00 +0530 Subject: [PATCH 19/44] bug fix --- src/requestLib.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/requestLib.js b/src/requestLib.js index 44461ce..bee063a 100644 --- a/src/requestLib.js +++ b/src/requestLib.js @@ -3,7 +3,7 @@ var https = require('https'); var constants = require('../config/constants'); var Utils = require('./utils'); var url = require('url'); -var HttpProxyAgent = require('https-proxy-agent'); +var HttpProxyAgent = require('http-proxy-agent'); var HttpsProxyAgent = require('https-proxy-agent'); var RdGlobalConfig = constants.RdGlobalConfig; @@ -22,7 +22,7 @@ var RequestLib = { requestOptions.agent = new schemeObj.Agent({keepAlive: true}); if(RdGlobalConfig.proxy) { var proxyOpts = url.parse(`${RdGlobalConfig.proxy.host}:${RdGlobalConfig.proxy.port}`); - if(RdGlobalConfig.proxy.username && RdGlobalConfig.proxy.password); + if(RdGlobalConfig.proxy.username && RdGlobalConfig.proxy.password) proxyOpts.auth = `${RdGlobalConfig.proxy.username}:${RdGlobalConfig.proxy.password}`; requestOptions.agent = RdGlobalConfig.SCHEME == 'http' ? new HttpProxyAgent(proxyOpts) : new HttpsProxyAgent(proxyOpts); } From b81b4eb7c92cd248951feff5171fd8784f0c21ef Mon Sep 17 00:00:00 2001 From: Neeraj Shukla Date: Mon, 19 Oct 2020 15:10:01 +0530 Subject: [PATCH 20/44] added two servers for proxy and reverse proxy --- config/constants.js | 10 ++++- src/requestsDebugger.js | 16 +++++-- src/server.js | 96 +++++++++++++++++++++++++++++++++-------- 3 files changed, 100 insertions(+), 22 deletions(-) diff --git a/config/constants.js b/config/constants.js index 99d7fe9..dd21852 100644 --- a/config/constants.js +++ b/config/constants.js @@ -12,6 +12,12 @@ module.exports.PORTS = { MAX: 65535, MIN: 1 }; + +module.exports.SERVER_TYPES = { + PROXY: "proxy", + REVERSE_PROXY: "reverse_proxy" +}; + module.exports.PROTOCOL_REGEX = /(^\w+:|^)\/\//; module.exports.LOGS = Object.freeze({ @@ -26,6 +32,7 @@ module.exports.LOGS = Object.freeze({ module.exports.RdGlobalConfig = { RETRY_DELAY: 1000, // in ms RD_HANDLER_PORT: process.env.NODE_ENV === 'test' ? 8787 : 9687, + RD_HANDLER_REVERSE_PROXY_PORT: process.env.NODE_ENV === 'test' ? 8788 : 9688, CLIENT_REQ_TIMEOUT: 260000, // in ms SCHEME: 'https' }; @@ -82,7 +89,8 @@ module.exports.STATIC_MESSAGES = Object.freeze({ CHECK_MEMORY_STATS: 'Stats : Checking Memory Stats', CHECK_CONNECTIVITY: 'Checks : Checking Connectivity With BrowserStack', ERR_STARTING_TOOL: 'Error in starting Requests Debugger Tool: ', - TOOL_STARTED_ON_PORT: 'Requests Debugger Tool Started on Port: ', + TOOL_STARTED_ON_PORT: 'Requests Debugger Tool Proxy Started on Port: ', + TOOL_STARTED_ON_PORT: 'Requests Debugger Tool Reverse Proxy Started on Port: ', CPU_STATS_COLLECTED: 'Stats : Initial CPU Stats Collected', NETWORK_STATS_COLLECTED: 'Stats : Initial Network Stats Collected', MEMORY_STATS_COLLECTED: 'Stats : Initial Memory Stats Collected', diff --git a/src/requestsDebugger.js b/src/requestsDebugger.js index 32b20ea..1e47e7e 100644 --- a/src/requestsDebugger.js +++ b/src/requestsDebugger.js @@ -139,14 +139,24 @@ var RdTool = { console.log(Utils.formatAndBeautifyLine(STATIC_MESSAGES.CONNECTIVITY_CHECKS_DONE, '', '-', 60, true)); }); - RdHandler.startServer(RdGlobalConfig.RD_HANDLER_PORT, function (err, result) { + RdHandler.startProxyServer(RdGlobalConfig.RD_HANDLER_PORT, function (err, result) { if (err) { console.log(STATIC_MESSAGES.ERR_STARTING_TOOL, err); - console.log('Exiting the Server...'); + console.log('Exiting the Proxy Server...'); process.exit(1); } - console.log(Utils.formatAndBeautifyLine(STATIC_MESSAGES.TOOL_STARTED_ON_PORT + result, '', '-', 60, true)); + console.log(Utils.formatAndBeautifyLine(STATIC_MESSAGES.TOOL_PROXY_STARTED_ON_PORT + result, '', '-', 60, true)); }); + + RdHandler.startReverseProxyServer(RdGlobalConfig.RD_HANDLER_REVERSE_PROXY_PORT, function (err, result) { + if (err) { + console.log(STATIC_MESSAGES.ERR_STARTING_TOOL, err); + console.log('Exiting the Reverse Server...'); + process.exit(1); + } + console.log(Utils.formatAndBeautifyLine(STATIC_MESSAGES.TOOL_REVESE_PROXY_STARTED_ON_PORT + result, '', '-', 60, true)); + }); + } }; diff --git a/src/server.js b/src/server.js index 80a5b0b..f82cd88 100644 --- a/src/server.js +++ b/src/server.js @@ -20,20 +20,24 @@ var RdHandler = { /** * Generates the request options for firing requests - * @param {http.IncomingMessage} clientRequest + * @param {http.IncomingMessage} clientRequest + * @param {String} serverType * @returns {Object} */ - _generateRequestOptions: function (clientRequest) { + _generateRequestOptions: function (clientRequest, serverType) { var requestOptions = { headers: {} }; var parsedClientUrl = url.parse(clientRequest.url); - requestOptions.host = constants.HUB_HOST; - requestOptions.port = RdGlobalConfig.SCHEME == 'http' ? 80 : 443; + requestOptions.host = parsedClientUrl.host; + requestOptions.port = RdGlobalConfig.SCHEME == 'http' ? 80 : 443; requestOptions.path = parsedClientUrl.path; requestOptions.method = clientRequest.method; requestOptions.headers = clientRequest.headers; - requestOptions.headers.host = constants.HUB_HOST; + if(serverType == constants.SERVER_TYPES.REVERSE_PROXY){ + requestOptions.host = constants.HUB_HOST; + requestOptions.headers.host = constants.HUB_HOST; + } requestOptions.headers['X-Requests-Debugger'] = clientRequest.id; if (parsedClientUrl.auth) { requestOptions.headers['Authorization'] = Utils.proxyAuthToBase64(parsedClientUrl.auth); @@ -80,8 +84,9 @@ var RdHandler = { * Handler for incoming requests to Requests Debugger Tool server. * @param {http.IncomingMessage} clientRequest * @param {http.ServerResponse} clientResponse + * @param {String} serverType */ - requestHandler: function (clientRequest, clientResponse) { + requestHandler: function (clientRequest, clientResponse, serverType) { clientRequest.id = ++RdHandler._requestCounter + '::' + uuidv4(); var path = url.parse(clientRequest.url).path; var request = { @@ -96,7 +101,7 @@ var RdHandler = { }, clientRequest.id); - var furtherRequestOptions = RdHandler._generateRequestOptions(clientRequest); + var furtherRequestOptions = RdHandler._generateRequestOptions(clientRequest, serverType); var paramsForRequest = { request: request, @@ -134,18 +139,56 @@ var RdHandler = { }, /** - * Starts the server on the given port + * Handler for incoming requests to Requests Debugger Tool proxy server. + * @param {http.IncomingMessage} clientRequest + * @param {http.ServerResponse} clientResponse + */ + proxyRequestHandler: function (clientRequest, clientResponse) { + RdHandler.requestHandler(clientRequest, clientResponse, constants.SERVER_TYPES.PROXY); + }, + + /** + * Handler for incoming requests to Requests Debugger Tool reverse proxy server. + * @param {http.IncomingMessage} clientRequest + * @param {http.ServerResponse} clientResponse + */ + reverseProxyRequestHandler: function (clientRequest, clientResponse) { + RdHandler.requestHandler(clientRequest, clientResponse, constants.SERVER_TYPES.REVERSE_PROXY); + }, + + /** + * Starts the proxy server on the given port * @param {String|Number} port * @param {Function} callback */ - startServer: function (port, callback) { + startProxyServer: function (port, callback) { try { - RdHandler.server = http.createServer(RdHandler.requestHandler); - RdHandler.server.listen(port); - RdHandler.server.on('listening', function () { + RdHandler.proxyServer = http.createServer(RdHandler.proxyRequestHandler); + RdHandler.proxyServer.listen(port); + RdHandler.proxyServer.on('listening', function () { callback(null, port); }); - RdHandler.server.on('error', function (err) { + RdHandler.proxyServer.on('error', function (err) { + callback(err.toString(), null); + }); + } catch (e) { + callback(e.toString(), null); + } + }, + + /** + * Starts the reverse proxy server on the given port + * @param {String|Number} port + * @param {Function} callback + */ + startReverseProxyServer: function (port, callback) { + try { + RdHandler.reverseProxyServer = http.createServer(RdHandler.reverseProxyRequestHandler); + RdHandler.reverseProxyServer.listen(port); + RdHandler.reverseProxyServer.on('listening', function () { + callback(null, port); + }); + RdHandler.reverseProxyServer.on('error', function (err) { callback(err.toString(), null); }); } catch (e) { @@ -154,20 +197,37 @@ var RdHandler = { }, /** - * Stops the currently running server + * Stops the currently running proxy server * @param {Function} callback */ - stopServer: function (callback) { + stopProxyServer: function (callback) { try { - if (RdHandler.server) { - RdHandler.server.close(); - RdHandler.server = null; + if (RdHandler.proxyServer) { + RdHandler.proxyServer.close(); + RdHandler.proxyServer = null; + } + callback(null, true); + } catch (e) { + callback(e.toString(), null); + } + }, + + /** + * Stops the currently running reverse proxy server + * @param {Function} callback + */ + stopReverseProxyServer: function (callback) { + try { + if (RdHandler.reverseProxyServer) { + RdHandler.reverseProxyServer.close(); + RdHandler.reverseProxyServer = null; } callback(null, true); } catch (e) { callback(e.toString(), null); } } + }; module.exports = RdHandler; From 99412f5af661086a2a07361df7dbf0be85c3ffec Mon Sep 17 00:00:00 2001 From: Neeraj Shukla Date: Mon, 19 Oct 2020 16:19:33 +0530 Subject: [PATCH 21/44] refactored code --- config/constants.js | 6 +++--- src/requestLib.js | 23 ++++++++++++++++------- src/requestsDebugger.js | 4 ++-- 3 files changed, 21 insertions(+), 12 deletions(-) diff --git a/config/constants.js b/config/constants.js index dd21852..9bc5912 100644 --- a/config/constants.js +++ b/config/constants.js @@ -31,7 +31,7 @@ module.exports.LOGS = Object.freeze({ module.exports.RdGlobalConfig = { RETRY_DELAY: 1000, // in ms - RD_HANDLER_PORT: process.env.NODE_ENV === 'test' ? 8787 : 9687, + RD_HANDLER_PROXY_PORT: process.env.NODE_ENV === 'test' ? 8787 : 9687, RD_HANDLER_REVERSE_PROXY_PORT: process.env.NODE_ENV === 'test' ? 8788 : 9688, CLIENT_REQ_TIMEOUT: 260000, // in ms SCHEME: 'https' @@ -89,8 +89,8 @@ module.exports.STATIC_MESSAGES = Object.freeze({ CHECK_MEMORY_STATS: 'Stats : Checking Memory Stats', CHECK_CONNECTIVITY: 'Checks : Checking Connectivity With BrowserStack', ERR_STARTING_TOOL: 'Error in starting Requests Debugger Tool: ', - TOOL_STARTED_ON_PORT: 'Requests Debugger Tool Proxy Started on Port: ', - TOOL_STARTED_ON_PORT: 'Requests Debugger Tool Reverse Proxy Started on Port: ', + TOOL_PROXY_STARTED_ON_PORT: 'Requests Debugger Tool Proxy Server Started on Port: ', + TOOL_REVESE_PROXY_STARTED_ON_PORT: 'Requests Debugger Tool Reverse Proxy Server Started on Port: ', CPU_STATS_COLLECTED: 'Stats : Initial CPU Stats Collected', NETWORK_STATS_COLLECTED: 'Stats : Initial Network Stats Collected', MEMORY_STATS_COLLECTED: 'Stats : Initial Memory Stats Collected', diff --git a/src/requestLib.js b/src/requestLib.js index bee063a..70f11b0 100644 --- a/src/requestLib.js +++ b/src/requestLib.js @@ -7,8 +7,12 @@ var HttpProxyAgent = require('http-proxy-agent'); var HttpsProxyAgent = require('https-proxy-agent'); var RdGlobalConfig = constants.RdGlobalConfig; - +var httpKeepAliveAgent = new http.Agent({keepAlive: true}); +var httpsKeepAliveAgent = new https.Agent({keepAlive: true}); +var httpProxyAgent = null; +var httpsProxyAgent = null; var RequestLib = { + /** * Method to perform the request on behalf of the client * @param {schemeObj: Object} schemeObj @@ -19,12 +23,17 @@ var RequestLib = { _makeRequest: function (schemeObj, params, clientRequest, retries) { return new Promise(function (resolve, reject) { var requestOptions = Object.assign({}, params.furtherRequestOptions); - requestOptions.agent = new schemeObj.Agent({keepAlive: true}); + requestOptions.agent = RdGlobalConfig.SCHEME === 'http' ? httpKeepAliveAgent : httpsKeepAliveAgent; if(RdGlobalConfig.proxy) { - var proxyOpts = url.parse(`${RdGlobalConfig.proxy.host}:${RdGlobalConfig.proxy.port}`); - if(RdGlobalConfig.proxy.username && RdGlobalConfig.proxy.password) - proxyOpts.auth = `${RdGlobalConfig.proxy.username}:${RdGlobalConfig.proxy.password}`; - requestOptions.agent = RdGlobalConfig.SCHEME == 'http' ? new HttpProxyAgent(proxyOpts) : new HttpsProxyAgent(proxyOpts); + if (!httpProxyAgent && !httpsProxyAgent) { + var proxyOpts = url.parse(`${RdGlobalConfig.proxy.host}:${RdGlobalConfig.proxy.port}`); + if(RdGlobalConfig.proxy.username && RdGlobalConfig.proxy.password) { + proxyOpts.auth = `${RdGlobalConfig.proxy.username}:${RdGlobalConfig.proxy.password}`; + } + httpProxyAgent = HttpProxyAgent(proxyOpts); + httpsProxyAgent = HttpsProxyAgent(proxyOpts); + } + requestOptions.agent = RdGlobalConfig.SCHEME === 'http' ? httpProxyAgent : httpsProxyAgent; } var request = schemeObj.request(requestOptions, function (response) { var responseToSend = { @@ -119,7 +128,7 @@ var RequestLib = { */ call: function (params, clientRequest, retries) { retries = (typeof retries === 'number') ? Math.min(constants.MAX_RETRIES, Math.max(retries, 0)) : constants.MAX_RETRIES; - var schemeObj = RdGlobalConfig.SCHEME == "http" ? http : https; + var schemeObj = RdGlobalConfig.SCHEME === "http" ? http : https; return RequestLib._makeRequest(schemeObj, params, clientRequest, retries) .catch(function (err) { var errTopic = err.customTopic || constants.TOPICS.UNEXPECTED_ERROR; diff --git a/src/requestsDebugger.js b/src/requestsDebugger.js index 1e47e7e..ce9cb5c 100644 --- a/src/requestsDebugger.js +++ b/src/requestsDebugger.js @@ -139,7 +139,7 @@ var RdTool = { console.log(Utils.formatAndBeautifyLine(STATIC_MESSAGES.CONNECTIVITY_CHECKS_DONE, '', '-', 60, true)); }); - RdHandler.startProxyServer(RdGlobalConfig.RD_HANDLER_PORT, function (err, result) { + RdHandler.startProxyServer(RdGlobalConfig.RD_HANDLER_PROXY_PORT, function (err, result) { if (err) { console.log(STATIC_MESSAGES.ERR_STARTING_TOOL, err); console.log('Exiting the Proxy Server...'); @@ -151,7 +151,7 @@ var RdTool = { RdHandler.startReverseProxyServer(RdGlobalConfig.RD_HANDLER_REVERSE_PROXY_PORT, function (err, result) { if (err) { console.log(STATIC_MESSAGES.ERR_STARTING_TOOL, err); - console.log('Exiting the Reverse Server...'); + console.log('Exiting the Reverse Proxy Server...'); process.exit(1); } console.log(Utils.formatAndBeautifyLine(STATIC_MESSAGES.TOOL_REVESE_PROXY_STARTED_ON_PORT + result, '', '-', 60, true)); From 8c62005d472be871014320a9c507fc702b2ca915 Mon Sep 17 00:00:00 2001 From: Neeraj Shukla Date: Mon, 19 Oct 2020 20:24:24 +0530 Subject: [PATCH 22/44] refactor --- src/commandLine.js | 59 +++++++++++++++++++++++++++-------------- src/requestLib.js | 7 ++--- src/requestsDebugger.js | 31 +++++++++++----------- 3 files changed, 59 insertions(+), 38 deletions(-) diff --git a/src/commandLine.js b/src/commandLine.js index 642e293..8063166 100644 --- a/src/commandLine.js +++ b/src/commandLine.js @@ -2,7 +2,6 @@ * Command Line Manager to parse the command line arguments * and set the necessary fields in RdGlobalConfig. */ -var url = require('url'); var constants = require('../config/constants'); var RdGlobalConfig = constants.RdGlobalConfig; @@ -14,23 +13,25 @@ var CommandLineManager = { + "\n" + "Usage: RequestsDebugger [ARGUMENTS]\n\n" + "ARGUMENTS:\n" - + " --port : Port on which the Requests Debugger Tool's Proxy will run\n" - + " Default: " + RdGlobalConfig.RD_HANDLER_PORT + "\n" - + " --scheme : Scheme for requests to browserstack.\n" - + " Default: " + RdGlobalConfig.SCHEME + "\n" - + " --proxy-host : Hostname of the Upstream Proxy\n" - + " --proxy-port : Port of the Upstream Proxy. Default: " + constants.DEFAULT_PROXY_PORT + " (if hostname is provided)\n" - + " --proxy-user : Username for auth of the Upstream Proxy\n" - + " --proxy-pass : Password for auth of the Upstream Proxy\n" - + " --retry-delay : Delay for the retry of a failed request. Default: " + RdGlobalConfig.RETRY_DELAY + "ms\n" - + " --request-timeout : Hard timeout for the requests being fired by the tool before receiving any response\n" - + " Default: " + RdGlobalConfig.CLIENT_REQ_TIMEOUT + "ms\n" - + " --logs-path : Directory where the '" + constants.LOGS_FOLDER + "' folder will be created\n" - + " for storing logs. Default: Current Working Directory\n" - + " --del-logs : Deletes any existing logs from the " + constants.LOGS_FOLDER + "/ directory and initializes\n" - + " new files for logging\n" - + " --help : Help for Requests Debugger Tool\n" - + " --version : Version of the Requests Debugger Tool\n"; + + " --proxy-port : Port on which the Requests Debugger Tool's Proxy will run\n" + + " Default: " + RdGlobalConfig.RD_HANDLER_PROXY_PORT + "\n" + + " --reverse-proxy-port : Port on which the Requests Debugger Tool's Reverse Proxy will run\n" + + " Default: " + RdGlobalConfig.RD_HANDLER_REVERSE_PROXY_PORT + "\n" + + " --scheme : Scheme for requests to browserstack.\n" + + " Default: " + RdGlobalConfig.SCHEME + "\n" + + " --proxy-host : Hostname of the Upstream Proxy\n" + + " --proxy-port : Port of the Upstream Proxy. Default: " + constants.DEFAULT_PROXY_PORT + " (if hostname is provided)\n" + + " --proxy-user : Username for auth of the Upstream Proxy\n" + + " --proxy-pass : Password for auth of the Upstream Proxy\n" + + " --retry-delay : Delay for the retry of a failed request. Default: " + RdGlobalConfig.RETRY_DELAY + "ms\n" + + " --request-timeout : Hard timeout for the requests being fired by the tool before receiving any response\n" + + " Default: " + RdGlobalConfig.CLIENT_REQ_TIMEOUT + "ms\n" + + " --logs-path : Directory where the '" + constants.LOGS_FOLDER + "' folder will be created\n" + + " for storing logs. Default: Current Working Directory\n" + + " --del-logs : Deletes any existing logs from the " + constants.LOGS_FOLDER + "/ directory and initializes\n" + + " new files for logging\n" + + " --help : Help for Requests Debugger Tool\n" + + " --version : Version of the Requests Debugger Tool\n"; console.log(helpOutput); }, @@ -70,7 +71,7 @@ var CommandLineManager = { if (CommandLineManager.validArgValue(argv[index + 1])) { var probablePort = parseInt(argv[index + 1]); if (!isNaN(probablePort) && (probablePort <= constants.PORTS.MAX) && (probablePort >= constants.PORTS.MIN)) { - RdGlobalConfig.RD_HANDLER_PORT = probablePort; + RdGlobalConfig.RD_HANDLER_PROXY_PORT = probablePort; } else { console.log("\nPort can only range from:", constants.PORTS.MIN, "to:", constants.PORTS.MAX); invalidArgs.add('--port'); @@ -81,6 +82,24 @@ var CommandLineManager = { argv.splice(index, 1); } } + + // port for Requests Debugger Reverse Proxy + index = argv.indexOf('--reverse-proxy-port'); + if (index !== -1) { + if (CommandLineManager.validArgValue(argv[index + 1])) { + var probableReverseProxyPort = parseInt(argv[index + 1]); + if (!isNaN(probableReverseProxyPort) && (probableReverseProxyPort <= constants.PORTS.MAX) && (probableReverseProxyPort >= constants.PORTS.MIN)) { + RdGlobalConfig.RD_HANDLER_REVERSE_PROXY_PORT = probablePort; + } else { + console.log("\nPort can only range from:", constants.PORTS.MIN, "to:", constants.PORTS.MAX); + invalidArgs.add('--reverse-proxy-port'); + } + argv.splice(index, 2); + } else { + invalidArgs.add('--reverse-proxy-port'); + argv.splice(index, 1); + } + } // delay for retries in case of request failures index = argv.indexOf('--retry-delay'); @@ -144,7 +163,7 @@ var CommandLineManager = { if (CommandLineManager.validArgValue(argv[index + 1])) { var host = argv[index + 1]; if(host.lastIndexOf("http") !== 0){ - host = `http://${host}`; + host = 'http://' + host; } RdGlobalConfig.proxy = RdGlobalConfig.proxy || {}; RdGlobalConfig.proxy.host = host; diff --git a/src/requestLib.js b/src/requestLib.js index 70f11b0..3e2553a 100644 --- a/src/requestLib.js +++ b/src/requestLib.js @@ -26,9 +26,9 @@ var RequestLib = { requestOptions.agent = RdGlobalConfig.SCHEME === 'http' ? httpKeepAliveAgent : httpsKeepAliveAgent; if(RdGlobalConfig.proxy) { if (!httpProxyAgent && !httpsProxyAgent) { - var proxyOpts = url.parse(`${RdGlobalConfig.proxy.host}:${RdGlobalConfig.proxy.port}`); + var proxyOpts = url.parse(RdGlobalConfig.proxy.host + ":" +RdGlobalConfig.proxy.port); if(RdGlobalConfig.proxy.username && RdGlobalConfig.proxy.password) { - proxyOpts.auth = `${RdGlobalConfig.proxy.username}:${RdGlobalConfig.proxy.password}`; + proxyOpts.auth = RdGlobalConfig.proxy.username + ":" + RdGlobalConfig.proxy.password; } httpProxyAgent = HttpProxyAgent(proxyOpts); httpsProxyAgent = HttpsProxyAgent(proxyOpts); @@ -61,8 +61,9 @@ var RequestLib = { // Log the request that will be initiated on behalf of the client request.on('finish', function () { + var url = RdGlobalConfig.SCHEME + "://" + requestOptions.host + requestOptions.path; RdGlobalConfig.reqLogger.info(constants.TOPICS.TOOL_REQUEST_WITH_RETRIES + retries, - clientRequest.method + ' ' + requestOptions.path, false, Object.assign({}, params.furtherRequestOptions, { + clientRequest.method + ' ' + url, false, Object.assign({}, params.furtherRequestOptions, { data: Buffer.concat(params.request.data).toString() }), clientRequest.id); diff --git a/src/requestsDebugger.js b/src/requestsDebugger.js index ce9cb5c..e391948 100644 --- a/src/requestsDebugger.js +++ b/src/requestsDebugger.js @@ -11,7 +11,8 @@ var RdGlobalConfig = constants.RdGlobalConfig; var STATIC_MESSAGES = constants.STATIC_MESSAGES; var CommandLineManager = require('./commandLine'); var ConnectivityChecker = require('./connectivity'); -var RdHandler = require('./server'); +var proxy = require('./proxy'); +var reverseProxy = require('./reverseProxy'); var StatsFactory = require('./stats/statsFactory'); var LogManager = require('./logger'); var fs = require('fs'); @@ -110,51 +111,51 @@ var RdTool = { * collection and connectivity checks. Finally, sets up the tool proxy */ start: function () { + var lineLength = 70; CommandLineManager.processArgs(process.argv); - console.log(Utils.formatAndBeautifyLine(STATIC_MESSAGES.STARTING_TOOL, '-', '-', 60, true)); + console.log(Utils.formatAndBeautifyLine(STATIC_MESSAGES.STARTING_TOOL, '-', '-', lineLength, true)); RdTool.initLoggersAndHandlers(); /* eslint-disable indent */ console.log(Utils.formatAndBeautifyLine("Refer '" + RdGlobalConfig.LOGS_DIRECTORY + "' folder for CPU/Network/Memory" + " Stats and Connectivity Checks with BrowserStack components", '', '-', 60, true)); /*eslint-enable indent*/ - - console.log(Utils.formatAndBeautifyLine(STATIC_MESSAGES.CHECK_CPU_STATS, '', '-', 60, true)); + console.log(Utils.formatAndBeautifyLine(STATIC_MESSAGES.CHECK_CPU_STATS, '', '-', lineLength, true)); RdGlobalConfig.cpuLogHandler('Initial CPU', null, function () { - console.log(Utils.formatAndBeautifyLine(STATIC_MESSAGES.CPU_STATS_COLLECTED, '', '-', 60, true)); + console.log(Utils.formatAndBeautifyLine(STATIC_MESSAGES.CPU_STATS_COLLECTED, '', '-', lineLength, true)); }); - console.log(Utils.formatAndBeautifyLine(STATIC_MESSAGES.CHECK_NETWORK_STATS, '', '-', 60, true)); + console.log(Utils.formatAndBeautifyLine(STATIC_MESSAGES.CHECK_NETWORK_STATS, '', '-', lineLength, true)); RdGlobalConfig.networkLogHandler('Initial Network', null, function () { - console.log(Utils.formatAndBeautifyLine(STATIC_MESSAGES.NETWORK_STATS_COLLECTED, '', '-', 60, true)); + console.log(Utils.formatAndBeautifyLine(STATIC_MESSAGES.NETWORK_STATS_COLLECTED, '', '-', lineLength, true)); }); - console.log(Utils.formatAndBeautifyLine(STATIC_MESSAGES.CHECK_MEMORY_STATS, '', '-', 60, true)); + console.log(Utils.formatAndBeautifyLine(STATIC_MESSAGES.CHECK_MEMORY_STATS, '', '-', lineLength, true)); RdGlobalConfig.memLogHandler('Initial Memory', null, function () { - console.log(Utils.formatAndBeautifyLine(STATIC_MESSAGES.MEMORY_STATS_COLLECTED, '', '-', 60, true)); + console.log(Utils.formatAndBeautifyLine(STATIC_MESSAGES.MEMORY_STATS_COLLECTED, '', '-', lineLength, true)); }); - console.log(Utils.formatAndBeautifyLine(STATIC_MESSAGES.CHECK_CONNECTIVITY, '', '-', 60, true)); + console.log(Utils.formatAndBeautifyLine(STATIC_MESSAGES.CHECK_CONNECTIVITY, '', '-', lineLength, true)); RdGlobalConfig.connHandler('Initial Connectivity', null, function () { - console.log(Utils.formatAndBeautifyLine(STATIC_MESSAGES.CONNECTIVITY_CHECKS_DONE, '', '-', 60, true)); + console.log(Utils.formatAndBeautifyLine(STATIC_MESSAGES.CONNECTIVITY_CHECKS_DONE, '', '-', lineLength, true)); }); - RdHandler.startProxyServer(RdGlobalConfig.RD_HANDLER_PROXY_PORT, function (err, result) { + proxy.startProxyServer(RdGlobalConfig.RD_HANDLER_PROXY_PORT, function (err, result) { if (err) { console.log(STATIC_MESSAGES.ERR_STARTING_TOOL, err); console.log('Exiting the Proxy Server...'); process.exit(1); } - console.log(Utils.formatAndBeautifyLine(STATIC_MESSAGES.TOOL_PROXY_STARTED_ON_PORT + result, '', '-', 60, true)); + console.log(Utils.formatAndBeautifyLine(STATIC_MESSAGES.TOOL_PROXY_STARTED_ON_PORT + result, '', '-', lineLength, true)); }); - RdHandler.startReverseProxyServer(RdGlobalConfig.RD_HANDLER_REVERSE_PROXY_PORT, function (err, result) { + reverseProxy.startReverseProxyServer(RdGlobalConfig.RD_HANDLER_REVERSE_PROXY_PORT, function (err, result) { if (err) { console.log(STATIC_MESSAGES.ERR_STARTING_TOOL, err); console.log('Exiting the Reverse Proxy Server...'); process.exit(1); } - console.log(Utils.formatAndBeautifyLine(STATIC_MESSAGES.TOOL_REVESE_PROXY_STARTED_ON_PORT + result, '', '-', 60, true)); + console.log(Utils.formatAndBeautifyLine(STATIC_MESSAGES.TOOL_REVESE_PROXY_STARTED_ON_PORT + result, '', '-', lineLength, true)); }); } From 1deee5a8a02bb698543b845bf5b605dc35666e46 Mon Sep 17 00:00:00 2001 From: Neeraj Shukla Date: Mon, 19 Oct 2020 20:55:58 +0530 Subject: [PATCH 23/44] more refactore --- config/constants.js | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/config/constants.js b/config/constants.js index 9bc5912..a6b5e75 100644 --- a/config/constants.js +++ b/config/constants.js @@ -1,8 +1,8 @@ module.exports.VERSION = '1.1.0'; module.exports.BS_DOMAIN = 'browserstack.com'; -module.exports.HUB_HOST = `hub-cloud.${this.BS_DOMAIN}`; -module.exports.HUB_STATUS_URL = `http://${this.HUB_HOST}/wd/hub/status`; -module.exports.RAILS_AUTOMATE = `http://automate.${this.BS_DOMAIN}`; +module.exports.HUB_HOST = 'hub-cloud.' + this.BS_DOMAIN; +module.exports.HUB_STATUS_URL = 'http://' + this.HUB_HOST + '/wd/hub/status'; +module.exports.RAILS_AUTOMATE = 'http://automate.' + this.BS_DOMAIN; module.exports.CONNECTIVITY_REQ_TIMEOUT = 30000; module.exports.DEFAULT_PROXY_PORT = 3128; module.exports.CUSTOM_ERROR_RESPONSE_CODE = 502; @@ -38,8 +38,8 @@ module.exports.RdGlobalConfig = { }; module.exports.COMMON = Object.freeze({ - PING_HUB: `ping -c 5 ${this.HUB_HOST}`, - PING_AUTOMATE: `ping -c 5 ${this.RAILS_AUTOMATE}` + PING_HUB: 'ping -c 5 ' + this.HUB_HOST, + PING_AUTOMATE: 'ping -c 5 ' + this.RAILS_AUTOMATE }); module.exports.MAC = Object.freeze({ @@ -59,8 +59,8 @@ module.exports.WIN = Object.freeze({ NETSTAT_ROUTING_TABLE: 'netstat -r', IPCONFIG_ALL: 'ipconfig /all', SWAP_USAGE: 'pagefile get AllocatedBaseSize, CurrentUsage', // this is a WMIC command. Prefix with WMIC Path - PING_HUB: `ping -n 5 ${this.HUB_HOST}`, - PING_AUTOMATE: `ping -n 5 ${this.RAILS_AUTOMATE}`, + PING_HUB: 'ping -n 5 ' + this.HUB_HOST, + PING_AUTOMATE: 'ping -n 5 ' + this.RAILS_AUTOMATE, LOAD_PERCENTAGE: 'cpu get loadpercentage', // prefix wmic path }); From 230551775377b70b8cb45917773b67fc5a8f22e5 Mon Sep 17 00:00:00 2001 From: Neeraj Shukla Date: Mon, 19 Oct 2020 23:07:30 +0530 Subject: [PATCH 24/44] proxy and reverse proxy server --- src/proxy.js | 173 +++++++++++++++++++++++++++++++++++++++++++ src/reverseProxy.js | 174 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 347 insertions(+) create mode 100644 src/proxy.js create mode 100644 src/reverseProxy.js diff --git a/src/proxy.js b/src/proxy.js new file mode 100644 index 0000000..73733ef --- /dev/null +++ b/src/proxy.js @@ -0,0 +1,173 @@ +/** + * Server to Intercept the client's requests and handle them on their behalf. + * Initiates stats and connectivity checks when a requests fails. + * It also responds in selenium understandable error when a request fails + * at tool. + */ + +var http = require('http'); +var url = require('url'); +var uuidv4 = require('uuid/v4'); +var Utils = require('./utils'); +var constants = require('../config/constants'); +var ReqLib = require('./requestLib'); + +var RdGlobalConfig = constants.RdGlobalConfig; + +var RdHandler = { + + _requestCounter: 0, + + /** + * Generates the request options for firing requests + * @param {http.IncomingMessage} clientRequest + * @returns {Object} + */ + _generateRequestOptions: function (clientRequest) { + var requestOptions = { + headers: {} + }; + var parsedClientUrl = url.parse(clientRequest.url); + requestOptions.host = parsedClientUrl.host; + requestOptions.port = RdGlobalConfig.SCHEME == 'http' ? 80 : 443; + requestOptions.path = parsedClientUrl.path; + requestOptions.method = clientRequest.method; + requestOptions.headers = clientRequest.headers; + requestOptions.headers['X-Requests-Debugger'] = clientRequest.id; + if (parsedClientUrl.auth) { + requestOptions.headers['Authorization'] = Utils.proxyAuthToBase64(parsedClientUrl.auth); + } + return requestOptions; + }, + + /** + * Frames the error response based on the type of request. + * i.e., if its a request originating for Hub, the response + * is in the format which the client binding would understand. + * @param {Object} parsedRequest + * @param {String} errorMessage + */ + _frameErrorResponse: function (parsedRequest, errorMessage) { + errorMessage += '. ' + constants.STATIC_MESSAGES.REQ_FAILED_MSG; + var parseSessionId = parsedRequest.path.match(/\/wd\/hub\/session\/([a-z0-9]+)\/*/); + if (parseSessionId) { + var sessionId = parseSessionId[1]; + return { + data: { + sessionId: sessionId, + status: 13, + value: { + message: errorMessage, + error: constants.STATIC_MESSAGES.REQ_FAILED_MSG + }, + state: 'error' + }, + statusCode: constants.CUSTOM_ERROR_RESPONSE_CODE + }; + } else { + return { + data: { + message: errorMessage, + error: constants.STATIC_MESSAGES.REQ_FAILED_MSG + }, + statusCode: constants.CUSTOM_ERROR_RESPONSE_CODE + }; + } + }, + + /** + * Handler for incoming requests to Requests Debugger Tool server. + * @param {http.IncomingMessage} clientRequest + * @param {http.ServerResponse} clientResponse + */ + requestHandler: function (clientRequest, clientResponse) { + clientRequest.id = ++RdHandler._requestCounter + '::' + uuidv4(); + var url = clientRequest.url; + var request = { + method: clientRequest.method, + url: clientRequest.url, + headers: clientRequest.headers, + data: [] + }; + RdGlobalConfig.reqLogger.info(constants.TOPICS.CLIENT_REQUEST_START, request.method + ' ' + url, + false, { + headers: request.headers + }, + clientRequest.id); + + var furtherRequestOptions = RdHandler._generateRequestOptions(clientRequest); + + var paramsForRequest = { + request: request, + furtherRequestOptions: furtherRequestOptions + }; + + ReqLib.call(paramsForRequest, clientRequest) + .then(function (response) { + RdGlobalConfig.reqLogger.info(constants.TOPICS.CLIENT_RESPONSE_END, clientRequest.method + ' ' + url + ', Status Code: ' + response.statusCode, + false, { + data: response.data, + headers: response.headers, + }, + clientRequest.id); + + clientResponse.writeHead(response.statusCode, response.headers); + clientResponse.end(response.data); + }) + .catch(function (err) { + RdGlobalConfig.reqLogger.error(err.customTopic || constants.TOPICS.UNEXPECTED_ERROR, clientRequest.method + ' ' + clientRequest.url, + false, { + errorMessage: err.message.toString() + }, + clientRequest.id); + + var errorResponse = RdHandler._frameErrorResponse(furtherRequestOptions, err.message.toString()); + RdGlobalConfig.reqLogger.error(constants.TOPICS.CLIENT_RESPONSE_END, clientRequest.method + ' ' + url + ', Status Code: ' + errorResponse.statusCode, + false, + errorResponse.data, + clientRequest.id); + + clientResponse.writeHead(errorResponse.statusCode); + clientResponse.end(JSON.stringify(errorResponse.data)); + }); + }, + + /** + * Starts the proxy server on the given port + * @param {String|Number} port + * @param {Function} callback + */ + startProxyServer: function (port, callback) { + try { + RdHandler.proxyServer = http.createServer(RdHandler.requestHandler); + RdHandler.proxyServer.listen(port); + RdHandler.proxyServer.on('listening', function () { + callback(null, port); + }); + RdHandler.proxyServer.on('error', function (err) { + callback(err.toString(), null); + }); + } catch (e) { + callback(e.toString(), null); + } + }, + + /** + * Stops the currently running proxy server + * @param {Function} callback + */ + stopProxyServer: function (callback) { + try { + if (RdHandler.proxyServer) { + RdHandler.proxyServer.close(); + RdHandler.proxyServer = null; + } + callback(null, true); + } catch (e) { + callback(e.toString(), null); + } + }, + +}; + +module.exports = RdHandler; diff --git a/src/reverseProxy.js b/src/reverseProxy.js new file mode 100644 index 0000000..7fd7515 --- /dev/null +++ b/src/reverseProxy.js @@ -0,0 +1,174 @@ +/** + * Server to Intercept the client's requests and handle them on their behalf. + * Initiates stats and connectivity checks when a requests fails. + * It also responds in selenium understandable error when a request fails + * at tool. + */ + +var http = require('http'); +var url = require('url'); +var uuidv4 = require('uuid/v4'); +var Utils = require('./utils'); +var constants = require('../config/constants'); +var ReqLib = require('./requestLib'); + +var RdGlobalConfig = constants.RdGlobalConfig; + +var RdHandler = { + + _requestCounter: 0, + + /** + * Generates the request options for firing requests + * @param {http.IncomingMessage} clientRequest + * @returns {Object} + */ + _generateRequestOptions: function (clientRequest) { + var requestOptions = { + headers: {} + }; + var parsedClientUrl = url.parse(clientRequest.url); + requestOptions.port = RdGlobalConfig.SCHEME == 'http' ? 80 : 443; + requestOptions.path = parsedClientUrl.path; + requestOptions.method = clientRequest.method; + requestOptions.headers = clientRequest.headers; + requestOptions.host = constants.HUB_HOST; + requestOptions.headers.host = constants.HUB_HOST; + requestOptions.headers['X-Requests-Debugger'] = clientRequest.id; + if (parsedClientUrl.auth) { + requestOptions.headers['Authorization'] = Utils.proxyAuthToBase64(parsedClientUrl.auth); + } + return requestOptions; + }, + + /** + * Frames the error response based on the type of request. + * i.e., if its a request originating for Hub, the response + * is in the format which the client binding would understand. + * @param {Object} parsedRequest + * @param {String} errorMessage + */ + _frameErrorResponse: function (parsedRequest, errorMessage) { + errorMessage += '. ' + constants.STATIC_MESSAGES.REQ_FAILED_MSG; + var parseSessionId = parsedRequest.path.match(/\/wd\/hub\/session\/([a-z0-9]+)\/*/); + if (parseSessionId) { + var sessionId = parseSessionId[1]; + return { + data: { + sessionId: sessionId, + status: 13, + value: { + message: errorMessage, + error: constants.STATIC_MESSAGES.REQ_FAILED_MSG + }, + state: 'error' + }, + statusCode: constants.CUSTOM_ERROR_RESPONSE_CODE + }; + } else { + return { + data: { + message: errorMessage, + error: constants.STATIC_MESSAGES.REQ_FAILED_MSG + }, + statusCode: constants.CUSTOM_ERROR_RESPONSE_CODE + }; + } + }, + + /** + * Handler for incoming requests to Requests Debugger Tool server. + * @param {http.IncomingMessage} clientRequest + * @param {http.ServerResponse} clientResponse + */ + requestHandler: function (clientRequest, clientResponse) { + clientRequest.id = ++RdHandler._requestCounter + '::' + uuidv4(); + var url = "http://" + clientRequest.headers.host + clientRequest.url; + var request = { + method: clientRequest.method, + url: clientRequest.url, + headers: clientRequest.headers, + data: [] + }; + RdGlobalConfig.reqLogger.info(constants.TOPICS.CLIENT_REQUEST_START, request.method + ' ' + url, + false, { + headers: request.headers + }, + clientRequest.id); + + var furtherRequestOptions = RdHandler._generateRequestOptions(clientRequest); + + var paramsForRequest = { + request: request, + furtherRequestOptions: furtherRequestOptions + }; + + ReqLib.call(paramsForRequest, clientRequest) + .then(function (response) { + RdGlobalConfig.reqLogger.info(constants.TOPICS.CLIENT_RESPONSE_END, clientRequest.method + ' ' + url + ', Status Code: ' + response.statusCode, + false, { + data: response.data, + headers: response.headers, + }, + clientRequest.id); + + clientResponse.writeHead(response.statusCode, response.headers); + clientResponse.end(response.data); + }) + .catch(function (err) { + RdGlobalConfig.reqLogger.error(err.customTopic || constants.TOPICS.UNEXPECTED_ERROR, clientRequest.method + ' ' + clientRequest.url, + false, { + errorMessage: err.message.toString() + }, + clientRequest.id); + + var errorResponse = RdHandler._frameErrorResponse(furtherRequestOptions, err.message.toString()); + RdGlobalConfig.reqLogger.error(constants.TOPICS.CLIENT_RESPONSE_END, clientRequest.method + ' ' + url + ', Status Code: ' + errorResponse.statusCode, + false, + errorResponse.data, + clientRequest.id); + + clientResponse.writeHead(errorResponse.statusCode); + clientResponse.end(JSON.stringify(errorResponse.data)); + }); + }, + + /** + * Starts the reverse proxy server on the given port + * @param {String|Number} port + * @param {Function} callback + */ + startReverseProxyServer: function (port, callback) { + try { + RdHandler.reverseProxyServer = http.createServer(RdHandler.requestHandler); + RdHandler.reverseProxyServer.listen(port); + RdHandler.reverseProxyServer.on('listening', function () { + callback(null, port); + }); + RdHandler.reverseProxyServer.on('error', function (err) { + callback(err.toString(), null); + }); + } catch (e) { + callback(e.toString(), null); + } + }, + + /** + * Stops the currently running reverse proxy server + * @param {Function} callback + */ + stopReverseProxyServer: function (callback) { + try { + if (RdHandler.reverseProxyServer) { + RdHandler.reverseProxyServer.close(); + RdHandler.reverseProxyServer = null; + } + callback(null, true); + } catch (e) { + callback(e.toString(), null); + } + } + +}; + +module.exports = RdHandler; From 0f314766aa73b33fb54350faaf381bbc6baf859e Mon Sep 17 00:00:00 2001 From: Neeraj Shukla Date: Tue, 20 Oct 2020 10:48:06 +0530 Subject: [PATCH 25/44] tests --- package.json | 2 +- src/commandLine.js | 2 +- src/requestLib.js | 4 +- src/server.js | 233 --------------------------------------- test/commandLine.test.js | 93 +++++++++++++++- 5 files changed, 95 insertions(+), 239 deletions(-) delete mode 100644 src/server.js diff --git a/package.json b/package.json index 4fa720d..254a817 100644 --- a/package.json +++ b/package.json @@ -6,7 +6,7 @@ "scripts": { "start": "NODE_ENV=prod node src/requestsDebugger.js", "lint": "./node_modules/.bin/eslint 'src/*' 'test/*' 'config/*.js'", - "test": "npm run lint; NODE_ENV=test nyc --reporter=html ./node_modules/mocha/bin/mocha 'test/**/*.test.js'", + "test": "npm run lint; NODE_ENV=test ./node_modules/nyc/bin/nyc.js --reporter=html ./node_modules/mocha/bin/mocha 'test/**/*.test.js'", "build:mac": "npm install; ./node_modules/pkg/lib-es5/bin.js -t node4-macos-x64 src/requestsDebugger.js; mv requestsDebugger RequestsDebugger-Mac", "build:linux-x86": "npm install; ./node_modules/pkg/lib-es5/bin.js -t node4-linux-x86 src/requestsDebugger.js; mv requestsDebugger RequestsDebugger-Linux-x86", "build:linux-x64": "npm install; ./node_modules/pkg/lib-es5/bin.js -t node4-linux-x64 src/requestsDebugger.js; mv requestsDebugger RequestsDebugger-Linux-x64", diff --git a/src/commandLine.js b/src/commandLine.js index 8063166..6ea853a 100644 --- a/src/commandLine.js +++ b/src/commandLine.js @@ -89,7 +89,7 @@ var CommandLineManager = { if (CommandLineManager.validArgValue(argv[index + 1])) { var probableReverseProxyPort = parseInt(argv[index + 1]); if (!isNaN(probableReverseProxyPort) && (probableReverseProxyPort <= constants.PORTS.MAX) && (probableReverseProxyPort >= constants.PORTS.MIN)) { - RdGlobalConfig.RD_HANDLER_REVERSE_PROXY_PORT = probablePort; + RdGlobalConfig.RD_HANDLER_REVERSE_PROXY_PORT = probableReverseProxyPort; } else { console.log("\nPort can only range from:", constants.PORTS.MIN, "to:", constants.PORTS.MAX); invalidArgs.add('--reverse-proxy-port'); diff --git a/src/requestLib.js b/src/requestLib.js index 3e2553a..8609892 100644 --- a/src/requestLib.js +++ b/src/requestLib.js @@ -30,8 +30,8 @@ var RequestLib = { if(RdGlobalConfig.proxy.username && RdGlobalConfig.proxy.password) { proxyOpts.auth = RdGlobalConfig.proxy.username + ":" + RdGlobalConfig.proxy.password; } - httpProxyAgent = HttpProxyAgent(proxyOpts); - httpsProxyAgent = HttpsProxyAgent(proxyOpts); + httpProxyAgent = new HttpProxyAgent(proxyOpts); + httpsProxyAgent = new HttpsProxyAgent(proxyOpts); } requestOptions.agent = RdGlobalConfig.SCHEME === 'http' ? httpProxyAgent : httpsProxyAgent; } diff --git a/src/server.js b/src/server.js deleted file mode 100644 index f82cd88..0000000 --- a/src/server.js +++ /dev/null @@ -1,233 +0,0 @@ -/** - * Server to Intercept the client's requests and handle them on their behalf. - * Initiates stats and connectivity checks when a requests fails. - * It also responds in selenium understandable error when a request fails - * at tool. - */ - -var http = require('http'); -var url = require('url'); -var uuidv4 = require('uuid/v4'); -var Utils = require('./utils'); -var constants = require('../config/constants'); -var ReqLib = require('./requestLib'); - -var RdGlobalConfig = constants.RdGlobalConfig; - -var RdHandler = { - - _requestCounter: 0, - - /** - * Generates the request options for firing requests - * @param {http.IncomingMessage} clientRequest - * @param {String} serverType - * @returns {Object} - */ - _generateRequestOptions: function (clientRequest, serverType) { - var requestOptions = { - headers: {} - }; - var parsedClientUrl = url.parse(clientRequest.url); - requestOptions.host = parsedClientUrl.host; - requestOptions.port = RdGlobalConfig.SCHEME == 'http' ? 80 : 443; - requestOptions.path = parsedClientUrl.path; - requestOptions.method = clientRequest.method; - requestOptions.headers = clientRequest.headers; - if(serverType == constants.SERVER_TYPES.REVERSE_PROXY){ - requestOptions.host = constants.HUB_HOST; - requestOptions.headers.host = constants.HUB_HOST; - } - requestOptions.headers['X-Requests-Debugger'] = clientRequest.id; - if (parsedClientUrl.auth) { - requestOptions.headers['Authorization'] = Utils.proxyAuthToBase64(parsedClientUrl.auth); - } - return requestOptions; - }, - - /** - * Frames the error response based on the type of request. - * i.e., if its a request originating for Hub, the response - * is in the format which the client binding would understand. - * @param {Object} parsedRequest - * @param {String} errorMessage - */ - _frameErrorResponse: function (parsedRequest, errorMessage) { - errorMessage += '. ' + constants.STATIC_MESSAGES.REQ_FAILED_MSG; - var parseSessionId = parsedRequest.path.match(/\/wd\/hub\/session\/([a-z0-9]+)\/*/); - if (parseSessionId) { - var sessionId = parseSessionId[1]; - return { - data: { - sessionId: sessionId, - status: 13, - value: { - message: errorMessage, - error: constants.STATIC_MESSAGES.REQ_FAILED_MSG - }, - state: 'error' - }, - statusCode: constants.CUSTOM_ERROR_RESPONSE_CODE - }; - } else { - return { - data: { - message: errorMessage, - error: constants.STATIC_MESSAGES.REQ_FAILED_MSG - }, - statusCode: constants.CUSTOM_ERROR_RESPONSE_CODE - }; - } - }, - - /** - * Handler for incoming requests to Requests Debugger Tool server. - * @param {http.IncomingMessage} clientRequest - * @param {http.ServerResponse} clientResponse - * @param {String} serverType - */ - requestHandler: function (clientRequest, clientResponse, serverType) { - clientRequest.id = ++RdHandler._requestCounter + '::' + uuidv4(); - var path = url.parse(clientRequest.url).path; - var request = { - method: clientRequest.method, - url: clientRequest.url, - headers: clientRequest.headers, - data: [] - }; - RdGlobalConfig.reqLogger.info(constants.TOPICS.CLIENT_REQUEST_START, request.method + ' ' + path, - false, { - headers: request.headers - }, - clientRequest.id); - - var furtherRequestOptions = RdHandler._generateRequestOptions(clientRequest, serverType); - - var paramsForRequest = { - request: request, - furtherRequestOptions: furtherRequestOptions - }; - - ReqLib.call(paramsForRequest, clientRequest) - .then(function (response) { - RdGlobalConfig.reqLogger.info(constants.TOPICS.CLIENT_RESPONSE_END, clientRequest.method + ' ' + path + ', Status Code: ' + response.statusCode, - false, { - data: response.data, - headers: response.headers, - }, - clientRequest.id); - - clientResponse.writeHead(response.statusCode, response.headers); - clientResponse.end(response.data); - }) - .catch(function (err) { - RdGlobalConfig.reqLogger.error(err.customTopic || constants.TOPICS.UNEXPECTED_ERROR, clientRequest.method + ' ' + clientRequest.url, - false, { - errorMessage: err.message.toString() - }, - clientRequest.id); - - var errorResponse = RdHandler._frameErrorResponse(furtherRequestOptions, err.message.toString()); - RdGlobalConfig.reqLogger.error(constants.TOPICS.CLIENT_RESPONSE_END, clientRequest.method + ' ' + path + ', Status Code: ' + errorResponse.statusCode, - false, - errorResponse.data, - clientRequest.id); - - clientResponse.writeHead(errorResponse.statusCode); - clientResponse.end(JSON.stringify(errorResponse.data)); - }); - }, - - /** - * Handler for incoming requests to Requests Debugger Tool proxy server. - * @param {http.IncomingMessage} clientRequest - * @param {http.ServerResponse} clientResponse - */ - proxyRequestHandler: function (clientRequest, clientResponse) { - RdHandler.requestHandler(clientRequest, clientResponse, constants.SERVER_TYPES.PROXY); - }, - - /** - * Handler for incoming requests to Requests Debugger Tool reverse proxy server. - * @param {http.IncomingMessage} clientRequest - * @param {http.ServerResponse} clientResponse - */ - reverseProxyRequestHandler: function (clientRequest, clientResponse) { - RdHandler.requestHandler(clientRequest, clientResponse, constants.SERVER_TYPES.REVERSE_PROXY); - }, - - /** - * Starts the proxy server on the given port - * @param {String|Number} port - * @param {Function} callback - */ - startProxyServer: function (port, callback) { - try { - RdHandler.proxyServer = http.createServer(RdHandler.proxyRequestHandler); - RdHandler.proxyServer.listen(port); - RdHandler.proxyServer.on('listening', function () { - callback(null, port); - }); - RdHandler.proxyServer.on('error', function (err) { - callback(err.toString(), null); - }); - } catch (e) { - callback(e.toString(), null); - } - }, - - /** - * Starts the reverse proxy server on the given port - * @param {String|Number} port - * @param {Function} callback - */ - startReverseProxyServer: function (port, callback) { - try { - RdHandler.reverseProxyServer = http.createServer(RdHandler.reverseProxyRequestHandler); - RdHandler.reverseProxyServer.listen(port); - RdHandler.reverseProxyServer.on('listening', function () { - callback(null, port); - }); - RdHandler.reverseProxyServer.on('error', function (err) { - callback(err.toString(), null); - }); - } catch (e) { - callback(e.toString(), null); - } - }, - - /** - * Stops the currently running proxy server - * @param {Function} callback - */ - stopProxyServer: function (callback) { - try { - if (RdHandler.proxyServer) { - RdHandler.proxyServer.close(); - RdHandler.proxyServer = null; - } - callback(null, true); - } catch (e) { - callback(e.toString(), null); - } - }, - - /** - * Stops the currently running reverse proxy server - * @param {Function} callback - */ - stopReverseProxyServer: function (callback) { - try { - if (RdHandler.reverseProxyServer) { - RdHandler.reverseProxyServer.close(); - RdHandler.reverseProxyServer = null; - } - callback(null, true); - } catch (e) { - callback(e.toString(), null); - } - } - -}; - -module.exports = RdHandler; diff --git a/test/commandLine.test.js b/test/commandLine.test.js index 833b4d7..08f6ac3 100644 --- a/test/commandLine.test.js +++ b/test/commandLine.test.js @@ -212,11 +212,11 @@ describe('CommandLineManager', function () { // --port it("sets the port of Requests Debugger Tool Proxy using the '--port' argument", function () { - argv = argv.concat(['--port', '9098']); + argv = argv.concat(['--port', '9687']); sinon.stub(console, 'log'); CommandLineManager.processArgs(argv); console.log.restore(); - expect(RdGlobalConfig.RD_HANDLER_PORT).to.eql(9098); + expect(RdGlobalConfig.RD_HANDLER_PROXY_PORT).to.eql(9687); }); it('Uses the default port of Requests Debugger Tool Proxy if not provided via arguments', function () { @@ -254,6 +254,95 @@ describe('CommandLineManager', function () { sinon.assert.called(process.exit); }); + // --reverse-proxy-port + it("sets the port of Requests Debugger Tool Reverse Proxy using the '--reverse-proxy-port' argument", function () { + argv = argv.concat(['--reverse-proxy-port', '9688']); + CommandLineManager.processArgs(argv); + expect(RdGlobalConfig.RD_HANDLER_REVERSE_PROXY_PORT).to.eql(9688); + }); + + it('Uses the default reverse port of Requests Debugger Tool Proxy if not provided via arguments', function () { + var portBeforeParsing = RdGlobalConfig.RD_HANDLER_REVERSE_PROXY_PORT; + CommandLineManager.processArgs(argv); + expect(RdGlobalConfig.RD_HANDLER_REVERSE_PROXY_PORT).to.eql(portBeforeParsing); + }); + + it("exits with invalid args if reverse port provided doesn't lie in the Max Min Range", function () { + argv = argv.concat(['--reverse-proxy-port', '99999']); + sinon.stub(console, 'log'); + CommandLineManager.processArgs(argv); + sinon.assert.calledWith(console.log, '\nInvalid Argument(s): ', '--reverse-proxy-port', '\n'); + console.log.restore(); + sinon.assert.called(process.exit); + }); + + it('exits with invalid args if the reverse port provided is not a number', function () { + argv = argv.concat(['--reverse-proxy-port', 'random string']); + sinon.stub(console, 'log'); + CommandLineManager.processArgs(argv); + sinon.assert.calledWith(console.log, '\nInvalid Argument(s): ', '--reverse-proxy-port', '\n'); + console.log.restore(); + sinon.assert.called(process.exit); + }); + + it('exits with invalid args if the reverse port arg is provided without any value', function () { + argv = argv.concat(['--reverse-proxy-port']); + sinon.stub(console, 'log'); + CommandLineManager.processArgs(argv); + sinon.assert.calledWith(console.log, '\nInvalid Argument(s): ', '--reverse-proxy-port', '\n'); + console.log.restore(); + sinon.assert.called(process.exit); + }); + + it('exits with invalid args if the reverse port arg is provided without any value', function () { + argv = argv.concat(['--reverse-proxy-port']); + sinon.stub(console, 'log'); + CommandLineManager.processArgs(argv); + sinon.assert.calledWith(console.log, '\nInvalid Argument(s): ', '--reverse-proxy-port', '\n'); + console.log.restore(); + sinon.assert.called(process.exit); + }); + + // --scheme + it('Uses the default https scheme for Requests Debugger Tool Proxy if not provided via arguments', function () { + sinon.stub(console, 'log'); + var schemeBeforeParsing = RdGlobalConfig.SCHEME; + CommandLineManager.processArgs(argv); + console.log.restore(); + expect(RdGlobalConfig.SCHEME).to.eql(schemeBeforeParsing); + }); + + it("sets the port of Requests Debugger Tool Reverse scheme to http using the '--scheme' argument", function () { + argv = argv.concat(['--scheme', 'http']); + CommandLineManager.processArgs(argv); + expect(RdGlobalConfig.SCHEME).to.eql('http'); + }); + + it('exits with invalid args if the scheme arg is provided with wrong value', function () { + argv = argv.concat(['--scheme', 'random string']); + sinon.stub(console, 'log'); + CommandLineManager.processArgs(argv); + sinon.assert.calledWith(console.log, '\nInvalid Argument(s): ', '--scheme', '\n'); + console.log.restore(); + sinon.assert.called(process.exit); + }); + + it('exits with invalid args if the scheme arg is provided without any value', function () { + argv = argv.concat(['--scheme']); + sinon.stub(console, 'log'); + CommandLineManager.processArgs(argv); + sinon.assert.calledWith(console.log, '\nInvalid Argument(s): ', '--scheme', '\n'); + console.log.restore(); + sinon.assert.called(process.exit); + }); + + // --reverse-proxy-port + it("sets the port of Requests Debugger Tool Reverse scheme to https using the '--scheme' argument", function () { + argv = argv.concat(['--scheme', 'https']); + CommandLineManager.processArgs(argv); + expect(RdGlobalConfig.SCHEME).to.eql('https'); + }); + // --request-timeout it("sets the timeout for the request being fired from the tool using the arg --request-timeout", function () { argv = argv.concat(['--request-timeout', '200000']); From 7b9b8c6e6d22a1571250724c582a7c2b2953905a Mon Sep 17 00:00:00 2001 From: Neeraj Shukla Date: Tue, 20 Oct 2020 10:50:20 +0530 Subject: [PATCH 26/44] lint fix and test --- src/commandLine.js | 2 +- test/{server.test.js => proxy.test.js} | 28 +++--- test/reverseProxy.test.js | 124 +++++++++++++++++++++++++ 3 files changed, 141 insertions(+), 13 deletions(-) rename test/{server.test.js => proxy.test.js} (79%) create mode 100644 test/reverseProxy.test.js diff --git a/src/commandLine.js b/src/commandLine.js index 6ea853a..36b352a 100644 --- a/src/commandLine.js +++ b/src/commandLine.js @@ -145,6 +145,7 @@ var CommandLineManager = { if (!(scheme === 'http' || scheme === 'https')){ console.log("\nScheme can only be http/https"); invalidArgs.add('--scheme'); + argv.splice(index, 2); } else { RdGlobalConfig.SCHEME = scheme; @@ -155,7 +156,6 @@ var CommandLineManager = { argv.splice(index, 1); } } - // process proxy host index = argv.indexOf('--proxy-host'); diff --git a/test/server.test.js b/test/proxy.test.js similarity index 79% rename from test/server.test.js rename to test/proxy.test.js index f489c38..bc76c6b 100644 --- a/test/server.test.js +++ b/test/proxy.test.js @@ -1,7 +1,7 @@ var constants = require('../config/constants'); var RdGlobalConfig = constants.RdGlobalConfig; var nock = require('nock'); -var RdHandler = require('../src/server'); +var RdHandler = require('../src/proxy'); var http = require('http'); var assert = require('chai').assert; var testHelper = require('./testHelper'); @@ -9,35 +9,39 @@ var testHelper = require('./testHelper'); describe('RdHandler', function () { context('Proxy Server', function () { - + var originalScheme; before(function (done) { this.timeout = 5000; testHelper.initializeDummyLoggers(); testHelper.initializeDummyHandlers(); + originalScheme = RdGlobalConfig.SCHEME; + RdGlobalConfig.SCHEME = 'http'; - RdHandler.startServer(RdGlobalConfig.RD_HANDLER_PORT, function (port) { - console.log('Test Network Utility Server Started on Port: ', port); + RdHandler.startProxyServer(RdGlobalConfig.RD_HANDLER_PROXY_PORT, function (port) { + console.log('Test Network Utility Proxy Server Started on Port: ', port); done(); }); }); after(function (done) { this.timeout = 5000; - RdHandler.stopServer(function () { + RdHandler.stopProxyServer(function () { done(); }); testHelper.deleteLoggers(); testHelper.deleteHandlers(); nock.cleanAll(); + RdGlobalConfig.SCHEME.restore(); + RdGlobalConfig.SCHEME = originalScheme; }); it('Requests on behalf of the client and returns the response', function (done) { this.timeout = 5000; - testHelper.nockGetRequest(constants.HUB_STATUS_URL, 'https', null, 200); + testHelper.nockGetRequest(constants.HUB_STATUS_URL, 'http', null, 200); var reqOptions = { method: 'GET', - host: 'localhost', - port: RdGlobalConfig.RD_HANDLER_PORT, + host: constants.HUB_HOST, + port: constants.RD_HANDLER_PROXY_PORT, headers: {}, path: '/wd/hub/status' }; @@ -60,13 +64,13 @@ describe('RdHandler', function () { it('Requests on behalf of the client via external proxy and returns the response', function (done) { this.timeout = 5000; - testHelper.nockGetRequest(constants.HUB_STATUS_URL, 'https', null, 200); + testHelper.nockGetRequest(constants.HUB_STATUS_URL, 'http', null, 200); testHelper.initializeDummyProxy(); testHelper.nockProxyUrl(RdGlobalConfig.proxy, 'http', 'hub', null, 200); var reqOptions = { method: 'GET', host: 'localhost', - port: RdGlobalConfig.RD_HANDLER_PORT, + port: RdGlobalConfig.RD_HANDLER_PROXY_PORT, headers: {}, path: constants.HUB_STATUS_URL }; @@ -91,12 +95,12 @@ describe('RdHandler', function () { it('Requests on behalf of the client via external proxy and returns the response even if request by tool fails', function (done) { this.timeout = 5000; for (var i = 0; i <= constants.MAX_RETRIES; i++) { - testHelper.nockGetRequestWithError(constants.HUB_STATUS_URL, 'https'); + testHelper.nockGetRequestWithError(constants.HUB_STATUS_URL, 'http'); } var reqOptions = { method: 'GET', host: 'localhost', - port: RdGlobalConfig.RD_HANDLER_PORT, + port: RdGlobalConfig.RD_HANDLER_PROXY_PORT, headers: {}, path: constants.HUB_STATUS_URL }; diff --git a/test/reverseProxy.test.js b/test/reverseProxy.test.js new file mode 100644 index 0000000..57e57e1 --- /dev/null +++ b/test/reverseProxy.test.js @@ -0,0 +1,124 @@ +var constants = require('../config/constants'); +var RdGlobalConfig = constants.RdGlobalConfig; +var nock = require('nock'); +var RdHandler = require('../src/reverseProxy'); +var http = require('http'); +var assert = require('chai').assert; +var testHelper = require('./testHelper'); + + +describe('RdHandler', function () { + context('Reverse Proxy Server', function () { + var originalScheme; + before(function (done) { + this.timeout = 5000; + testHelper.initializeDummyLoggers(); + testHelper.initializeDummyHandlers(); + originalScheme = RdGlobalConfig.SCHEME; + RdGlobalConfig.SCHEME = 'https'; + + RdHandler.startReverseProxyServer(RdGlobalConfig.RD_HANDLER_REVERSE_PROXY_PORT, function (port) { + console.log('Test Network Utility Reverse Proxy Server Started on Port: ', port); + done(); + }); + }); + + after(function (done) { + this.timeout = 5000; + RdHandler.stopReverseProxyServer(function () { + done(); + }); + testHelper.deleteLoggers(); + testHelper.deleteHandlers(); + nock.cleanAll(); + RdGlobalConfig.SCHEME.restore(); + RdGlobalConfig.SCHEME = originalScheme; + }); + + it('Requests reverse proxy on behalf of the client and returns the response', function (done) { + this.timeout = 5000; + testHelper.nockGetRequest(constants.HUB_STATUS_URL, 'https', null, 200); + var reqOptions = { + method: 'GET', + host: 'localhost', + port: RdGlobalConfig.RD_HANDLER_REVERSE_PROXY_PORT, + headers: {}, + path: '/wd/hub/status' + }; + + var responseData = []; + var request = http.request(reqOptions, function (response) { + + response.on('data', function (chunk) { + responseData.push(chunk); + }); + + response.on('end', function () { + assert(Buffer.concat(responseData).toString() === '{"data":"value"}'); + done(); + }); + }); + + request.end(); + }); + + it('Requests reverse proxy on behalf of the client via external proxy and returns the response', function (done) { + this.timeout = 5000; + testHelper.nockGetRequest(constants.HUB_STATUS_URL, 'https', null, 200); + testHelper.initializeDummyProxy(); + testHelper.nockProxyUrl(RdGlobalConfig.proxy, 'https', 'hub', null, 200); + var reqOptions = { + method: 'GET', + host: 'localhost', + port: RdGlobalConfig.RD_HANDLER_REVERSE_PROXY_PORT, + headers: {}, + path: '/wd/hub/status' + }; + + var responseData = []; + var request = http.request(reqOptions, function (response) { + + response.on('data', function (chunk) { + responseData.push(chunk); + }); + + response.on('end', function () { + assert(Buffer.concat(responseData).toString() === '{"data":"value"}'); + done(); + }); + }); + + request.end(); + testHelper.deleteProxy(); + }); + + it('Requests reverse proxy on behalf of the client via external proxy and returns the response even if request by tool fails', function (done) { + this.timeout = 5000; + for (var i = 0; i <= constants.MAX_RETRIES; i++) { + testHelper.nockGetRequestWithError(constants.HUB_STATUS_URL, 'https'); + } + var reqOptions = { + method: 'GET', + host: 'localhost', + port: RdGlobalConfig.RD_HANDLER_REVERSE_PROXY_PORT, + headers: {}, + path: '/wd/hub/status' + }; + + var responseData = []; + var request = http.request(reqOptions, function (response) { + + response.on('data', function (chunk) { + responseData.push(chunk); + }); + + response.on('end', function () { + assert(Buffer.concat(responseData).toString() === '{"message":"Error: something terrible. Request Failed At Requests Debugger","error":"Request Failed At Requests Debugger"}'); + done(); + }); + }); + + request.end(); + }); + }); +}); From 2c70cbf20f1308edd966b7c2664386ab6dcb0e30 Mon Sep 17 00:00:00 2001 From: Neeraj Shukla Date: Tue, 20 Oct 2020 11:07:12 +0530 Subject: [PATCH 27/44] readme and refactor --- README.md | 3 ++- config/constants.js | 8 ++------ src/requestsDebugger.js | 4 ++-- 3 files changed, 6 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 4d52ba6..1f6fd26 100644 --- a/README.md +++ b/README.md @@ -13,6 +13,7 @@ - Start Requests Debugger with the required arguments: `npm run start -- `. - Supported `args`: - `--port `: Port on which the Requests Debugger Tool's Proxy will run. Default: 9687 + - `--reverse-proxy-port `: Port on which the Requests Debugger Tool's Reverse Proxy will run. Default: 9688 - `--scheme `: Scheme for requests to browserstack. Default: https" - `--proxy-host `: Hostname of the Upstream Proxy - `--proxy-port `: Port of the Upstream Proxy. Default: 3128 (if hostname is provided) @@ -41,7 +42,7 @@ - Set your system's env variable `http_proxy=localhost:9687` and Ruby's Selenium Client Binding will pick the value. Or, - Run you test by giving the environment variable to your command itself, i.e. `http_proxy=localhost:9687 ruby ` - Similarly, you can also set proxy for other client bindings. -- To use tool as reverse proxy, you will have to replace hub-cloud.browserstack.com in hub url with `localhost:9687`. +- To use tool as reverse proxy, you will have to replace hub-cloud.browserstack.com in hub url with `localhost:9688`. ## Steps to build the executables - Linux diff --git a/config/constants.js b/config/constants.js index a6b5e75..8f4f6e4 100644 --- a/config/constants.js +++ b/config/constants.js @@ -13,11 +13,6 @@ module.exports.PORTS = { MIN: 1 }; -module.exports.SERVER_TYPES = { - PROXY: "proxy", - REVERSE_PROXY: "reverse_proxy" -}; - module.exports.PROTOCOL_REGEX = /(^\w+:|^)\/\//; module.exports.LOGS = Object.freeze({ @@ -88,7 +83,8 @@ module.exports.STATIC_MESSAGES = Object.freeze({ CHECK_NETWORK_STATS: 'Stats : Checking Network Stats', CHECK_MEMORY_STATS: 'Stats : Checking Memory Stats', CHECK_CONNECTIVITY: 'Checks : Checking Connectivity With BrowserStack', - ERR_STARTING_TOOL: 'Error in starting Requests Debugger Tool: ', + ERR_STARTING_TOOL_PROXY: 'Error in starting Requests Debugger Tool Proxy: ', + ERR_STARTING_TOOL_REVERSE_PROXY: 'Error in starting Requests Debugger Tool Reverse Proxy: ', TOOL_PROXY_STARTED_ON_PORT: 'Requests Debugger Tool Proxy Server Started on Port: ', TOOL_REVESE_PROXY_STARTED_ON_PORT: 'Requests Debugger Tool Reverse Proxy Server Started on Port: ', CPU_STATS_COLLECTED: 'Stats : Initial CPU Stats Collected', diff --git a/src/requestsDebugger.js b/src/requestsDebugger.js index e391948..a9c2a54 100644 --- a/src/requestsDebugger.js +++ b/src/requestsDebugger.js @@ -142,7 +142,7 @@ var RdTool = { proxy.startProxyServer(RdGlobalConfig.RD_HANDLER_PROXY_PORT, function (err, result) { if (err) { - console.log(STATIC_MESSAGES.ERR_STARTING_TOOL, err); + console.log(STATIC_MESSAGES.ERR_STARTING_TOOL_PROXY, err); console.log('Exiting the Proxy Server...'); process.exit(1); } @@ -151,7 +151,7 @@ var RdTool = { reverseProxy.startReverseProxyServer(RdGlobalConfig.RD_HANDLER_REVERSE_PROXY_PORT, function (err, result) { if (err) { - console.log(STATIC_MESSAGES.ERR_STARTING_TOOL, err); + console.log(STATIC_MESSAGES.ERR_STARTING_TOOL_REVERSE_PROXY, err); console.log('Exiting the Reverse Proxy Server...'); process.exit(1); } From d79969760dcce21a72c1bc2f400d3c05b6eb226e Mon Sep 17 00:00:00 2001 From: Neeraj Shukla Date: Tue, 20 Oct 2020 14:51:23 +0530 Subject: [PATCH 28/44] bug fix --- .eslintrc.js | 3 ++- src/commandLine.js | 2 +- src/connectivity.js | 38 ++++++++++++++++++++++++++++++++++---- src/requestLib.js | 4 ++-- 4 files changed, 39 insertions(+), 8 deletions(-) diff --git a/.eslintrc.js b/.eslintrc.js index 79f8658..1ae3d06 100644 --- a/.eslintrc.js +++ b/.eslintrc.js @@ -16,7 +16,8 @@ module.exports = { "indent": ["error", 2], "linebreak-style": [ "error", "unix"], "semi": ["error", "always"], - "eol-last": ["error", "always"] + "eol-last": ["error", "always"], + "keyword-spacing": [1] }, "overrides": [{ "files": ["src/requestsDebugger.js", "src/commandLine.js", "test/**/*.test.js"], diff --git a/src/commandLine.js b/src/commandLine.js index 36b352a..d79f86a 100644 --- a/src/commandLine.js +++ b/src/commandLine.js @@ -162,7 +162,7 @@ var CommandLineManager = { if (index !== -1) { if (CommandLineManager.validArgValue(argv[index + 1])) { var host = argv[index + 1]; - if(host.lastIndexOf("http") !== 0){ + if (host.lastIndexOf("http") !== 0){ host = 'http://' + host; } RdGlobalConfig.proxy = RdGlobalConfig.proxy || {}; diff --git a/src/connectivity.js b/src/connectivity.js index 529fd1a..c993565 100644 --- a/src/connectivity.js +++ b/src/connectivity.js @@ -12,6 +12,8 @@ var constants = require('../config/constants'); var RdGlobalConfig = constants.RdGlobalConfig; var Utils = require('./utils'); var https = require('https'); +var HttpProxyAgent = require('http-proxy-agent'); +var HttpsProxyAgent = require('https-proxy-agent'); /** * Fires the requests to perform connectivity checks @@ -116,6 +118,21 @@ var ConnectivityChecker = { }); }, + httpsToHubWithProxy: function (callback) { + var requestUrl = constants.HUB_STATUS_URL; + var requestOptions = ConnectivityChecker.reqOpsWithProxy(requestUrl, 'https'); + fireRequest(requestOptions, 'https', 'HTTPS Request To ' + requestUrl + ' With Proxy', [200], function (response) { + callback(response); + }); + }, + + httpsToRailsWithProxy: function (callback) { + var requestUrl = constants.RAILS_AUTOMATE; + var requestOptions = ConnectivityChecker.reqOpsWithProxy(requestUrl, 'https'); + fireRequest(requestOptions, 'https', 'HTTPS Request To ' + requestUrl + ' With Proxy', [200], function (response) { + callback(response); + }); + }, /** * Decides the checks to perform based on whether any proxy is provided by the @@ -138,16 +155,29 @@ var ConnectivityChecker = { }; if (RdGlobalConfig.proxy) { - ConnectivityChecker.connectionChecks.push(this.httpToHubWithProxy, this.httpToRailsWithProxy); + var proxyOpts = url.parse(RdGlobalConfig.proxy.host + ":" +RdGlobalConfig.proxy.port); + if (RdGlobalConfig.proxy.username && RdGlobalConfig.proxy.password) { + proxyOpts.auth = RdGlobalConfig.proxy.username + ":" + RdGlobalConfig.proxy.password; + } + var agent = null; + if (RdGlobalConfig.SCHEME == "http") { + agent = new HttpProxyAgent(proxyOpts); + ConnectivityChecker.connectionChecks.push(this.httpToHubWithProxy, this.httpToRailsWithProxy); + } else { + agent = new HttpsProxyAgent(proxyOpts); + ConnectivityChecker.connectionChecks.push(this.httpsToHubWithProxy, this.httpsToRailsWithProxy); + } + /* eslint-disable-next-line no-unused-vars */ ConnectivityChecker.reqOpsWithProxy = function (reqUrl, reqType) { var parsedUrl = url.parse(reqUrl); var reqOptions = { method: 'GET', headers: {}, - host: RdGlobalConfig.proxy.host, - port: RdGlobalConfig.proxy.port, - path: parsedUrl.href + host: parsedUrl.hostname, + port: parsedUrl.port || ( reqType === 'http' ? 80 : 443 ), + path: parsedUrl.path, + agent: agent }; if (RdGlobalConfig.proxy.username && RdGlobalConfig.proxy.password) { reqOptions.headers['Proxy-Authorization'] = Utils.proxyAuthToBase64(RdGlobalConfig.proxy); diff --git a/src/requestLib.js b/src/requestLib.js index 8609892..62c385f 100644 --- a/src/requestLib.js +++ b/src/requestLib.js @@ -24,10 +24,10 @@ var RequestLib = { return new Promise(function (resolve, reject) { var requestOptions = Object.assign({}, params.furtherRequestOptions); requestOptions.agent = RdGlobalConfig.SCHEME === 'http' ? httpKeepAliveAgent : httpsKeepAliveAgent; - if(RdGlobalConfig.proxy) { + if (RdGlobalConfig.proxy) { if (!httpProxyAgent && !httpsProxyAgent) { var proxyOpts = url.parse(RdGlobalConfig.proxy.host + ":" +RdGlobalConfig.proxy.port); - if(RdGlobalConfig.proxy.username && RdGlobalConfig.proxy.password) { + if (RdGlobalConfig.proxy.username && RdGlobalConfig.proxy.password) { proxyOpts.auth = RdGlobalConfig.proxy.username + ":" + RdGlobalConfig.proxy.password; } httpProxyAgent = new HttpProxyAgent(proxyOpts); From 939d4b7e0986a6b92e1d16928b02aad1e48be631 Mon Sep 17 00:00:00 2001 From: Neeraj Shukla Date: Tue, 20 Oct 2020 20:48:11 +0530 Subject: [PATCH 29/44] unit tests --- src/connectivity.js | 49 +++++++++++++++++++++++++-------------- test/connectivity.test.js | 26 +++++++++++++++++---- test/proxy.test.js | 6 ++--- test/reverseProxy.test.js | 3 +-- 4 files changed, 57 insertions(+), 27 deletions(-) diff --git a/src/connectivity.js b/src/connectivity.js index c993565..64aea02 100644 --- a/src/connectivity.js +++ b/src/connectivity.js @@ -69,6 +69,7 @@ var ConnectivityChecker = { reqOpsWithoutProxy: function () {}, reqOpsWithProxy: function () {}, + reqOpsHttpsWithProxy: function () {}, httpToHubWithoutProxy: function (callback) { var requestUrl = constants.HUB_STATUS_URL; @@ -87,7 +88,7 @@ var ConnectivityChecker = { }, httpsToHubWithoutProxy: function (callback) { - var requestUrl = constants.HUB_STATUS_URL; + var requestUrl = constants.HUB_STATUS_URL.replace("http://", "https://"); var requestOptions = ConnectivityChecker.reqOpsWithoutProxy(requestUrl, 'https'); fireRequest(requestOptions, 'https', 'HTTPS Request To ' + requestUrl + ' Without Proxy', [200], function (response) { callback(response); @@ -95,7 +96,7 @@ var ConnectivityChecker = { }, httpsToRailsWithoutProxy: function (callback) { - var requestUrl = constants.RAILS_AUTOMATE; + var requestUrl = constants.RAILS_AUTOMATE.replace("http://", "https://"); var requestOptions = ConnectivityChecker.reqOpsWithoutProxy(requestUrl, 'https'); fireRequest(requestOptions, 'https', 'HTTPS Request to ' + requestUrl + ' Without Proxy', [301, 302], function (response) { callback(response); @@ -113,23 +114,23 @@ var ConnectivityChecker = { httpToRailsWithProxy: function (callback) { var requestUrl = constants.RAILS_AUTOMATE; var requestOptions = ConnectivityChecker.reqOpsWithProxy(requestUrl, 'http'); - fireRequest(requestOptions, 'http', 'HTTP Request To ' + requestUrl + ' With Proxy', [301], function (response) { + fireRequest(requestOptions, 'http', 'HTTP Request To ' + requestUrl + ' With Proxy', [301, 302], function (response) { callback(response); }); }, httpsToHubWithProxy: function (callback) { - var requestUrl = constants.HUB_STATUS_URL; - var requestOptions = ConnectivityChecker.reqOpsWithProxy(requestUrl, 'https'); + var requestUrl = constants.HUB_STATUS_URL.replace("http", "https"); + var requestOptions = ConnectivityChecker.reqOpsHttpsWithProxy(requestUrl, 'https'); fireRequest(requestOptions, 'https', 'HTTPS Request To ' + requestUrl + ' With Proxy', [200], function (response) { callback(response); }); }, httpsToRailsWithProxy: function (callback) { - var requestUrl = constants.RAILS_AUTOMATE; - var requestOptions = ConnectivityChecker.reqOpsWithProxy(requestUrl, 'https'); - fireRequest(requestOptions, 'https', 'HTTPS Request To ' + requestUrl + ' With Proxy', [200], function (response) { + var requestUrl = constants.RAILS_AUTOMATE.replace("http", "https"); + var requestOptions = ConnectivityChecker.reqOpsHttpsWithProxy(requestUrl, 'https'); + fireRequest(requestOptions, 'https', 'HTTPS Request To ' + requestUrl + ' With Proxy', [301, 302], function (response) { callback(response); }); }, @@ -154,20 +155,14 @@ var ConnectivityChecker = { return reqOptions; }; + if (RdGlobalConfig.proxy) { var proxyOpts = url.parse(RdGlobalConfig.proxy.host + ":" +RdGlobalConfig.proxy.port); if (RdGlobalConfig.proxy.username && RdGlobalConfig.proxy.password) { proxyOpts.auth = RdGlobalConfig.proxy.username + ":" + RdGlobalConfig.proxy.password; } - var agent = null; - if (RdGlobalConfig.SCHEME == "http") { - agent = new HttpProxyAgent(proxyOpts); - ConnectivityChecker.connectionChecks.push(this.httpToHubWithProxy, this.httpToRailsWithProxy); - } else { - agent = new HttpsProxyAgent(proxyOpts); - ConnectivityChecker.connectionChecks.push(this.httpsToHubWithProxy, this.httpsToRailsWithProxy); - } - + + ConnectivityChecker.connectionChecks.push(this.httpToHubWithProxy, this.httpToRailsWithProxy); /* eslint-disable-next-line no-unused-vars */ ConnectivityChecker.reqOpsWithProxy = function (reqUrl, reqType) { var parsedUrl = url.parse(reqUrl); @@ -177,7 +172,25 @@ var ConnectivityChecker = { host: parsedUrl.hostname, port: parsedUrl.port || ( reqType === 'http' ? 80 : 443 ), path: parsedUrl.path, - agent: agent + agent: new HttpProxyAgent(proxyOpts) + }; + if (RdGlobalConfig.proxy.username && RdGlobalConfig.proxy.password) { + reqOptions.headers['Proxy-Authorization'] = Utils.proxyAuthToBase64(RdGlobalConfig.proxy); + } + return reqOptions; + }; + + ConnectivityChecker.connectionChecks.push(this.httpsToHubWithProxy, this.httpsToRailsWithProxy); + /* eslint-disable-next-line no-unused-vars */ + ConnectivityChecker.reqOpsHttpsWithProxy = function (reqUrl, reqType) { + var parsedUrl = url.parse(reqUrl); + var reqOptions = { + method: 'GET', + headers: {}, + host: parsedUrl.hostname, + port: parsedUrl.port || ( reqType === 'http' ? 80 : 443 ), + path: parsedUrl.path, + agent: new HttpsProxyAgent(proxyOpts) }; if (RdGlobalConfig.proxy.username && RdGlobalConfig.proxy.password) { reqOptions.headers['Proxy-Authorization'] = Utils.proxyAuthToBase64(RdGlobalConfig.proxy); diff --git a/test/connectivity.test.js b/test/connectivity.test.js index 58621af..1f5c40d 100644 --- a/test/connectivity.test.js +++ b/test/connectivity.test.js @@ -32,13 +32,13 @@ describe('Connectivity Checker for BrowserStack Components', function () { data: '{"data":"value"}', statusCode: 200, errorMessage: null, - description: 'HTTPS Request To ' + constants.HUB_STATUS_URL + ' Without Proxy', + description: 'HTTPS Request To ' + constants.HUB_STATUS_URL.replace("http://", "https://") + ' Without Proxy', result: 'Passed' }, { data: '{"data":"value"}', statusCode: 302, errorMessage: null, - description: 'HTTPS Request to ' + constants.RAILS_AUTOMATE + ' Without Proxy', + description: 'HTTPS Request to ' + constants.RAILS_AUTOMATE.replace("http://", "https://") + ' Without Proxy', result: 'Passed' }]; @@ -58,13 +58,13 @@ describe('Connectivity Checker for BrowserStack Components', function () { data: [], statusCode: null, errorMessage: 'Error: something terrible', - description: 'HTTPS Request To ' + constants.HUB_STATUS_URL + ' Without Proxy', + description: 'HTTPS Request To ' + constants.HUB_STATUS_URL.replace("http://", "https://") + ' Without Proxy', result: 'Failed' }, { data: [], statusCode: null, errorMessage: 'Error: something terrible', - description: 'HTTPS Request to ' + constants.RAILS_AUTOMATE + ' Without Proxy', + description: 'HTTPS Request to ' + constants.RAILS_AUTOMATE.replace("http://", "https://") + ' Without Proxy', result: 'Failed' }]; @@ -99,8 +99,11 @@ describe('Connectivity Checker for BrowserStack Components', function () { testHelper.initializeDummyProxy(); ConnectivityChecker.connectionChecks = []; testHelper.nockGetRequest(constants.HUB_STATUS_URL, 'http', null, 200); + testHelper.nockGetRequest(constants.HUB_STATUS_URL, 'http', null, 200); + testHelper.nockGetRequest(constants.HUB_STATUS_URL, 'https', null, 200); testHelper.nockGetRequest(constants.HUB_STATUS_URL, 'https', null, 200); testHelper.nockGetRequest(constants.RAILS_AUTOMATE, 'http', null, 301); + testHelper.nockGetRequest(constants.RAILS_AUTOMATE, 'http', null, 301); testHelper.nockGetRequest(constants.RAILS_AUTOMATE, 'https', null, 302); testHelper.nockProxyUrl(RdGlobalConfig.proxy, 'http', 'hub', null, 200); testHelper.nockProxyUrl(RdGlobalConfig.proxy, 'http', 'automate', null, 301); @@ -113,6 +116,8 @@ describe('Connectivity Checker for BrowserStack Components', function () { it('HTTP(S) to Hub & Rails', function (done) { this.timeout(2000); + testHelper.nockGetRequest(constants.HUB_STATUS_URL, 'https', null, 200); + testHelper.nockGetRequest(constants.RAILS_AUTOMATE, 'https', null, 302); sinon.stub(Utils, 'beautifyObject'); var resultWithProxy = resultWithoutProxy.concat([{ data: '{"data":"value"}', @@ -126,6 +131,18 @@ describe('Connectivity Checker for BrowserStack Components', function () { errorMessage: null, result: "Passed", statusCode: 301 + },{ + data: '{"data":"value"}', + description: "HTTPS Request To " + constants.HUB_STATUS_URL.replace("http://", "https://") + " With Proxy", + errorMessage: null, + result: "Passed", + statusCode: 200 + }, { + data: '{"data":"value"}', + description: "HTTPS Request To " + constants.RAILS_AUTOMATE.replace("http://", "https://")+ " With Proxy", + errorMessage: null, + result: "Passed", + statusCode: 302 }]); ConnectivityChecker.fireChecks("some topic", 1, function () { @@ -146,6 +163,7 @@ describe('Connectivity Checker for BrowserStack Components', function () { testHelper.nockGetRequestWithError(constants.HUB_STATUS_URL, 'https'); testHelper.nockGetRequestWithError(constants.RAILS_AUTOMATE, 'http'); testHelper.nockGetRequestWithError(constants.RAILS_AUTOMATE, 'https'); + }); afterEach(function () { diff --git a/test/proxy.test.js b/test/proxy.test.js index bc76c6b..2bffdd3 100644 --- a/test/proxy.test.js +++ b/test/proxy.test.js @@ -14,6 +14,7 @@ describe('RdHandler', function () { this.timeout = 5000; testHelper.initializeDummyLoggers(); testHelper.initializeDummyHandlers(); + testHelper.initializeDummyProxy(); originalScheme = RdGlobalConfig.SCHEME; RdGlobalConfig.SCHEME = 'http'; @@ -72,7 +73,7 @@ describe('RdHandler', function () { host: 'localhost', port: RdGlobalConfig.RD_HANDLER_PROXY_PORT, headers: {}, - path: constants.HUB_STATUS_URL + path: "http://user1:pass1@" + constants.HUB_HOST + "/wd/hub/status" }; var responseData = []; @@ -85,11 +86,10 @@ describe('RdHandler', function () { response.on('end', function () { assert(Buffer.concat(responseData).toString() === '{"data":"value"}'); done(); + testHelper.deleteProxy(); }); }); - request.end(); - testHelper.deleteProxy(); }); it('Requests on behalf of the client via external proxy and returns the response even if request by tool fails', function (done) { diff --git a/test/reverseProxy.test.js b/test/reverseProxy.test.js index 57e57e1..b4548cb 100644 --- a/test/reverseProxy.test.js +++ b/test/reverseProxy.test.js @@ -85,11 +85,10 @@ describe('RdHandler', function () { response.on('end', function () { assert(Buffer.concat(responseData).toString() === '{"data":"value"}'); done(); + testHelper.deleteProxy(); }); }); - request.end(); - testHelper.deleteProxy(); }); it('Requests reverse proxy on behalf of the client via external proxy and returns the response even if request by tool fails', function (done) { From 269f7eb7c0ed5153b043499e92c015acf886f5d2 Mon Sep 17 00:00:00 2001 From: Neeraj Shukla Date: Tue, 20 Oct 2020 21:12:45 +0530 Subject: [PATCH 30/44] lint fix --- .eslintrc.js | 3 ++- src/commandLine.js | 6 +++--- src/connectivity.js | 16 +++++++-------- src/logger.js | 2 +- src/proxy.js | 22 ++++++++++----------- src/requestLib.js | 24 +++++++++++------------ src/reverseProxy.js | 20 +++++++++---------- src/utils.js | 42 ++++++++++++++++++++-------------------- test/commandLine.test.js | 2 +- 9 files changed, 69 insertions(+), 68 deletions(-) diff --git a/.eslintrc.js b/.eslintrc.js index 1ae3d06..ec71f90 100644 --- a/.eslintrc.js +++ b/.eslintrc.js @@ -17,7 +17,8 @@ module.exports = { "linebreak-style": [ "error", "unix"], "semi": ["error", "always"], "eol-last": ["error", "always"], - "keyword-spacing": [1] + "keyword-spacing": [1], + "no-trailing-spaces": ["error", { "skipBlankLines": true }] }, "overrides": [{ "files": ["src/requestsDebugger.js", "src/commandLine.js", "test/**/*.test.js"], diff --git a/src/commandLine.js b/src/commandLine.js index d79f86a..957fddb 100644 --- a/src/commandLine.js +++ b/src/commandLine.js @@ -42,7 +42,7 @@ var CommandLineManager = { /** * Processes the args from the given input array and sets the global config. - * @param {Array} argv + * @param {Array} argv */ processArgs: function (argv) { @@ -149,7 +149,7 @@ var CommandLineManager = { } else { RdGlobalConfig.SCHEME = scheme; - argv.splice(index, 2); + argv.splice(index, 2); } } else { invalidArgs.add('--scheme'); @@ -163,7 +163,7 @@ var CommandLineManager = { if (CommandLineManager.validArgValue(argv[index + 1])) { var host = argv[index + 1]; if (host.lastIndexOf("http") !== 0){ - host = 'http://' + host; + host = 'http://' + host; } RdGlobalConfig.proxy = RdGlobalConfig.proxy || {}; RdGlobalConfig.proxy.host = host; diff --git a/src/connectivity.js b/src/connectivity.js index 64aea02..9e0afb9 100644 --- a/src/connectivity.js +++ b/src/connectivity.js @@ -17,11 +17,11 @@ var HttpsProxyAgent = require('https-proxy-agent'); /** * Fires the requests to perform connectivity checks - * @param {Object} requestOptions - * @param {'http'|'https'} requestType - * @param {String} description - * @param {Array} successCodes - * @param {Function} callback + * @param {Object} requestOptions + * @param {'http'|'https'} requestType + * @param {String} description + * @param {Array} successCodes + * @param {Function} callback */ var fireRequest = function (requestOptions, requestType, description, successCodes, callback) { var httpOrHttps = requestType === 'http' ? http : https; @@ -203,9 +203,9 @@ var ConnectivityChecker = { /** * Fires the Connectivity Checks in Async Manner - * @param {String} topic - * @param {Number|String} uuid - * @param {Function} callback + * @param {String} topic + * @param {Number|String} uuid + * @param {Function} callback */ fireChecks: function (topic, uuid, callback) { ConnectivityChecker.decideConnectionChecks(); diff --git a/src/logger.js b/src/logger.js index 6bbdc43..f434a28 100644 --- a/src/logger.js +++ b/src/logger.js @@ -19,7 +19,7 @@ var LogManager = { * Initializes a logger to the given file and returns the methods to * interact with the logger. * Currently, 'info' and 'error' are defined. THe rest can be taken up later if required. - * @param {String} filename + * @param {String} filename */ initializeLogger: function (filename) { var newLogger = LogManager.getLogger(filename); diff --git a/src/proxy.js b/src/proxy.js index 73733ef..6579ff9 100644 --- a/src/proxy.js +++ b/src/proxy.js @@ -44,8 +44,8 @@ var RdHandler = { * Frames the error response based on the type of request. * i.e., if its a request originating for Hub, the response * is in the format which the client binding would understand. - * @param {Object} parsedRequest - * @param {String} errorMessage + * @param {Object} parsedRequest + * @param {String} errorMessage */ _frameErrorResponse: function (parsedRequest, errorMessage) { errorMessage += '. ' + constants.STATIC_MESSAGES.REQ_FAILED_MSG; @@ -77,8 +77,8 @@ var RdHandler = { /** * Handler for incoming requests to Requests Debugger Tool server. - * @param {http.IncomingMessage} clientRequest - * @param {http.ServerResponse} clientResponse + * @param {http.IncomingMessage} clientRequest + * @param {http.ServerResponse} clientResponse */ requestHandler: function (clientRequest, clientResponse) { clientRequest.id = ++RdHandler._requestCounter + '::' + uuidv4(); @@ -90,9 +90,9 @@ var RdHandler = { data: [] }; RdGlobalConfig.reqLogger.info(constants.TOPICS.CLIENT_REQUEST_START, request.method + ' ' + url, - false, { - headers: request.headers - }, + false, { + headers: request.headers + }, clientRequest.id); var furtherRequestOptions = RdHandler._generateRequestOptions(clientRequest); @@ -134,8 +134,8 @@ var RdHandler = { /** * Starts the proxy server on the given port - * @param {String|Number} port - * @param {Function} callback + * @param {String|Number} port + * @param {Function} callback */ startProxyServer: function (port, callback) { try { @@ -151,10 +151,10 @@ var RdHandler = { callback(e.toString(), null); } }, - + /** * Stops the currently running proxy server - * @param {Function} callback + * @param {Function} callback */ stopProxyServer: function (callback) { try { diff --git a/src/requestLib.js b/src/requestLib.js index 62c385f..feef787 100644 --- a/src/requestLib.js +++ b/src/requestLib.js @@ -15,25 +15,25 @@ var RequestLib = { /** * Method to perform the request on behalf of the client - * @param {schemeObj: Object} schemeObj - * @param {{request: Object, furtherRequestOptions: Object}} params - * @param {http.IncomingMessage} clientRequest - * @param {Number} retries + * @param {schemeObj: Object} schemeObj + * @param {{request: Object, furtherRequestOptions: Object}} params + * @param {http.IncomingMessage} clientRequest + * @param {Number} retries */ _makeRequest: function (schemeObj, params, clientRequest, retries) { return new Promise(function (resolve, reject) { var requestOptions = Object.assign({}, params.furtherRequestOptions); requestOptions.agent = RdGlobalConfig.SCHEME === 'http' ? httpKeepAliveAgent : httpsKeepAliveAgent; - if (RdGlobalConfig.proxy) { + if (RdGlobalConfig.proxy) { if (!httpProxyAgent && !httpsProxyAgent) { var proxyOpts = url.parse(RdGlobalConfig.proxy.host + ":" +RdGlobalConfig.proxy.port); if (RdGlobalConfig.proxy.username && RdGlobalConfig.proxy.password) { proxyOpts.auth = RdGlobalConfig.proxy.username + ":" + RdGlobalConfig.proxy.password; } httpProxyAgent = new HttpProxyAgent(proxyOpts); - httpsProxyAgent = new HttpsProxyAgent(proxyOpts); + httpsProxyAgent = new HttpsProxyAgent(proxyOpts); } - requestOptions.agent = RdGlobalConfig.SCHEME === 'http' ? httpProxyAgent : httpsProxyAgent; + requestOptions.agent = RdGlobalConfig.SCHEME === 'http' ? httpProxyAgent : httpsProxyAgent; } var request = schemeObj.request(requestOptions, function (response) { var responseToSend = { @@ -41,7 +41,7 @@ var RequestLib = { headers: response.headers, data: [] }; - + response.on('data', function (chunk) { responseToSend.data.push(chunk); }); @@ -62,7 +62,7 @@ var RequestLib = { // Log the request that will be initiated on behalf of the client request.on('finish', function () { var url = RdGlobalConfig.SCHEME + "://" + requestOptions.host + requestOptions.path; - RdGlobalConfig.reqLogger.info(constants.TOPICS.TOOL_REQUEST_WITH_RETRIES + retries, + RdGlobalConfig.reqLogger.info(constants.TOPICS.TOOL_REQUEST_WITH_RETRIES + retries, clientRequest.method + ' ' + url, false, Object.assign({}, params.furtherRequestOptions, { data: Buffer.concat(params.request.data).toString() }), @@ -123,9 +123,9 @@ var RequestLib = { /** * Handler for performing request. Includes the retry mechanism when request fails. - * @param {{request: Object, furtherRequestOptions: Object}} params - * @param {http.IncomingMessage} clientRequest - * @param {Number} retries + * @param {{request: Object, furtherRequestOptions: Object}} params + * @param {http.IncomingMessage} clientRequest + * @param {Number} retries */ call: function (params, clientRequest, retries) { retries = (typeof retries === 'number') ? Math.min(constants.MAX_RETRIES, Math.max(retries, 0)) : constants.MAX_RETRIES; diff --git a/src/reverseProxy.js b/src/reverseProxy.js index 7fd7515..e2e7110 100644 --- a/src/reverseProxy.js +++ b/src/reverseProxy.js @@ -45,8 +45,8 @@ var RdHandler = { * Frames the error response based on the type of request. * i.e., if its a request originating for Hub, the response * is in the format which the client binding would understand. - * @param {Object} parsedRequest - * @param {String} errorMessage + * @param {Object} parsedRequest + * @param {String} errorMessage */ _frameErrorResponse: function (parsedRequest, errorMessage) { errorMessage += '. ' + constants.STATIC_MESSAGES.REQ_FAILED_MSG; @@ -78,8 +78,8 @@ var RdHandler = { /** * Handler for incoming requests to Requests Debugger Tool server. - * @param {http.IncomingMessage} clientRequest - * @param {http.ServerResponse} clientResponse + * @param {http.IncomingMessage} clientRequest + * @param {http.ServerResponse} clientResponse */ requestHandler: function (clientRequest, clientResponse) { clientRequest.id = ++RdHandler._requestCounter + '::' + uuidv4(); @@ -91,9 +91,9 @@ var RdHandler = { data: [] }; RdGlobalConfig.reqLogger.info(constants.TOPICS.CLIENT_REQUEST_START, request.method + ' ' + url, - false, { - headers: request.headers - }, + false, { + headers: request.headers + }, clientRequest.id); var furtherRequestOptions = RdHandler._generateRequestOptions(clientRequest); @@ -135,8 +135,8 @@ var RdHandler = { /** * Starts the reverse proxy server on the given port - * @param {String|Number} port - * @param {Function} callback + * @param {String|Number} port + * @param {Function} callback */ startReverseProxyServer: function (port, callback) { try { @@ -155,7 +155,7 @@ var RdHandler = { /** * Stops the currently running reverse proxy server - * @param {Function} callback + * @param {Function} callback */ stopReverseProxyServer: function (callback) { try { diff --git a/src/utils.js b/src/utils.js index e71821a..20c69b3 100644 --- a/src/utils.js +++ b/src/utils.js @@ -8,7 +8,7 @@ var fs = require('fs'); * Input can be in the form of: * 1. 'user:pass' * 2. { username: 'user', password: 'pass' } - * @param {String|{username: String, password: String, host: String, port: String|Number}} proxyObj + * @param {String|{username: String, password: String, host: String, port: String|Number}} proxyObj * @returns {String} */ var proxyAuthToBase64 = function (proxyObj) { @@ -23,9 +23,9 @@ var proxyAuthToBase64 = function (proxyObj) { /** * Fetch the property value from the string array of content, each separated by a separator - * @param {Array} content - * @param {String} propertyToFetch - * @param {String} separator + * @param {Array} content + * @param {String} propertyToFetch + * @param {String} separator * @returns {String} */ var fetchPropertyValue = function (content, propertyToFetch, separator) { @@ -48,11 +48,11 @@ var fetchPropertyValue = function (content, propertyToFetch, separator) { /** * Beautifies the lines and add prefix/suffix characters to make the line of the required length. - * @param {String} line - * @param {String} prefix - * @param {String} suffix - * @param {Number} idealLength - * @param {Boolean} newLine + * @param {String} line + * @param {String} prefix + * @param {String} suffix + * @param {Number} idealLength + * @param {Boolean} newLine * @returns {String} */ var formatAndBeautifyLine = function (line, prefix, suffix, idealLength, newLine) { @@ -79,10 +79,10 @@ var formatAndBeautifyLine = function (line, prefix, suffix, idealLength, newLine /** * Generates header and footer for the given content. - * @param {String} content - * @param {String} title - * @param {Date} generatedAt - * @param {Date} startTime + * @param {String} content + * @param {String} title + * @param {Date} generatedAt + * @param {Date} startTime * @returns {String} */ var generateHeaderAndFooter = function (content, title, generatedAt, startTime) { @@ -109,8 +109,8 @@ var generateHeaderAndFooter = function (content, title, generatedAt, startTime) /** * Performs multiple exec commands asynchronously and returns the * result in the same order of the commands array. - * @param {Array} commands - * @param {Function} callback + * @param {Array} commands + * @param {Function} callback * @returns {Array} */ var execMultiple = function (commands, callback) { @@ -131,7 +131,7 @@ var execMultiple = function (commands, callback) { if (++totalCommandsCompleted === commands.length) { if (isValidCallback(callback)) callback(resultArray); - } + } }); }); }; @@ -164,9 +164,9 @@ var getWmicPath = function () { /** * Beautifies the whole object and returns in a format which can be logged and read easily. * Can take an object or array of objects as input. - * @param {Object|Array} obj - * @param {String} keysTitle - * @param {String} valuesTitle + * @param {Object|Array} obj + * @param {String} keysTitle + * @param {String} valuesTitle * @param {Number} maxKeyLength Optional * @param {Number} maxValLength Optional * @returns {String} @@ -215,7 +215,7 @@ var beautifyObject = function (obj, keysTitle, valuesTitle, maxKeyLength, maxVal /** * Returns the length of the longest entry in the Array - * @param {Array} arr + * @param {Array} arr * @returns {Number} */ var getLongestVal = function (arr) { @@ -230,7 +230,7 @@ var getLongestVal = function (arr) { /** * Returns string value by trying .toString() & JSON.stringify() - * @param {any} val + * @param {any} val */ var safeToString = function (val) { try { diff --git a/test/commandLine.test.js b/test/commandLine.test.js index 08f6ac3..b479886 100644 --- a/test/commandLine.test.js +++ b/test/commandLine.test.js @@ -94,7 +94,7 @@ describe('CommandLineManager', function () { }); it("proxy won't be set if proxy host is not provided", function () { - sinon.stub(console, 'log'); + sinon.stub(console, 'log'); argv = argv.concat(['--proxy-port', '9687']); CommandLineManager.processArgs(argv); console.log.restore(); From 1d0b7070662ab3f6169c1f7cfc5a77d100f8255c Mon Sep 17 00:00:00 2001 From: Neeraj Shukla Date: Tue, 20 Oct 2020 21:38:26 +0530 Subject: [PATCH 31/44] refactor --- config/constants.js | 1 + src/connectivity.js | 12 ++++++------ src/proxy.js | 2 +- src/requestsDebugger.js | 23 +++++++++++------------ src/reverseProxy.js | 2 +- 5 files changed, 20 insertions(+), 20 deletions(-) diff --git a/config/constants.js b/config/constants.js index 8f4f6e4..9d7e199 100644 --- a/config/constants.js +++ b/config/constants.js @@ -13,6 +13,7 @@ module.exports.PORTS = { MIN: 1 }; +module.exports.LINE_LENGTH = 70; module.exports.PROTOCOL_REGEX = /(^\w+:|^)\/\//; module.exports.LOGS = Object.freeze({ diff --git a/src/connectivity.js b/src/connectivity.js index 9e0afb9..8431b35 100644 --- a/src/connectivity.js +++ b/src/connectivity.js @@ -68,7 +68,7 @@ var ConnectivityChecker = { connectionChecks: [], reqOpsWithoutProxy: function () {}, - reqOpsWithProxy: function () {}, + reqOpsHttpWithProxy: function () {}, reqOpsHttpsWithProxy: function () {}, httpToHubWithoutProxy: function (callback) { @@ -105,7 +105,7 @@ var ConnectivityChecker = { httpToHubWithProxy: function (callback) { var requestUrl = constants.HUB_STATUS_URL; - var requestOptions = ConnectivityChecker.reqOpsWithProxy(requestUrl, 'http'); + var requestOptions = ConnectivityChecker.reqOpsHttpWithProxy(requestUrl, 'http'); fireRequest(requestOptions, 'http', 'HTTP Request To ' + requestUrl + ' With Proxy', [200], function (response) { callback(response); }); @@ -113,7 +113,7 @@ var ConnectivityChecker = { httpToRailsWithProxy: function (callback) { var requestUrl = constants.RAILS_AUTOMATE; - var requestOptions = ConnectivityChecker.reqOpsWithProxy(requestUrl, 'http'); + var requestOptions = ConnectivityChecker.reqOpsHttpWithProxy(requestUrl, 'http'); fireRequest(requestOptions, 'http', 'HTTP Request To ' + requestUrl + ' With Proxy', [301, 302], function (response) { callback(response); }); @@ -164,13 +164,13 @@ var ConnectivityChecker = { ConnectivityChecker.connectionChecks.push(this.httpToHubWithProxy, this.httpToRailsWithProxy); /* eslint-disable-next-line no-unused-vars */ - ConnectivityChecker.reqOpsWithProxy = function (reqUrl, reqType) { + ConnectivityChecker.reqOpsHttpWithProxy = function (reqUrl, reqType) { var parsedUrl = url.parse(reqUrl); var reqOptions = { method: 'GET', headers: {}, host: parsedUrl.hostname, - port: parsedUrl.port || ( reqType === 'http' ? 80 : 443 ), + port: parsedUrl.port || 80, path: parsedUrl.path, agent: new HttpProxyAgent(proxyOpts) }; @@ -188,7 +188,7 @@ var ConnectivityChecker = { method: 'GET', headers: {}, host: parsedUrl.hostname, - port: parsedUrl.port || ( reqType === 'http' ? 80 : 443 ), + port: parsedUrl.port || 443, path: parsedUrl.path, agent: new HttpsProxyAgent(proxyOpts) }; diff --git a/src/proxy.js b/src/proxy.js index 6579ff9..46d909b 100644 --- a/src/proxy.js +++ b/src/proxy.js @@ -1,5 +1,5 @@ /** - * Server to Intercept the client's requests and handle them on their behalf. + * Proxy Server to Intercept the client's requests and handle them on their behalf. * Initiates stats and connectivity checks when a requests fails. * It also responds in selenium understandable error when a request fails * at tool. diff --git a/src/requestsDebugger.js b/src/requestsDebugger.js index a9c2a54..b35f718 100644 --- a/src/requestsDebugger.js +++ b/src/requestsDebugger.js @@ -111,33 +111,32 @@ var RdTool = { * collection and connectivity checks. Finally, sets up the tool proxy */ start: function () { - var lineLength = 70; CommandLineManager.processArgs(process.argv); - console.log(Utils.formatAndBeautifyLine(STATIC_MESSAGES.STARTING_TOOL, '-', '-', lineLength, true)); + console.log(Utils.formatAndBeautifyLine(STATIC_MESSAGES.STARTING_TOOL, '-', '-', constants.LINE_LENGTH, true)); RdTool.initLoggersAndHandlers(); /* eslint-disable indent */ console.log(Utils.formatAndBeautifyLine("Refer '" + RdGlobalConfig.LOGS_DIRECTORY + "' folder for CPU/Network/Memory" + " Stats and Connectivity Checks with BrowserStack components", '', '-', 60, true)); /*eslint-enable indent*/ - console.log(Utils.formatAndBeautifyLine(STATIC_MESSAGES.CHECK_CPU_STATS, '', '-', lineLength, true)); + console.log(Utils.formatAndBeautifyLine(STATIC_MESSAGES.CHECK_CPU_STATS, '', '-', constants.LINE_LENGTH, true)); RdGlobalConfig.cpuLogHandler('Initial CPU', null, function () { - console.log(Utils.formatAndBeautifyLine(STATIC_MESSAGES.CPU_STATS_COLLECTED, '', '-', lineLength, true)); + console.log(Utils.formatAndBeautifyLine(STATIC_MESSAGES.CPU_STATS_COLLECTED, '', '-', constants.LINE_LENGTH, true)); }); - console.log(Utils.formatAndBeautifyLine(STATIC_MESSAGES.CHECK_NETWORK_STATS, '', '-', lineLength, true)); + console.log(Utils.formatAndBeautifyLine(STATIC_MESSAGES.CHECK_NETWORK_STATS, '', '-', constants.LINE_LENGTH, true)); RdGlobalConfig.networkLogHandler('Initial Network', null, function () { - console.log(Utils.formatAndBeautifyLine(STATIC_MESSAGES.NETWORK_STATS_COLLECTED, '', '-', lineLength, true)); + console.log(Utils.formatAndBeautifyLine(STATIC_MESSAGES.NETWORK_STATS_COLLECTED, '', '-', constants.LINE_LENGTH, true)); }); - console.log(Utils.formatAndBeautifyLine(STATIC_MESSAGES.CHECK_MEMORY_STATS, '', '-', lineLength, true)); + console.log(Utils.formatAndBeautifyLine(STATIC_MESSAGES.CHECK_MEMORY_STATS, '', '-', constants.LINE_LENGTH, true)); RdGlobalConfig.memLogHandler('Initial Memory', null, function () { - console.log(Utils.formatAndBeautifyLine(STATIC_MESSAGES.MEMORY_STATS_COLLECTED, '', '-', lineLength, true)); + console.log(Utils.formatAndBeautifyLine(STATIC_MESSAGES.MEMORY_STATS_COLLECTED, '', '-', constants.LINE_LENGTH, true)); }); - console.log(Utils.formatAndBeautifyLine(STATIC_MESSAGES.CHECK_CONNECTIVITY, '', '-', lineLength, true)); + console.log(Utils.formatAndBeautifyLine(STATIC_MESSAGES.CHECK_CONNECTIVITY, '', '-', constants.LINE_LENGTH, true)); RdGlobalConfig.connHandler('Initial Connectivity', null, function () { - console.log(Utils.formatAndBeautifyLine(STATIC_MESSAGES.CONNECTIVITY_CHECKS_DONE, '', '-', lineLength, true)); + console.log(Utils.formatAndBeautifyLine(STATIC_MESSAGES.CONNECTIVITY_CHECKS_DONE, '', '-', constants.LINE_LENGTH, true)); }); proxy.startProxyServer(RdGlobalConfig.RD_HANDLER_PROXY_PORT, function (err, result) { @@ -146,7 +145,7 @@ var RdTool = { console.log('Exiting the Proxy Server...'); process.exit(1); } - console.log(Utils.formatAndBeautifyLine(STATIC_MESSAGES.TOOL_PROXY_STARTED_ON_PORT + result, '', '-', lineLength, true)); + console.log(Utils.formatAndBeautifyLine(STATIC_MESSAGES.TOOL_PROXY_STARTED_ON_PORT + result, '', '-', constants.LINE_LENGTH, true)); }); reverseProxy.startReverseProxyServer(RdGlobalConfig.RD_HANDLER_REVERSE_PROXY_PORT, function (err, result) { @@ -155,7 +154,7 @@ var RdTool = { console.log('Exiting the Reverse Proxy Server...'); process.exit(1); } - console.log(Utils.formatAndBeautifyLine(STATIC_MESSAGES.TOOL_REVESE_PROXY_STARTED_ON_PORT + result, '', '-', lineLength, true)); + console.log(Utils.formatAndBeautifyLine(STATIC_MESSAGES.TOOL_REVESE_PROXY_STARTED_ON_PORT + result, '', '-', constants.LINE_LENGTH, true)); }); } diff --git a/src/reverseProxy.js b/src/reverseProxy.js index e2e7110..38f720f 100644 --- a/src/reverseProxy.js +++ b/src/reverseProxy.js @@ -1,5 +1,5 @@ /** - * Server to Intercept the client's requests and handle them on their behalf. + * Reverse Proxy Server to Intercept the client's requests and handle them on their behalf. * Initiates stats and connectivity checks when a requests fails. * It also responds in selenium understandable error when a request fails * at tool. From cef8c1e434a3e5e809525d5d13a72143444a5585 Mon Sep 17 00:00:00 2001 From: Neeraj Shukla Date: Tue, 20 Oct 2020 22:06:46 +0530 Subject: [PATCH 32/44] removed irrelevant testsaces --- test/commandLine.test.js | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/test/commandLine.test.js b/test/commandLine.test.js index b479886..1866583 100644 --- a/test/commandLine.test.js +++ b/test/commandLine.test.js @@ -39,20 +39,6 @@ describe('CommandLineManager', function () { expect(RdGlobalConfig.proxy.port).to.eql(9687); }); - it('remove any protocol part from proxy-host', function () { - sinon.stub(console, 'log'); - argv = argv.concat(['--proxy-host', 'host']); - CommandLineManager.processArgs(argv); - expect(RdGlobalConfig.proxy.host).to.eql(proxy_host_actual_value); - }); - - it('remove any prefix slashes from proxy-host', function () { - argv = argv.concat(['--proxy-host', 'host']); - CommandLineManager.processArgs(argv); - console.log.restore(); - expect(RdGlobalConfig.proxy.host).to.eql(proxy_host_actual_value); - }); - it('proxy-port is set to the default value when its not in the expected range', function () { sinon.stub(console, 'log'); argv = argv.concat(['--proxy-host', 'host', '--proxy-port', '99999']); From 9e382d1e9749f4f7ce2ee25d388f3c7bdc99888a Mon Sep 17 00:00:00 2001 From: Neeraj Shukla Date: Tue, 20 Oct 2020 22:35:20 +0530 Subject: [PATCH 33/44] more refactor --- config/constants.js | 5 ++++- src/connectivity.js | 8 ++++---- test/connectivity.test.js | 12 ++++++------ test/proxy.test.js | 6 ++++-- test/reverseProxy.test.js | 6 +++--- 5 files changed, 21 insertions(+), 16 deletions(-) diff --git a/config/constants.js b/config/constants.js index 9d7e199..ca5ec41 100644 --- a/config/constants.js +++ b/config/constants.js @@ -1,8 +1,11 @@ module.exports.VERSION = '1.1.0'; module.exports.BS_DOMAIN = 'browserstack.com'; module.exports.HUB_HOST = 'hub-cloud.' + this.BS_DOMAIN; -module.exports.HUB_STATUS_URL = 'http://' + this.HUB_HOST + '/wd/hub/status'; +module.exports.HUB_STATUS_PATH = '/wd/hub/status'; +module.exports.HUB_STATUS_URL = 'http://' + this.HUB_HOST + this.HUB_STATUS_PATH; +module.exports.HUB_STATUS_URL_HTTPS = 'https://' + this.HUB_HOST + this.HUB_STATUS_PATH; module.exports.RAILS_AUTOMATE = 'http://automate.' + this.BS_DOMAIN; +module.exports.RAILS_AUTOMATE_HTTPS = 'https://automate.' + this.BS_DOMAIN; module.exports.CONNECTIVITY_REQ_TIMEOUT = 30000; module.exports.DEFAULT_PROXY_PORT = 3128; module.exports.CUSTOM_ERROR_RESPONSE_CODE = 502; diff --git a/src/connectivity.js b/src/connectivity.js index 8431b35..7331473 100644 --- a/src/connectivity.js +++ b/src/connectivity.js @@ -88,7 +88,7 @@ var ConnectivityChecker = { }, httpsToHubWithoutProxy: function (callback) { - var requestUrl = constants.HUB_STATUS_URL.replace("http://", "https://"); + var requestUrl = constants.HUB_STATUS_URL_HTTPS; var requestOptions = ConnectivityChecker.reqOpsWithoutProxy(requestUrl, 'https'); fireRequest(requestOptions, 'https', 'HTTPS Request To ' + requestUrl + ' Without Proxy', [200], function (response) { callback(response); @@ -96,7 +96,7 @@ var ConnectivityChecker = { }, httpsToRailsWithoutProxy: function (callback) { - var requestUrl = constants.RAILS_AUTOMATE.replace("http://", "https://"); + var requestUrl = constants.RAILS_AUTOMATE_HTTPS; var requestOptions = ConnectivityChecker.reqOpsWithoutProxy(requestUrl, 'https'); fireRequest(requestOptions, 'https', 'HTTPS Request to ' + requestUrl + ' Without Proxy', [301, 302], function (response) { callback(response); @@ -120,7 +120,7 @@ var ConnectivityChecker = { }, httpsToHubWithProxy: function (callback) { - var requestUrl = constants.HUB_STATUS_URL.replace("http", "https"); + var requestUrl = constants.HUB_STATUS_URL_HTTPS; var requestOptions = ConnectivityChecker.reqOpsHttpsWithProxy(requestUrl, 'https'); fireRequest(requestOptions, 'https', 'HTTPS Request To ' + requestUrl + ' With Proxy', [200], function (response) { callback(response); @@ -128,7 +128,7 @@ var ConnectivityChecker = { }, httpsToRailsWithProxy: function (callback) { - var requestUrl = constants.RAILS_AUTOMATE.replace("http", "https"); + var requestUrl = constants.RAILS_AUTOMATE_HTTPS; var requestOptions = ConnectivityChecker.reqOpsHttpsWithProxy(requestUrl, 'https'); fireRequest(requestOptions, 'https', 'HTTPS Request To ' + requestUrl + ' With Proxy', [301, 302], function (response) { callback(response); diff --git a/test/connectivity.test.js b/test/connectivity.test.js index 1f5c40d..87d7cc6 100644 --- a/test/connectivity.test.js +++ b/test/connectivity.test.js @@ -32,13 +32,13 @@ describe('Connectivity Checker for BrowserStack Components', function () { data: '{"data":"value"}', statusCode: 200, errorMessage: null, - description: 'HTTPS Request To ' + constants.HUB_STATUS_URL.replace("http://", "https://") + ' Without Proxy', + description: 'HTTPS Request To ' + constants.HUB_STATUS_URL_HTTPS + ' Without Proxy', result: 'Passed' }, { data: '{"data":"value"}', statusCode: 302, errorMessage: null, - description: 'HTTPS Request to ' + constants.RAILS_AUTOMATE.replace("http://", "https://") + ' Without Proxy', + description: 'HTTPS Request to ' + constants.RAILS_AUTOMATE_HTTPS + ' Without Proxy', result: 'Passed' }]; @@ -58,13 +58,13 @@ describe('Connectivity Checker for BrowserStack Components', function () { data: [], statusCode: null, errorMessage: 'Error: something terrible', - description: 'HTTPS Request To ' + constants.HUB_STATUS_URL.replace("http://", "https://") + ' Without Proxy', + description: 'HTTPS Request To ' + constants.HUB_STATUS_URL_HTTPS+ ' Without Proxy', result: 'Failed' }, { data: [], statusCode: null, errorMessage: 'Error: something terrible', - description: 'HTTPS Request to ' + constants.RAILS_AUTOMATE.replace("http://", "https://") + ' Without Proxy', + description: 'HTTPS Request to ' + constants.RAILS_AUTOMATE_HTTPS + ' Without Proxy', result: 'Failed' }]; @@ -133,13 +133,13 @@ describe('Connectivity Checker for BrowserStack Components', function () { statusCode: 301 },{ data: '{"data":"value"}', - description: "HTTPS Request To " + constants.HUB_STATUS_URL.replace("http://", "https://") + " With Proxy", + description: "HTTPS Request To " + constants.HUB_STATUS_URL_HTTPS + " With Proxy", errorMessage: null, result: "Passed", statusCode: 200 }, { data: '{"data":"value"}', - description: "HTTPS Request To " + constants.RAILS_AUTOMATE.replace("http://", "https://")+ " With Proxy", + description: "HTTPS Request To " + constants.RAILS_AUTOMATE_HTTPS + " With Proxy", errorMessage: null, result: "Passed", statusCode: 302 diff --git a/test/proxy.test.js b/test/proxy.test.js index 2bffdd3..91a916c 100644 --- a/test/proxy.test.js +++ b/test/proxy.test.js @@ -44,7 +44,7 @@ describe('RdHandler', function () { host: constants.HUB_HOST, port: constants.RD_HANDLER_PROXY_PORT, headers: {}, - path: '/wd/hub/status' + path: constants.HUB_STATUS_PATH }; var responseData = []; @@ -73,7 +73,7 @@ describe('RdHandler', function () { host: 'localhost', port: RdGlobalConfig.RD_HANDLER_PROXY_PORT, headers: {}, - path: "http://user1:pass1@" + constants.HUB_HOST + "/wd/hub/status" + path: "http://user1:pass1@" + constants.HUB_HOST + constants.HUB_STATUS_PATH }; var responseData = []; @@ -84,6 +84,8 @@ describe('RdHandler', function () { }); response.on('end', function () { + console.log("$$$$$$"); + console.log(Buffer.concat(responseData).toString()); assert(Buffer.concat(responseData).toString() === '{"data":"value"}'); done(); testHelper.deleteProxy(); diff --git a/test/reverseProxy.test.js b/test/reverseProxy.test.js index b4548cb..31ebadd 100644 --- a/test/reverseProxy.test.js +++ b/test/reverseProxy.test.js @@ -43,7 +43,7 @@ describe('RdHandler', function () { host: 'localhost', port: RdGlobalConfig.RD_HANDLER_REVERSE_PROXY_PORT, headers: {}, - path: '/wd/hub/status' + path: constants.HUB_STATUS_PATH }; var responseData = []; @@ -72,7 +72,7 @@ describe('RdHandler', function () { host: 'localhost', port: RdGlobalConfig.RD_HANDLER_REVERSE_PROXY_PORT, headers: {}, - path: '/wd/hub/status' + path: constants.HUB_STATUS_PATH }; var responseData = []; @@ -101,7 +101,7 @@ describe('RdHandler', function () { host: 'localhost', port: RdGlobalConfig.RD_HANDLER_REVERSE_PROXY_PORT, headers: {}, - path: '/wd/hub/status' + path: constants.HUB_STATUS_PATH }; var responseData = []; From b6af389b7253461dd55ef8d3d03545cb92502fef Mon Sep 17 00:00:00 2001 From: Neeraj Shukla Date: Wed, 21 Oct 2020 18:01:19 +0530 Subject: [PATCH 34/44] addressed review comments --- README.md | 2 +- config/constants.js | 9 +++++---- src/connectivity.js | 6 ------ test/proxy.test.js | 8 +++----- test/reverseProxy.test.js | 6 +++--- 5 files changed, 12 insertions(+), 19 deletions(-) diff --git a/README.md b/README.md index 1f6fd26..e7fc50c 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,7 @@ - Supported `args`: - `--port `: Port on which the Requests Debugger Tool's Proxy will run. Default: 9687 - `--reverse-proxy-port `: Port on which the Requests Debugger Tool's Reverse Proxy will run. Default: 9688 - - `--scheme `: Scheme for requests to browserstack. Default: https" + - `--scheme `: Scheme for requests to browserstack. Default: https - `--proxy-host `: Hostname of the Upstream Proxy - `--proxy-port `: Port of the Upstream Proxy. Default: 3128 (if hostname is provided) - `--proxy-user `: Username for auth of the Upstream Proxy diff --git a/config/constants.js b/config/constants.js index ca5ec41..fbec60a 100644 --- a/config/constants.js +++ b/config/constants.js @@ -1,11 +1,12 @@ module.exports.VERSION = '1.1.0'; module.exports.BS_DOMAIN = 'browserstack.com'; module.exports.HUB_HOST = 'hub-cloud.' + this.BS_DOMAIN; +module.exports.RAILS_HOST = 'automate.' + this.BS_DOMAIN; module.exports.HUB_STATUS_PATH = '/wd/hub/status'; module.exports.HUB_STATUS_URL = 'http://' + this.HUB_HOST + this.HUB_STATUS_PATH; module.exports.HUB_STATUS_URL_HTTPS = 'https://' + this.HUB_HOST + this.HUB_STATUS_PATH; -module.exports.RAILS_AUTOMATE = 'http://automate.' + this.BS_DOMAIN; -module.exports.RAILS_AUTOMATE_HTTPS = 'https://automate.' + this.BS_DOMAIN; +module.exports.RAILS_AUTOMATE = 'http://' + this.RAILS_HOST; +module.exports.RAILS_AUTOMATE_HTTPS = 'https://' + this.RAILS_HOST; module.exports.CONNECTIVITY_REQ_TIMEOUT = 30000; module.exports.DEFAULT_PROXY_PORT = 3128; module.exports.CUSTOM_ERROR_RESPONSE_CODE = 502; @@ -38,7 +39,7 @@ module.exports.RdGlobalConfig = { module.exports.COMMON = Object.freeze({ PING_HUB: 'ping -c 5 ' + this.HUB_HOST, - PING_AUTOMATE: 'ping -c 5 ' + this.RAILS_AUTOMATE + PING_AUTOMATE: 'ping -c 5 ' + this.RAILS_HOST }); module.exports.MAC = Object.freeze({ @@ -59,7 +60,7 @@ module.exports.WIN = Object.freeze({ IPCONFIG_ALL: 'ipconfig /all', SWAP_USAGE: 'pagefile get AllocatedBaseSize, CurrentUsage', // this is a WMIC command. Prefix with WMIC Path PING_HUB: 'ping -n 5 ' + this.HUB_HOST, - PING_AUTOMATE: 'ping -n 5 ' + this.RAILS_AUTOMATE, + PING_AUTOMATE: 'ping -n 5 ' + this.RAILS_HOST, LOAD_PERCENTAGE: 'cpu get loadpercentage', // prefix wmic path }); diff --git a/src/connectivity.js b/src/connectivity.js index 7331473..001a651 100644 --- a/src/connectivity.js +++ b/src/connectivity.js @@ -174,9 +174,6 @@ var ConnectivityChecker = { path: parsedUrl.path, agent: new HttpProxyAgent(proxyOpts) }; - if (RdGlobalConfig.proxy.username && RdGlobalConfig.proxy.password) { - reqOptions.headers['Proxy-Authorization'] = Utils.proxyAuthToBase64(RdGlobalConfig.proxy); - } return reqOptions; }; @@ -192,9 +189,6 @@ var ConnectivityChecker = { path: parsedUrl.path, agent: new HttpsProxyAgent(proxyOpts) }; - if (RdGlobalConfig.proxy.username && RdGlobalConfig.proxy.password) { - reqOptions.headers['Proxy-Authorization'] = Utils.proxyAuthToBase64(RdGlobalConfig.proxy); - } return reqOptions; }; } diff --git a/test/proxy.test.js b/test/proxy.test.js index 91a916c..1319045 100644 --- a/test/proxy.test.js +++ b/test/proxy.test.js @@ -41,10 +41,10 @@ describe('RdHandler', function () { testHelper.nockGetRequest(constants.HUB_STATUS_URL, 'http', null, 200); var reqOptions = { method: 'GET', - host: constants.HUB_HOST, - port: constants.RD_HANDLER_PROXY_PORT, + host: 'localhost', + port: RdGlobalConfig.RD_HANDLER_PROXY_PORT, headers: {}, - path: constants.HUB_STATUS_PATH + path: constants.HUB_STATUS_URL }; var responseData = []; @@ -84,8 +84,6 @@ describe('RdHandler', function () { }); response.on('end', function () { - console.log("$$$$$$"); - console.log(Buffer.concat(responseData).toString()); assert(Buffer.concat(responseData).toString() === '{"data":"value"}'); done(); testHelper.deleteProxy(); diff --git a/test/reverseProxy.test.js b/test/reverseProxy.test.js index 31ebadd..eb9c74b 100644 --- a/test/reverseProxy.test.js +++ b/test/reverseProxy.test.js @@ -35,7 +35,7 @@ describe('RdHandler', function () { RdGlobalConfig.SCHEME = originalScheme; }); - it('Requests reverse proxy on behalf of the client and returns the response', function (done) { + it('Requests on behalf of the client and returns the response', function (done) { this.timeout = 5000; testHelper.nockGetRequest(constants.HUB_STATUS_URL, 'https', null, 200); var reqOptions = { @@ -62,7 +62,7 @@ describe('RdHandler', function () { request.end(); }); - it('Requests reverse proxy on behalf of the client via external proxy and returns the response', function (done) { + it('Requests on behalf of the client via external proxy and returns the response', function (done) { this.timeout = 5000; testHelper.nockGetRequest(constants.HUB_STATUS_URL, 'https', null, 200); testHelper.initializeDummyProxy(); @@ -91,7 +91,7 @@ describe('RdHandler', function () { request.end(); }); - it('Requests reverse proxy on behalf of the client via external proxy and returns the response even if request by tool fails', function (done) { + it('Requests on behalf of the client via external proxy and returns the response even if request by tool fails', function (done) { this.timeout = 5000; for (var i = 0; i <= constants.MAX_RETRIES; i++) { testHelper.nockGetRequestWithError(constants.HUB_STATUS_URL, 'https'); From 3ee7f4499928197c941541682f5b3820ba5db5b2 Mon Sep 17 00:00:00 2001 From: Neeraj Shukla Date: Wed, 21 Oct 2020 18:20:40 +0530 Subject: [PATCH 35/44] added tests in proxy and reverseProxy for http and https --- test/proxy.test.js | 116 +++++++++++++++++++++++++++++++++++++- test/reverseProxy.test.js | 115 ++++++++++++++++++++++++++++++++++++- 2 files changed, 229 insertions(+), 2 deletions(-) diff --git a/test/proxy.test.js b/test/proxy.test.js index 1319045..84de46f 100644 --- a/test/proxy.test.js +++ b/test/proxy.test.js @@ -8,7 +8,7 @@ var testHelper = require('./testHelper'); describe('RdHandler', function () { - context('Proxy Server', function () { + context('Proxy Server with http scheme', function () { var originalScheme; before(function (done) { this.timeout = 5000; @@ -121,4 +121,118 @@ describe('RdHandler', function () { request.end(); }); }); + + context('Proxy Server with https scheme', function () { + var originalScheme; + before(function (done) { + this.timeout = 5000; + testHelper.initializeDummyLoggers(); + testHelper.initializeDummyHandlers(); + testHelper.initializeDummyProxy(); + originalScheme = RdGlobalConfig.SCHEME; + RdGlobalConfig.SCHEME = 'https'; + + RdHandler.startProxyServer(RdGlobalConfig.RD_HANDLER_PROXY_PORT, function (port) { + console.log('Test Network Utility Proxy Server Started on Port: ', port); + done(); + }); + }); + + after(function (done) { + this.timeout = 5000; + RdHandler.stopProxyServer(function () { + done(); + }); + testHelper.deleteLoggers(); + testHelper.deleteHandlers(); + nock.cleanAll(); + RdGlobalConfig.SCHEME.restore(); + RdGlobalConfig.SCHEME = originalScheme; + }); + + it('Requests on behalf of the client and returns the response', function (done) { + this.timeout = 5000; + testHelper.nockGetRequest(constants.HUB_STATUS_URL, 'https', null, 200); + var reqOptions = { + method: 'GET', + host: 'localhost', + port: RdGlobalConfig.RD_HANDLER_PROXY_PORT, + headers: {}, + path: constants.HUB_STATUS_URL + }; + + var responseData = []; + var request = http.request(reqOptions, function (response) { + + response.on('data', function (chunk) { + responseData.push(chunk); + }); + + response.on('end', function () { + assert(Buffer.concat(responseData).toString() === '{"data":"value"}'); + done(); + }); + }); + + request.end(); + }); + + it('Requests on behalf of the client via external proxy and returns the response', function (done) { + this.timeout = 5000; + testHelper.nockGetRequest(constants.HUB_STATUS_URL, 'https', null, 200); + testHelper.initializeDummyProxy(); + testHelper.nockProxyUrl(RdGlobalConfig.proxy, 'https', 'hub', null, 200); + var reqOptions = { + method: 'GET', + host: 'localhost', + port: RdGlobalConfig.RD_HANDLER_PROXY_PORT, + headers: {}, + path: "http://user1:pass1@" + constants.HUB_HOST + constants.HUB_STATUS_PATH + }; + + var responseData = []; + var request = http.request(reqOptions, function (response) { + + response.on('data', function (chunk) { + responseData.push(chunk); + }); + + response.on('end', function () { + assert(Buffer.concat(responseData).toString() === '{"data":"value"}'); + done(); + testHelper.deleteProxy(); + }); + }); + request.end(); + }); + + it('Requests on behalf of the client via external proxy and returns the response even if request by tool fails', function (done) { + this.timeout = 5000; + for (var i = 0; i <= constants.MAX_RETRIES; i++) { + testHelper.nockGetRequestWithError(constants.HUB_STATUS_URL, 'https'); + } + var reqOptions = { + method: 'GET', + host: 'localhost', + port: RdGlobalConfig.RD_HANDLER_PROXY_PORT, + headers: {}, + path: constants.HUB_STATUS_URL + }; + + var responseData = []; + var request = http.request(reqOptions, function (response) { + + response.on('data', function (chunk) { + responseData.push(chunk); + }); + + response.on('end', function () { + assert(Buffer.concat(responseData).toString() === '{"message":"Error: something terrible. Request Failed At Requests Debugger","error":"Request Failed At Requests Debugger"}'); + done(); + }); + }); + + request.end(); + }); + }); }); diff --git a/test/reverseProxy.test.js b/test/reverseProxy.test.js index eb9c74b..1eff9df 100644 --- a/test/reverseProxy.test.js +++ b/test/reverseProxy.test.js @@ -8,7 +8,120 @@ var testHelper = require('./testHelper'); describe('RdHandler', function () { - context('Reverse Proxy Server', function () { + context('Reverse Proxy Server with http scheme', function () { + var originalScheme; + before(function (done) { + this.timeout = 5000; + testHelper.initializeDummyLoggers(); + testHelper.initializeDummyHandlers(); + originalScheme = RdGlobalConfig.SCHEME; + RdGlobalConfig.SCHEME = 'http'; + + RdHandler.startReverseProxyServer(RdGlobalConfig.RD_HANDLER_REVERSE_PROXY_PORT, function (port) { + console.log('Test Network Utility Reverse Proxy Server Started on Port: ', port); + done(); + }); + }); + + after(function (done) { + this.timeout = 5000; + RdHandler.stopReverseProxyServer(function () { + done(); + }); + testHelper.deleteLoggers(); + testHelper.deleteHandlers(); + nock.cleanAll(); + RdGlobalConfig.SCHEME.restore(); + RdGlobalConfig.SCHEME = originalScheme; + }); + + it('Requests on behalf of the client and returns the response', function (done) { + this.timeout = 5000; + testHelper.nockGetRequest(constants.HUB_STATUS_URL, 'http', null, 200); + var reqOptions = { + method: 'GET', + host: 'localhost', + port: RdGlobalConfig.RD_HANDLER_REVERSE_PROXY_PORT, + headers: {}, + path: constants.HUB_STATUS_PATH + }; + + var responseData = []; + var request = http.request(reqOptions, function (response) { + + response.on('data', function (chunk) { + responseData.push(chunk); + }); + + response.on('end', function () { + assert(Buffer.concat(responseData).toString() === '{"data":"value"}'); + done(); + }); + }); + + request.end(); + }); + + it('Requests on behalf of the client via external proxy and returns the response', function (done) { + this.timeout = 5000; + testHelper.nockGetRequest(constants.HUB_STATUS_URL, 'http', null, 200); + testHelper.initializeDummyProxy(); + testHelper.nockProxyUrl(RdGlobalConfig.proxy, 'http', 'hub', null, 200); + var reqOptions = { + method: 'GET', + host: 'localhost', + port: RdGlobalConfig.RD_HANDLER_REVERSE_PROXY_PORT, + headers: {}, + path: constants.HUB_STATUS_PATH + }; + + var responseData = []; + var request = http.request(reqOptions, function (response) { + + response.on('data', function (chunk) { + responseData.push(chunk); + }); + + response.on('end', function () { + assert(Buffer.concat(responseData).toString() === '{"data":"value"}'); + done(); + testHelper.deleteProxy(); + }); + }); + request.end(); + }); + + it('Requests on behalf of the client via external proxy and returns the response even if request by tool fails', function (done) { + this.timeout = 5000; + for (var i = 0; i <= constants.MAX_RETRIES; i++) { + testHelper.nockGetRequestWithError(constants.HUB_STATUS_URL, 'http'); + } + var reqOptions = { + method: 'GET', + host: 'localhost', + port: RdGlobalConfig.RD_HANDLER_REVERSE_PROXY_PORT, + headers: {}, + path: constants.HUB_STATUS_PATH + }; + + var responseData = []; + var request = http.request(reqOptions, function (response) { + + response.on('data', function (chunk) { + responseData.push(chunk); + }); + + response.on('end', function () { + assert(Buffer.concat(responseData).toString() === '{"message":"Error: something terrible. Request Failed At Requests Debugger","error":"Request Failed At Requests Debugger"}'); + done(); + }); + }); + + request.end(); + }); + }); + + context('Reverse Proxy Server with https scheme', function () { var originalScheme; before(function (done) { this.timeout = 5000; From d0464c285195ddec90fa943e07e2e7c52ef767b6 Mon Sep 17 00:00:00 2001 From: Neeraj Shukla Date: Wed, 21 Oct 2020 19:31:44 +0530 Subject: [PATCH 36/44] request end log url fix --- config/constants.js | 5 +++++ src/proxy.js | 2 +- src/requestLib.js | 9 +++++---- src/reverseProxy.js | 8 ++++---- 4 files changed, 15 insertions(+), 9 deletions(-) diff --git a/config/constants.js b/config/constants.js index fbec60a..9fea375 100644 --- a/config/constants.js +++ b/config/constants.js @@ -17,6 +17,11 @@ module.exports.PORTS = { MIN: 1 }; +module.exports.REQUEST_TYPES = { + PROXY: "proxy", + REVERSE_PROXY: "reverse proxy" +}; + module.exports.LINE_LENGTH = 70; module.exports.PROTOCOL_REGEX = /(^\w+:|^)\/\//; diff --git a/src/proxy.js b/src/proxy.js index 46d909b..23ffa8d 100644 --- a/src/proxy.js +++ b/src/proxy.js @@ -102,7 +102,7 @@ var RdHandler = { furtherRequestOptions: furtherRequestOptions }; - ReqLib.call(paramsForRequest, clientRequest) + ReqLib.call(paramsForRequest, clientRequest, constants.REQUEST_TYPES.PROXY) .then(function (response) { RdGlobalConfig.reqLogger.info(constants.TOPICS.CLIENT_RESPONSE_END, clientRequest.method + ' ' + url + ', Status Code: ' + response.statusCode, false, { diff --git a/src/requestLib.js b/src/requestLib.js index feef787..79e80ca 100644 --- a/src/requestLib.js +++ b/src/requestLib.js @@ -20,7 +20,7 @@ var RequestLib = { * @param {http.IncomingMessage} clientRequest * @param {Number} retries */ - _makeRequest: function (schemeObj, params, clientRequest, retries) { + _makeRequest: function (schemeObj, params, clientRequest, requestType, retries) { return new Promise(function (resolve, reject) { var requestOptions = Object.assign({}, params.furtherRequestOptions); requestOptions.agent = RdGlobalConfig.SCHEME === 'http' ? httpKeepAliveAgent : httpsKeepAliveAgent; @@ -108,7 +108,8 @@ var RequestLib = { }); clientRequest.on('end', function () { - RdGlobalConfig.reqLogger.info(constants.TOPICS.CLIENT_REQUEST_END, params.request.method + ' ' + requestOptions.path, false, { + var url = requestType === constants.REQUEST_TYPES.PROXY ? clientRequest.url : "http://" + clientRequest.headers.host + clientRequest.url; + RdGlobalConfig.reqLogger.info(constants.TOPICS.CLIENT_REQUEST_END, params.request.method + ' ' + url, false, { data: Buffer.concat(params.request.data).toString() }, clientRequest.id); @@ -127,10 +128,10 @@ var RequestLib = { * @param {http.IncomingMessage} clientRequest * @param {Number} retries */ - call: function (params, clientRequest, retries) { + call: function (params, clientRequest, requestType, retries) { retries = (typeof retries === 'number') ? Math.min(constants.MAX_RETRIES, Math.max(retries, 0)) : constants.MAX_RETRIES; var schemeObj = RdGlobalConfig.SCHEME === "http" ? http : https; - return RequestLib._makeRequest(schemeObj, params, clientRequest, retries) + return RequestLib._makeRequest(schemeObj, params, clientRequest, requestType, retries) .catch(function (err) { var errTopic = err.customTopic || constants.TOPICS.UNEXPECTED_ERROR; // Collect Network & Connectivity Logs whenever a request fails diff --git a/src/reverseProxy.js b/src/reverseProxy.js index 38f720f..7773442 100644 --- a/src/reverseProxy.js +++ b/src/reverseProxy.js @@ -31,7 +31,8 @@ var RdHandler = { requestOptions.port = RdGlobalConfig.SCHEME == 'http' ? 80 : 443; requestOptions.path = parsedClientUrl.path; requestOptions.method = clientRequest.method; - requestOptions.headers = clientRequest.headers; + // Below line copies clientRequest.headers object to requestOptions.headers instead of refering same object + requestOptions.headers = Object.assign({}, clientRequest.headers); requestOptions.host = constants.HUB_HOST; requestOptions.headers.host = constants.HUB_HOST; requestOptions.headers['X-Requests-Debugger'] = clientRequest.id; @@ -95,15 +96,14 @@ var RdHandler = { headers: request.headers }, clientRequest.id); - + var furtherRequestOptions = RdHandler._generateRequestOptions(clientRequest); var paramsForRequest = { request: request, furtherRequestOptions: furtherRequestOptions }; - - ReqLib.call(paramsForRequest, clientRequest) + ReqLib.call(paramsForRequest, clientRequest, constants.REQUEST_TYPES.REVERSE_PROXY) .then(function (response) { RdGlobalConfig.reqLogger.info(constants.TOPICS.CLIENT_RESPONSE_END, clientRequest.method + ' ' + url + ', Status Code: ' + response.statusCode, false, { From 59987ea803c0959e3ae9199d559835443c305dc4 Mon Sep 17 00:00:00 2001 From: Neeraj Shukla Date: Wed, 21 Oct 2020 20:35:39 +0530 Subject: [PATCH 37/44] added tests --- src/proxy.js | 7 +++---- test/commandLine.test.js | 24 ++++++++++++++++++++++++ test/proxy.test.js | 1 - test/reverseProxy.test.js | 1 - 4 files changed, 27 insertions(+), 6 deletions(-) diff --git a/src/proxy.js b/src/proxy.js index 23ffa8d..5c37cb7 100644 --- a/src/proxy.js +++ b/src/proxy.js @@ -82,14 +82,13 @@ var RdHandler = { */ requestHandler: function (clientRequest, clientResponse) { clientRequest.id = ++RdHandler._requestCounter + '::' + uuidv4(); - var url = clientRequest.url; var request = { method: clientRequest.method, url: clientRequest.url, headers: clientRequest.headers, data: [] }; - RdGlobalConfig.reqLogger.info(constants.TOPICS.CLIENT_REQUEST_START, request.method + ' ' + url, + RdGlobalConfig.reqLogger.info(constants.TOPICS.CLIENT_REQUEST_START, request.method + ' ' + clientRequest.url, false, { headers: request.headers }, @@ -104,7 +103,7 @@ var RdHandler = { ReqLib.call(paramsForRequest, clientRequest, constants.REQUEST_TYPES.PROXY) .then(function (response) { - RdGlobalConfig.reqLogger.info(constants.TOPICS.CLIENT_RESPONSE_END, clientRequest.method + ' ' + url + ', Status Code: ' + response.statusCode, + RdGlobalConfig.reqLogger.info(constants.TOPICS.CLIENT_RESPONSE_END, clientRequest.method + ' ' + clientRequest.url + ', Status Code: ' + response.statusCode, false, { data: response.data, headers: response.headers, @@ -122,7 +121,7 @@ var RdHandler = { clientRequest.id); var errorResponse = RdHandler._frameErrorResponse(furtherRequestOptions, err.message.toString()); - RdGlobalConfig.reqLogger.error(constants.TOPICS.CLIENT_RESPONSE_END, clientRequest.method + ' ' + url + ', Status Code: ' + errorResponse.statusCode, + RdGlobalConfig.reqLogger.error(constants.TOPICS.CLIENT_RESPONSE_END, clientRequest.method + ' ' + clientRequest.url + ', Status Code: ' + errorResponse.statusCode, false, errorResponse.data, clientRequest.id); diff --git a/test/commandLine.test.js b/test/commandLine.test.js index 1866583..d5a9140 100644 --- a/test/commandLine.test.js +++ b/test/commandLine.test.js @@ -39,6 +39,30 @@ describe('CommandLineManager', function () { expect(RdGlobalConfig.proxy.port).to.eql(9687); }); + it('parse proxy-host with without protocol', function () { + sinon.stub(console, 'log'); + argv = argv.concat(['--proxy-host', 'host']); + CommandLineManager.processArgs(argv); + console.log.restore(); + expect(RdGlobalConfig.proxy.host).to.eql(proxy_host_actual_value); + }); + + it('parse proxy-host with protocol http', function () { + sinon.stub(console, 'log'); + argv = argv.concat(['--proxy-host', 'http://host']); + CommandLineManager.processArgs(argv); + console.log.restore(); + expect(RdGlobalConfig.proxy.host).to.eql(proxy_host_actual_value); + }); + + it('parse proxy-host with protocol https', function () { + sinon.stub(console, 'log'); + argv = argv.concat(['--proxy-host', 'https://host', '--proxy-port', '9687']); + CommandLineManager.processArgs(argv); + console.log.restore(); + expect(RdGlobalConfig.proxy.host).to.eql('https://host'); + }); + it('proxy-port is set to the default value when its not in the expected range', function () { sinon.stub(console, 'log'); argv = argv.concat(['--proxy-host', 'host', '--proxy-port', '99999']); diff --git a/test/proxy.test.js b/test/proxy.test.js index 84de46f..0d7011d 100644 --- a/test/proxy.test.js +++ b/test/proxy.test.js @@ -32,7 +32,6 @@ describe('RdHandler', function () { testHelper.deleteLoggers(); testHelper.deleteHandlers(); nock.cleanAll(); - RdGlobalConfig.SCHEME.restore(); RdGlobalConfig.SCHEME = originalScheme; }); diff --git a/test/reverseProxy.test.js b/test/reverseProxy.test.js index 1eff9df..2bcccbb 100644 --- a/test/reverseProxy.test.js +++ b/test/reverseProxy.test.js @@ -31,7 +31,6 @@ describe('RdHandler', function () { testHelper.deleteLoggers(); testHelper.deleteHandlers(); nock.cleanAll(); - RdGlobalConfig.SCHEME.restore(); RdGlobalConfig.SCHEME = originalScheme; }); From 406abf6aa9bd1d6fec3b8984bb17c0649ac4715d Mon Sep 17 00:00:00 2001 From: Neeraj Shukla Date: Wed, 21 Oct 2020 20:37:57 +0530 Subject: [PATCH 38/44] bug fix --- src/requestLib.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/requestLib.js b/src/requestLib.js index 79e80ca..cbe9dcb 100644 --- a/src/requestLib.js +++ b/src/requestLib.js @@ -147,7 +147,7 @@ var RequestLib = { return Utils.delay(RdGlobalConfig.RETRY_DELAY) .then(function () { - return RequestLib.call(params, clientRequest, retries - 1, false); + return RequestLib.call(params, clientRequest, requestType, retries - 1, false); }); } else { throw err; From 4930ed165e597902206a6ad5a2bf5dc360812abd Mon Sep 17 00:00:00 2001 From: Neeraj Shukla Date: Wed, 21 Oct 2020 21:45:38 +0530 Subject: [PATCH 39/44] refactor --- config/constants.js | 2 +- src/commandLine.js | 8 ++++---- src/logger.js | 2 +- src/proxy.js | 2 +- src/requestLib.js | 24 ++++++++++++------------ src/reverseProxy.js | 15 ++++++++------- src/stats/linuxStats.js | 1 - src/stats/macStats.js | 4 ++-- test/commandLine.test.js | 8 ++++---- test/connectivity.test.js | 1 - test/proxy.test.js | 4 ++-- test/reverseProxy.test.js | 8 ++++---- test/stats/macStats.test.js | 12 ++++++------ 13 files changed, 45 insertions(+), 46 deletions(-) diff --git a/config/constants.js b/config/constants.js index 9fea375..763b3eb 100644 --- a/config/constants.js +++ b/config/constants.js @@ -17,7 +17,7 @@ module.exports.PORTS = { MIN: 1 }; -module.exports.REQUEST_TYPES = { +module.exports.SERVER_TYPES = { PROXY: "proxy", REVERSE_PROXY: "reverse proxy" }; diff --git a/src/commandLine.js b/src/commandLine.js index 957fddb..c0850ea 100644 --- a/src/commandLine.js +++ b/src/commandLine.js @@ -100,7 +100,7 @@ var CommandLineManager = { argv.splice(index, 1); } } - + // delay for retries in case of request failures index = argv.indexOf('--retry-delay'); if (index !== -1) { @@ -156,7 +156,7 @@ var CommandLineManager = { argv.splice(index, 1); } } - + // process proxy host index = argv.indexOf('--proxy-host'); if (index !== -1) { @@ -195,7 +195,7 @@ var CommandLineManager = { argv.splice(index, 1); } } - + // if proxy port value in invalid or doesn't exist and host exists, set the default value if (RdGlobalConfig.proxy && RdGlobalConfig.proxy.host && (invalidArgs.has('--proxy-port') || !RdGlobalConfig.proxy.port)) { console.log('\nSetting Default Proxy Port:', constants.DEFAULT_PROXY_PORT, '\n'); @@ -234,7 +234,7 @@ var CommandLineManager = { argv.splice(index, 1); } } - + // if proxy pass is invalid or doesn't exist and username exists, set the password as empty if (RdGlobalConfig.proxy && RdGlobalConfig.proxy.username && (invalidArgs.has('--proxy-pass') || !RdGlobalConfig.proxy.password)) { console.log('Setting Proxy Password as Empty\n'); diff --git a/src/logger.js b/src/logger.js index f434a28..efdccb0 100644 --- a/src/logger.js +++ b/src/logger.js @@ -36,7 +36,7 @@ var LogManager = { newLogger.transports.file.json = false; newLogger.info("************* LOGGER INITIALIZED **************\r\n"); - + return { info: function (topic, message, stringify, data, uuid) { stringify = stringify || false; diff --git a/src/proxy.js b/src/proxy.js index 5c37cb7..6f5faf5 100644 --- a/src/proxy.js +++ b/src/proxy.js @@ -101,7 +101,7 @@ var RdHandler = { furtherRequestOptions: furtherRequestOptions }; - ReqLib.call(paramsForRequest, clientRequest, constants.REQUEST_TYPES.PROXY) + ReqLib.call(paramsForRequest, clientRequest, constants.SERVER_TYPES.PROXY) .then(function (response) { RdGlobalConfig.reqLogger.info(constants.TOPICS.CLIENT_RESPONSE_END, clientRequest.method + ' ' + clientRequest.url + ', Status Code: ' + response.statusCode, false, { diff --git a/src/requestLib.js b/src/requestLib.js index cbe9dcb..6c356b7 100644 --- a/src/requestLib.js +++ b/src/requestLib.js @@ -20,7 +20,7 @@ var RequestLib = { * @param {http.IncomingMessage} clientRequest * @param {Number} retries */ - _makeRequest: function (schemeObj, params, clientRequest, requestType, retries) { + _makeRequest: function (schemeObj, params, clientRequest, serverType, retries) { return new Promise(function (resolve, reject) { var requestOptions = Object.assign({}, params.furtherRequestOptions); requestOptions.agent = RdGlobalConfig.SCHEME === 'http' ? httpKeepAliveAgent : httpsKeepAliveAgent; @@ -45,12 +45,12 @@ var RequestLib = { response.on('data', function (chunk) { responseToSend.data.push(chunk); }); - + response.on('end', function () { responseToSend.data = Buffer.concat(responseToSend.data).toString(); resolve(responseToSend); }); - + response.on('error', function (err) { reject({ message: err, @@ -61,9 +61,9 @@ var RequestLib = { // Log the request that will be initiated on behalf of the client request.on('finish', function () { - var url = RdGlobalConfig.SCHEME + "://" + requestOptions.host + requestOptions.path; + var requestUrl = RdGlobalConfig.SCHEME + "://" + requestOptions.host + requestOptions.path; RdGlobalConfig.reqLogger.info(constants.TOPICS.TOOL_REQUEST_WITH_RETRIES + retries, - clientRequest.method + ' ' + url, false, Object.assign({}, params.furtherRequestOptions, { + clientRequest.method + ' ' + requestUrl, false, Object.assign({}, params.furtherRequestOptions, { data: Buffer.concat(params.request.data).toString() }), clientRequest.id); @@ -98,7 +98,7 @@ var RequestLib = { }); } }); - + clientRequest.on('error', function (err) { request.end(); reject({ @@ -106,10 +106,10 @@ var RequestLib = { customTopic: constants.TOPICS.CLIENT_REQUEST_WITH_RETRIES + retries }); }); - + clientRequest.on('end', function () { - var url = requestType === constants.REQUEST_TYPES.PROXY ? clientRequest.url : "http://" + clientRequest.headers.host + clientRequest.url; - RdGlobalConfig.reqLogger.info(constants.TOPICS.CLIENT_REQUEST_END, params.request.method + ' ' + url, false, { + var requestUrl = serverType === constants.SERVER_TYPES.PROXY ? clientRequest.url : "http://" + clientRequest.headers.host + requestOptions.path; + RdGlobalConfig.reqLogger.info(constants.TOPICS.CLIENT_REQUEST_END, params.request.method + ' ' + requestUrl, false, { data: Buffer.concat(params.request.data).toString() }, clientRequest.id); @@ -128,10 +128,10 @@ var RequestLib = { * @param {http.IncomingMessage} clientRequest * @param {Number} retries */ - call: function (params, clientRequest, requestType, retries) { + call: function (params, clientRequest, serverType, retries) { retries = (typeof retries === 'number') ? Math.min(constants.MAX_RETRIES, Math.max(retries, 0)) : constants.MAX_RETRIES; var schemeObj = RdGlobalConfig.SCHEME === "http" ? http : https; - return RequestLib._makeRequest(schemeObj, params, clientRequest, requestType, retries) + return RequestLib._makeRequest(schemeObj, params, clientRequest, serverType, retries) .catch(function (err) { var errTopic = err.customTopic || constants.TOPICS.UNEXPECTED_ERROR; // Collect Network & Connectivity Logs whenever a request fails @@ -147,7 +147,7 @@ var RequestLib = { return Utils.delay(RdGlobalConfig.RETRY_DELAY) .then(function () { - return RequestLib.call(params, clientRequest, requestType, retries - 1, false); + return RequestLib.call(params, clientRequest, serverType, retries - 1, false); }); } else { throw err; diff --git a/src/reverseProxy.js b/src/reverseProxy.js index 7773442..0884fac 100644 --- a/src/reverseProxy.js +++ b/src/reverseProxy.js @@ -84,28 +84,29 @@ var RdHandler = { */ requestHandler: function (clientRequest, clientResponse) { clientRequest.id = ++RdHandler._requestCounter + '::' + uuidv4(); - var url = "http://" + clientRequest.headers.host + clientRequest.url; + var parsedClientUrl = url.parse(clientRequest.url); + var requestUrl = "http://" + clientRequest.headers.host + parsedClientUrl.path; var request = { method: clientRequest.method, url: clientRequest.url, headers: clientRequest.headers, data: [] }; - RdGlobalConfig.reqLogger.info(constants.TOPICS.CLIENT_REQUEST_START, request.method + ' ' + url, + RdGlobalConfig.reqLogger.info(constants.TOPICS.CLIENT_REQUEST_START, request.method + ' ' + requestUrl, false, { headers: request.headers }, clientRequest.id); - + var furtherRequestOptions = RdHandler._generateRequestOptions(clientRequest); var paramsForRequest = { request: request, furtherRequestOptions: furtherRequestOptions }; - ReqLib.call(paramsForRequest, clientRequest, constants.REQUEST_TYPES.REVERSE_PROXY) + ReqLib.call(paramsForRequest, clientRequest, constants.SERVER_TYPES.REVERSE_PROXY) .then(function (response) { - RdGlobalConfig.reqLogger.info(constants.TOPICS.CLIENT_RESPONSE_END, clientRequest.method + ' ' + url + ', Status Code: ' + response.statusCode, + RdGlobalConfig.reqLogger.info(constants.TOPICS.CLIENT_RESPONSE_END, clientRequest.method + ' ' + requestUrl + ', Status Code: ' + response.statusCode, false, { data: response.data, headers: response.headers, @@ -116,14 +117,14 @@ var RdHandler = { clientResponse.end(response.data); }) .catch(function (err) { - RdGlobalConfig.reqLogger.error(err.customTopic || constants.TOPICS.UNEXPECTED_ERROR, clientRequest.method + ' ' + clientRequest.url, + RdGlobalConfig.reqLogger.error(err.customTopic || constants.TOPICS.UNEXPECTED_ERROR, clientRequest.method + ' ' + requestUrl, false, { errorMessage: err.message.toString() }, clientRequest.id); var errorResponse = RdHandler._frameErrorResponse(furtherRequestOptions, err.message.toString()); - RdGlobalConfig.reqLogger.error(constants.TOPICS.CLIENT_RESPONSE_END, clientRequest.method + ' ' + url + ', Status Code: ' + errorResponse.statusCode, + RdGlobalConfig.reqLogger.error(constants.TOPICS.CLIENT_RESPONSE_END, clientRequest.method + ' ' + requestUrl + ', Status Code: ' + errorResponse.statusCode, false, errorResponse.data, clientRequest.id); diff --git a/src/stats/linuxStats.js b/src/stats/linuxStats.js index 3aff278..4c70d72 100644 --- a/src/stats/linuxStats.js +++ b/src/stats/linuxStats.js @@ -45,7 +45,6 @@ LinuxStats.mem = function (callback) { memStats.free = parseInt(Utils.fetchPropertyValue(memStatLines, 'memfree')); memStats.free = memStats.free ? memStats.free * 1024 : os.freemem(); memStats.used = memStats.total - memStats.free; - memStats.swapTotal = parseInt(Utils.fetchPropertyValue(memStatLines, 'swaptotal')); memStats.swapTotal = memStats.swapTotal ? memStats.swapTotal * 1024 : 0; memStats.swapFree = parseInt(Utils.fetchPropertyValue(memStatLines, 'swapfree')); diff --git a/src/stats/macStats.js b/src/stats/macStats.js index 49afc7d..fe32928 100644 --- a/src/stats/macStats.js +++ b/src/stats/macStats.js @@ -49,11 +49,11 @@ MacStats.mem = function (callback) { case 'total': memStats.swapTotal = parseFloat(statLines[index].split('=')[1].trim()) * 1024 * 1024; break; - + case 'used': memStats.swapUsed = parseFloat(statLines[index].split('=')[1].trim()) * 1024 * 1024; break; - + case 'free': memStats.swapFree = parseFloat(statLines[index].split('=')[1].trim()) * 1024 * 1024; break; diff --git a/test/commandLine.test.js b/test/commandLine.test.js index d5a9140..445813b 100644 --- a/test/commandLine.test.js +++ b/test/commandLine.test.js @@ -9,7 +9,7 @@ describe('CommandLineManager', function () { var argv; var proxy_host_actual_value = "http://host"; - + before(function () { console.log("NOTE: 'console.log' will be stubbed. In case any test fails, try removing the stub to see the logs"); }); @@ -39,7 +39,7 @@ describe('CommandLineManager', function () { expect(RdGlobalConfig.proxy.port).to.eql(9687); }); - it('parse proxy-host with without protocol', function () { + it('parse proxy-host inputted without protocol', function () { sinon.stub(console, 'log'); argv = argv.concat(['--proxy-host', 'host']); CommandLineManager.processArgs(argv); @@ -303,7 +303,7 @@ describe('CommandLineManager', function () { console.log.restore(); sinon.assert.called(process.exit); }); - + it('exits with invalid args if the reverse port arg is provided without any value', function () { argv = argv.concat(['--reverse-proxy-port']); sinon.stub(console, 'log'); @@ -352,7 +352,7 @@ describe('CommandLineManager', function () { CommandLineManager.processArgs(argv); expect(RdGlobalConfig.SCHEME).to.eql('https'); }); - + // --request-timeout it("sets the timeout for the request being fired from the tool using the arg --request-timeout", function () { argv = argv.concat(['--request-timeout', '200000']); diff --git a/test/connectivity.test.js b/test/connectivity.test.js index 87d7cc6..b64e759 100644 --- a/test/connectivity.test.js +++ b/test/connectivity.test.js @@ -163,7 +163,6 @@ describe('Connectivity Checker for BrowserStack Components', function () { testHelper.nockGetRequestWithError(constants.HUB_STATUS_URL, 'https'); testHelper.nockGetRequestWithError(constants.RAILS_AUTOMATE, 'http'); testHelper.nockGetRequestWithError(constants.RAILS_AUTOMATE, 'https'); - }); afterEach(function () { diff --git a/test/proxy.test.js b/test/proxy.test.js index 0d7011d..fbfa9a9 100644 --- a/test/proxy.test.js +++ b/test/proxy.test.js @@ -17,7 +17,7 @@ describe('RdHandler', function () { testHelper.initializeDummyProxy(); originalScheme = RdGlobalConfig.SCHEME; RdGlobalConfig.SCHEME = 'http'; - + RdHandler.startProxyServer(RdGlobalConfig.RD_HANDLER_PROXY_PORT, function (port) { console.log('Test Network Utility Proxy Server Started on Port: ', port); done(); @@ -130,7 +130,7 @@ describe('RdHandler', function () { testHelper.initializeDummyProxy(); originalScheme = RdGlobalConfig.SCHEME; RdGlobalConfig.SCHEME = 'https'; - + RdHandler.startProxyServer(RdGlobalConfig.RD_HANDLER_PROXY_PORT, function (port) { console.log('Test Network Utility Proxy Server Started on Port: ', port); done(); diff --git a/test/reverseProxy.test.js b/test/reverseProxy.test.js index 2bcccbb..9ae95b8 100644 --- a/test/reverseProxy.test.js +++ b/test/reverseProxy.test.js @@ -16,7 +16,7 @@ describe('RdHandler', function () { testHelper.initializeDummyHandlers(); originalScheme = RdGlobalConfig.SCHEME; RdGlobalConfig.SCHEME = 'http'; - + RdHandler.startReverseProxyServer(RdGlobalConfig.RD_HANDLER_REVERSE_PROXY_PORT, function (port) { console.log('Test Network Utility Reverse Proxy Server Started on Port: ', port); done(); @@ -44,7 +44,7 @@ describe('RdHandler', function () { headers: {}, path: constants.HUB_STATUS_PATH }; - + var responseData = []; var request = http.request(reqOptions, function (response) { @@ -128,7 +128,7 @@ describe('RdHandler', function () { testHelper.initializeDummyHandlers(); originalScheme = RdGlobalConfig.SCHEME; RdGlobalConfig.SCHEME = 'https'; - + RdHandler.startReverseProxyServer(RdGlobalConfig.RD_HANDLER_REVERSE_PROXY_PORT, function (port) { console.log('Test Network Utility Reverse Proxy Server Started on Port: ', port); done(); @@ -157,7 +157,7 @@ describe('RdHandler', function () { headers: {}, path: constants.HUB_STATUS_PATH }; - + var responseData = []; var request = http.request(reqOptions, function (response) { diff --git a/test/stats/macStats.test.js b/test/stats/macStats.test.js index d2e3266..b9bf4b4 100644 --- a/test/stats/macStats.test.js +++ b/test/stats/macStats.test.js @@ -11,21 +11,21 @@ describe('MacStats', function () { it('callbacks with the result of cpu stats', function () { var stats = "CPU Stats Generated"; var statsWithHeaderFooter = "Header" + os.EOL + stats + os.EOL + "Footer" + os.EOL; - + sinon.stub(cp, 'exec').callsArgWith(1, null, stats); sinon.stub(Utils, 'generateHeaderAndFooter').returns(statsWithHeaderFooter); - + MacStats.cpu(function (result) { expect(result).to.eql(statsWithHeaderFooter); }); - + cp.exec.restore(); Utils.generateHeaderAndFooter.restore(); }); it('callbacks with proper message when no stats are available', function () { sinon.stub(cp, 'exec').callsArgWith(1, "err", null); - + MacStats.cpu(function (result) { expect(result).to.eql(constants.STATIC_MESSAGES.NO_REPORT_GENERATED + 'CPU' + os.EOL); }); @@ -99,10 +99,10 @@ describe('MacStats', function () { content: 'resultThree', generatedAt: new Date().toISOString() }]; - + sinon.stub(Utils, 'execMultiple').callsArgWith(1, results); sinon.stub(Utils, 'generateHeaderAndFooter').returns('headerFooterContent'); - + MacStats.network(function (result) { sinon.assert.calledThrice(Utils.generateHeaderAndFooter); expect(result).to.eql('headerFooterContent'.repeat(3)); From 95f96f2f0e0556208ed8b1aca05b9f2c7032dc69 Mon Sep 17 00:00:00 2001 From: Neeraj Shukla Date: Fri, 23 Oct 2020 18:49:09 +0530 Subject: [PATCH 40/44] bug fix for logging --- config/constants.js | 5 ----- src/proxy.js | 15 ++++++++++----- src/requestLib.js | 16 +++++++--------- src/reverseProxy.js | 17 ++++++++++------- test/reverseProxy.test.js | 2 +- 5 files changed, 28 insertions(+), 27 deletions(-) diff --git a/config/constants.js b/config/constants.js index 763b3eb..fbec60a 100644 --- a/config/constants.js +++ b/config/constants.js @@ -17,11 +17,6 @@ module.exports.PORTS = { MIN: 1 }; -module.exports.SERVER_TYPES = { - PROXY: "proxy", - REVERSE_PROXY: "reverse proxy" -}; - module.exports.LINE_LENGTH = 70; module.exports.PROTOCOL_REGEX = /(^\w+:|^)\/\//; diff --git a/src/proxy.js b/src/proxy.js index 6f5faf5..bf18c67 100644 --- a/src/proxy.js +++ b/src/proxy.js @@ -81,6 +81,11 @@ var RdHandler = { * @param {http.ServerResponse} clientResponse */ requestHandler: function (clientRequest, clientResponse) { + var parsedClientUrl = url.parse(clientRequest.url); + var metaData = { + clientRequestUrl: clientRequest.url, + toolRequestUrl: RdGlobalConfig.SCHEME + "://" + parsedClientUrl.host + parsedClientUrl.path + }; clientRequest.id = ++RdHandler._requestCounter + '::' + uuidv4(); var request = { method: clientRequest.method, @@ -88,7 +93,7 @@ var RdHandler = { headers: clientRequest.headers, data: [] }; - RdGlobalConfig.reqLogger.info(constants.TOPICS.CLIENT_REQUEST_START, request.method + ' ' + clientRequest.url, + RdGlobalConfig.reqLogger.info(constants.TOPICS.CLIENT_REQUEST_START, clientRequest.method + ' ' + metaData.clientRequestUrl, false, { headers: request.headers }, @@ -101,9 +106,9 @@ var RdHandler = { furtherRequestOptions: furtherRequestOptions }; - ReqLib.call(paramsForRequest, clientRequest, constants.SERVER_TYPES.PROXY) + ReqLib.call(paramsForRequest, clientRequest, metaData) .then(function (response) { - RdGlobalConfig.reqLogger.info(constants.TOPICS.CLIENT_RESPONSE_END, clientRequest.method + ' ' + clientRequest.url + ', Status Code: ' + response.statusCode, + RdGlobalConfig.reqLogger.info(constants.TOPICS.CLIENT_RESPONSE_END, clientRequest.method + ' ' + metaData.clientRequestUrl + ', Status Code: ' + response.statusCode, false, { data: response.data, headers: response.headers, @@ -114,14 +119,14 @@ var RdHandler = { clientResponse.end(response.data); }) .catch(function (err) { - RdGlobalConfig.reqLogger.error(err.customTopic || constants.TOPICS.UNEXPECTED_ERROR, clientRequest.method + ' ' + clientRequest.url, + RdGlobalConfig.reqLogger.error(err.customTopic || constants.TOPICS.UNEXPECTED_ERROR, clientRequest.method + ' ' + metaData.toolRequestUrl, false, { errorMessage: err.message.toString() }, clientRequest.id); var errorResponse = RdHandler._frameErrorResponse(furtherRequestOptions, err.message.toString()); - RdGlobalConfig.reqLogger.error(constants.TOPICS.CLIENT_RESPONSE_END, clientRequest.method + ' ' + clientRequest.url + ', Status Code: ' + errorResponse.statusCode, + RdGlobalConfig.reqLogger.error(constants.TOPICS.CLIENT_RESPONSE_END, clientRequest.method + ' ' + metaData.clientRequestUrl + ', Status Code: ' + errorResponse.statusCode, false, errorResponse.data, clientRequest.id); diff --git a/src/requestLib.js b/src/requestLib.js index 6c356b7..159663b 100644 --- a/src/requestLib.js +++ b/src/requestLib.js @@ -20,7 +20,7 @@ var RequestLib = { * @param {http.IncomingMessage} clientRequest * @param {Number} retries */ - _makeRequest: function (schemeObj, params, clientRequest, serverType, retries) { + _makeRequest: function (schemeObj, params, clientRequest, metaData, retries) { return new Promise(function (resolve, reject) { var requestOptions = Object.assign({}, params.furtherRequestOptions); requestOptions.agent = RdGlobalConfig.SCHEME === 'http' ? httpKeepAliveAgent : httpsKeepAliveAgent; @@ -61,9 +61,8 @@ var RequestLib = { // Log the request that will be initiated on behalf of the client request.on('finish', function () { - var requestUrl = RdGlobalConfig.SCHEME + "://" + requestOptions.host + requestOptions.path; RdGlobalConfig.reqLogger.info(constants.TOPICS.TOOL_REQUEST_WITH_RETRIES + retries, - clientRequest.method + ' ' + requestUrl, false, Object.assign({}, params.furtherRequestOptions, { + clientRequest.method + ' ' + metaData.toolRequestUrl, false, Object.assign({}, params.furtherRequestOptions, { data: Buffer.concat(params.request.data).toString() }), clientRequest.id); @@ -108,8 +107,7 @@ var RequestLib = { }); clientRequest.on('end', function () { - var requestUrl = serverType === constants.SERVER_TYPES.PROXY ? clientRequest.url : "http://" + clientRequest.headers.host + requestOptions.path; - RdGlobalConfig.reqLogger.info(constants.TOPICS.CLIENT_REQUEST_END, params.request.method + ' ' + requestUrl, false, { + RdGlobalConfig.reqLogger.info(constants.TOPICS.CLIENT_REQUEST_END, params.request.method + ' ' + metaData.clientRequestUrl, false, { data: Buffer.concat(params.request.data).toString() }, clientRequest.id); @@ -128,10 +126,10 @@ var RequestLib = { * @param {http.IncomingMessage} clientRequest * @param {Number} retries */ - call: function (params, clientRequest, serverType, retries) { + call: function (params, clientRequest, metaData, retries) { retries = (typeof retries === 'number') ? Math.min(constants.MAX_RETRIES, Math.max(retries, 0)) : constants.MAX_RETRIES; var schemeObj = RdGlobalConfig.SCHEME === "http" ? http : https; - return RequestLib._makeRequest(schemeObj, params, clientRequest, serverType, retries) + return RequestLib._makeRequest(schemeObj, params, clientRequest, metaData, retries) .catch(function (err) { var errTopic = err.customTopic || constants.TOPICS.UNEXPECTED_ERROR; // Collect Network & Connectivity Logs whenever a request fails @@ -139,7 +137,7 @@ var RequestLib = { RdGlobalConfig.connHandler(errTopic, clientRequest.id); if (retries > 0) { - RdGlobalConfig.reqLogger.error(errTopic, clientRequest.method + ' ' + clientRequest.url, + RdGlobalConfig.reqLogger.error(errTopic, clientRequest.method + ' ' + metaData.toolRequestUrl, false, { errorMessage: err.message.toString() }, @@ -147,7 +145,7 @@ var RequestLib = { return Utils.delay(RdGlobalConfig.RETRY_DELAY) .then(function () { - return RequestLib.call(params, clientRequest, serverType, retries - 1, false); + return RequestLib.call(params, clientRequest, metaData, retries - 1, false); }); } else { throw err; diff --git a/src/reverseProxy.js b/src/reverseProxy.js index 0884fac..0b7580b 100644 --- a/src/reverseProxy.js +++ b/src/reverseProxy.js @@ -83,16 +83,19 @@ var RdHandler = { * @param {http.ServerResponse} clientResponse */ requestHandler: function (clientRequest, clientResponse) { - clientRequest.id = ++RdHandler._requestCounter + '::' + uuidv4(); var parsedClientUrl = url.parse(clientRequest.url); - var requestUrl = "http://" + clientRequest.headers.host + parsedClientUrl.path; + var metaData = { + clientRequestUrl: "http://" + clientRequest.headers.host + parsedClientUrl.path, + toolRequestUrl: RdGlobalConfig.SCHEME + "://" + constants.HUB_HOST + parsedClientUrl.path + }; + clientRequest.id = ++RdHandler._requestCounter + '::' + uuidv4(); var request = { method: clientRequest.method, url: clientRequest.url, headers: clientRequest.headers, data: [] }; - RdGlobalConfig.reqLogger.info(constants.TOPICS.CLIENT_REQUEST_START, request.method + ' ' + requestUrl, + RdGlobalConfig.reqLogger.info(constants.TOPICS.CLIENT_REQUEST_START, clientRequest.method + ' ' + metaData.clientRequestUrl, false, { headers: request.headers }, @@ -104,9 +107,9 @@ var RdHandler = { request: request, furtherRequestOptions: furtherRequestOptions }; - ReqLib.call(paramsForRequest, clientRequest, constants.SERVER_TYPES.REVERSE_PROXY) + ReqLib.call(paramsForRequest, clientRequest, metaData) .then(function (response) { - RdGlobalConfig.reqLogger.info(constants.TOPICS.CLIENT_RESPONSE_END, clientRequest.method + ' ' + requestUrl + ', Status Code: ' + response.statusCode, + RdGlobalConfig.reqLogger.info(constants.TOPICS.CLIENT_RESPONSE_END, clientRequest.method + ' ' + metaData.clientRequestUrl + ', Status Code: ' + response.statusCode, false, { data: response.data, headers: response.headers, @@ -117,14 +120,14 @@ var RdHandler = { clientResponse.end(response.data); }) .catch(function (err) { - RdGlobalConfig.reqLogger.error(err.customTopic || constants.TOPICS.UNEXPECTED_ERROR, clientRequest.method + ' ' + requestUrl, + RdGlobalConfig.reqLogger.error(err.customTopic || constants.TOPICS.UNEXPECTED_ERROR, clientRequest.method + ' ' + metaData.toolRequestUrl, false, { errorMessage: err.message.toString() }, clientRequest.id); var errorResponse = RdHandler._frameErrorResponse(furtherRequestOptions, err.message.toString()); - RdGlobalConfig.reqLogger.error(constants.TOPICS.CLIENT_RESPONSE_END, clientRequest.method + ' ' + requestUrl + ', Status Code: ' + errorResponse.statusCode, + RdGlobalConfig.reqLogger.error(constants.TOPICS.CLIENT_RESPONSE_END, clientRequest.method + ' ' + metaData.clientRequestUrl + ', Status Code: ' + errorResponse.statusCode, false, errorResponse.data, clientRequest.id); diff --git a/test/reverseProxy.test.js b/test/reverseProxy.test.js index 9ae95b8..99def5e 100644 --- a/test/reverseProxy.test.js +++ b/test/reverseProxy.test.js @@ -184,7 +184,7 @@ describe('RdHandler', function () { host: 'localhost', port: RdGlobalConfig.RD_HANDLER_REVERSE_PROXY_PORT, headers: {}, - path: constants.HUB_STATUS_PATH + path: "http://user1:pass1@" + constants.HUB_HOST + constants.HUB_STATUS_PATH }; var responseData = []; From 4f25a94c3a5aa397f0674288d74870775b6becee Mon Sep 17 00:00:00 2001 From: Neeraj Shukla Date: Fri, 23 Oct 2020 19:20:24 +0530 Subject: [PATCH 41/44] tests with credentials in url --- test/proxy.test.js | 29 +++++++++++++++++++++++++++++ test/reverseProxy.test.js | 29 +++++++++++++++++++++++++++++ 2 files changed, 58 insertions(+) diff --git a/test/proxy.test.js b/test/proxy.test.js index fbfa9a9..cf5c993 100644 --- a/test/proxy.test.js +++ b/test/proxy.test.js @@ -177,6 +177,35 @@ describe('RdHandler', function () { }); it('Requests on behalf of the client via external proxy and returns the response', function (done) { + this.timeout = 5000; + testHelper.nockGetRequest(constants.HUB_STATUS_URL, 'https', null, 200); + testHelper.initializeDummyProxy(); + testHelper.nockProxyUrl(RdGlobalConfig.proxy, 'https', 'hub', null, 200); + var reqOptions = { + method: 'GET', + host: 'localhost', + port: RdGlobalConfig.RD_HANDLER_PROXY_PORT, + headers: {}, + path: constants.HUB_STATUS_URL + }; + + var responseData = []; + var request = http.request(reqOptions, function (response) { + + response.on('data', function (chunk) { + responseData.push(chunk); + }); + + response.on('end', function () { + assert(Buffer.concat(responseData).toString() === '{"data":"value"}'); + done(); + testHelper.deleteProxy(); + }); + }); + request.end(); + }); + + it('Requests on behalf of the client with username and password in url via external proxy and returns the response', function (done) { this.timeout = 5000; testHelper.nockGetRequest(constants.HUB_STATUS_URL, 'https', null, 200); testHelper.initializeDummyProxy(); diff --git a/test/reverseProxy.test.js b/test/reverseProxy.test.js index 99def5e..5a1b62b 100644 --- a/test/reverseProxy.test.js +++ b/test/reverseProxy.test.js @@ -175,6 +175,35 @@ describe('RdHandler', function () { }); it('Requests on behalf of the client via external proxy and returns the response', function (done) { + this.timeout = 5000; + testHelper.nockGetRequest(constants.HUB_STATUS_URL, 'https', null, 200); + testHelper.initializeDummyProxy(); + testHelper.nockProxyUrl(RdGlobalConfig.proxy, 'https', 'hub', null, 200); + var reqOptions = { + method: 'GET', + host: 'localhost', + port: RdGlobalConfig.RD_HANDLER_REVERSE_PROXY_PORT, + headers: {}, + path: constants.HUB_STATUS_PATH + }; + + var responseData = []; + var request = http.request(reqOptions, function (response) { + + response.on('data', function (chunk) { + responseData.push(chunk); + }); + + response.on('end', function () { + assert(Buffer.concat(responseData).toString() === '{"data":"value"}'); + done(); + testHelper.deleteProxy(); + }); + }); + request.end(); + }); + + it('Requests on behalf of the client with username and password in url via external proxy and returns the response', function (done) { this.timeout = 5000; testHelper.nockGetRequest(constants.HUB_STATUS_URL, 'https', null, 200); testHelper.initializeDummyProxy(); From 8828e6d0bc8d454c9f1b894206e39eb4a67d0695 Mon Sep 17 00:00:00 2001 From: Neeraj Shukla Date: Mon, 26 Oct 2020 18:23:56 +0530 Subject: [PATCH 42/44] doc changes --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index e7fc50c..d9eb7b6 100644 --- a/README.md +++ b/README.md @@ -32,6 +32,7 @@ - Windows: `RequestsDebugger.exe ` ## How to use +- To use tool as reverse proxy, you will have to replace `hub-cloud.browserstack.com` in hub url with `localhost:9688`. - To use tool as a proxy, you will have to set the proxy to be used by your client binding to `localhost:9687`. i.e. - For Java: - ``` @@ -42,8 +43,7 @@ - Set your system's env variable `http_proxy=localhost:9687` and Ruby's Selenium Client Binding will pick the value. Or, - Run you test by giving the environment variable to your command itself, i.e. `http_proxy=localhost:9687 ruby ` - Similarly, you can also set proxy for other client bindings. -- To use tool as reverse proxy, you will have to replace hub-cloud.browserstack.com in hub url with `localhost:9688`. - + ## Steps to build the executables - Linux - `npm run build:linux` From a9ff6538515e280326681967eb54c334e2b459aa Mon Sep 17 00:00:00 2001 From: Rohan Chougule Date: Mon, 26 Oct 2020 18:33:09 +0530 Subject: [PATCH 43/44] add: github actions to trigger unit tests. logo and badges --- .github/workflows/unit-tests.yml | 32 ++++++++++++++++++++++++++++++++ README.md | 9 +++++++++ 2 files changed, 41 insertions(+) create mode 100644 .github/workflows/unit-tests.yml diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml new file mode 100644 index 0000000..fea2ae5 --- /dev/null +++ b/.github/workflows/unit-tests.yml @@ -0,0 +1,32 @@ +name: 'unit-tests' +on: + pull_request: + paths: + - './**' + - '.github/workflows/unit-tests*' + push: + paths: + - './**' + - '.github/workflows/unit-tests*' + +jobs: + unit-tests: + runs-on: ${{ matrix.operating-system }} + strategy: + matrix: + operating-system: [ubuntu-latest, macos-latest] + steps: + - uses: actions/checkout@v2 + + - name: Set Node.js 4.9.1 + uses: actions/setup-node@master + with: + node-version: 4.9.1 + + - name: npm install + working-directory: ./ + run: npm install + + - name: Lint and Unit tests + working-directory: ./ + run: npm run test \ No newline at end of file diff --git a/README.md b/README.md index e7fc50c..6cf0a83 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,13 @@ ## Requests Debugger + +

+ BrowserStack Logo +

+ +

+ unit-tests build status +

+ ### Tool for debugging client side failure of requests, leading to requests getting dropped or not reaching BrowserStack. ## Features From 2970e3c88cf0d4d19adcf48166d89e09143fcb42 Mon Sep 17 00:00:00 2001 From: Rohan Chougule Date: Mon, 26 Oct 2020 18:36:47 +0530 Subject: [PATCH 44/44] fix: EOL. Replaced browserstack links for badge --- .github/workflows/unit-tests.yml | 2 +- README.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index fea2ae5..b382911 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -29,4 +29,4 @@ jobs: - name: Lint and Unit tests working-directory: ./ - run: npm run test \ No newline at end of file + run: npm run test diff --git a/README.md b/README.md index 6cf0a83..e9a3d97 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@

- unit-tests build status + unit-tests build status

### Tool for debugging client side failure of requests, leading to requests getting dropped or not reaching BrowserStack.