diff --git a/.gitignore b/.gitignore index c2658d7..fa7b2a6 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,4 @@ +coverage/ node_modules/ +.nyc_output/ +package-lock.json diff --git a/.travis.yml b/.travis.yml index 0a74840..7b2edde 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,5 +1,6 @@ language: node_js node_js: -- "4" -- "6" -script: npm test + - 6 + - 8 + - node +script: npm run ci diff --git a/README.md b/README.md index 92649ef..a2afc79 100644 --- a/README.md +++ b/README.md @@ -1,31 +1,23 @@ -# limited-request-queue [![NPM Version][npm-image]][npm-url] [![Bower Version][bower-image]][bower-url] [![Build Status][travis-image]][travis-url] [![Dependency Status][david-image]][david-url] +# limited-request-queue [![NPM Version][npm-image]][npm-url] ![File Size][filesize-image] [![Build Status][travis-image]][travis-url] [![Coverage Status][coveralls-image]][coveralls-url] [![Dependency Monitor][greenkeeper-image]][greenkeeper-url] > Interactively manage concurrency for outbound requests. -Features: -* Concurrency & rate limiting prevents overload on your server -* Per-Host concurrency limiting prevents overload on everyone else's servers -* Pause/Resume at any time -* Works in the browser (~4.4KB not gzipped) + +* Concurrency & rate limiting prevents overload on your network. +* Per-Host concurrency limiting prevents overload on your target network(s). +* Pause/Resume at any time. ```js -// Will work with any similar module, not just "request" -var request = require("request"); -var RequestQueue = require("limited-request-queue"); - -var queue = new RequestQueue(null, { - item: function(input, done) { - request(input.url, function(error, response) { - done(); - }); - }, - end: function() { - console.log("Queue completed!"); - } -}); - -var urls = ["http://website.com/dir1/", "http://website.com/dir2/"]; -urls.forEach(queue.enqueue, queue); +const RequestQueue = require('limited-request-queue'); + +const queue = new RequestQueue() + .on('item', (url, data, done) => { + yourRequestLib(url, () => done()); + }) + .on('end', () => console.log('Queue completed!')); + +const urls = ['http://domain.com/dir1/', 'http://domain.com/dir2/']; +urls.forEach(url => queue.enqueue(new URL(url))); setTimeout(queue.pause, 500); setTimeout(queue.resume, 5000); @@ -34,41 +26,44 @@ setTimeout(queue.resume, 5000); ## Installation -[Node.js](http://nodejs.org/) `>= 4` is required. To install, type this at the command line: +[Node.js](http://nodejs.org/) `>= 6` is required. To install, type this at the command line: ```shell npm install limited-request-queue ``` -Note: for use in a web browser, you will likely need [`Object.assign`](https://developer.mozilla.org/en/docs/Web/JavaScript/Reference/Global_Objects/Object/assign) and [`URL`](https://developer.mozilla.org/en/docs/Web/API/URL/URL) polyfills for maximum coverage. +Note: for use in a web browser, you will likely need an ES2015/ES6 transpiler for maximum coverage. ## Constructor ```js -new RequestQueue(options, handlers); +new RequestQueue(options); ``` -## Methods +## Methods & Properties + +All methods from [`EventEmitter`](https://nodejs.org/api/events.html#events_class_eventemitter) are available. ### `.dequeue(id)` -Removes a queue item from the queue. Use of this function is likely not needed as items are auto-dequeued when their turn is reached. Returns `true` on success or an `Error` on failure. +Removes a queue item from the queue. Returns `true` if a queue item was removed and `false` if not. Use of this function is likely not needed as items are auto-dequeued when their turn is reached. -### `.enqueue(input)` -Adds a URL to the queue. `input` can either be a URL `String` or a configuration `Object`. Returns a queue ID on success or an `Error` on failure. +### `.enqueue(url[, data, options])` +Adds a URL to the queue. Returns a queue item ID on success. -If `input` is an `Object`, it will accept the following keys: +* `url` *must* a [`URL`](https://developer.mozilla.org/en/docs/Web/API/URL/) instance. +* `data` is optional and can be of any type. +* `options` is an optional `Object` that overrides any defined options in the constructor (except for `maxSockets`). -* `url`: a URL `String`, [`URL`](https://developer.mozilla.org/en/docs/Web/API/URL/) or [Node URL](https://nodejs.org/api/url.html#url_url_strings_and_url_objects)-compatible `Object`. -* `data`: additional data to be stored in the queue item. -* `id`: a unique ID (`String` or `Number`). If not defined, one will be generated. +### `.isPaused` +Returns `true` if the queue is currently paused and `false` if not. -### `.length()` +### `.length` Returns the total number of items in the queue, active and inactive. -### `.numActive()` +### `.numActive` Returns the number of items whose requests are currently in progress. -### `.numQueued()` +### `.numQueued` Returns the number of items that have not yet made requests. ### `.pause()` @@ -83,17 +78,17 @@ Resumes the queue. ### `options.ignorePorts` Type: `Boolean` Default value: `true` -Whether or not to treat identical hosts of different ports as a single concurrent group. **Example:** when `true`, http://mywebsite.com:80 and http://mywebsite.com:8080 may not have outgoing connections at the same time, but http://mywebsite.com:80 and http://yourwebsite.com:8080 will. +Whether or not to treat identical hosts of different ports as a single concurrent group. **Example:** when `true`, http://mydomain.com:80 and http://mydomain.com:8080 may not have outgoing connections at the same time, but http://mydomain.com:80 and http://yourdomain.com:8080 will. -### `options.ignoreSchemes` +### `options.ignoreProtocols` Type: `Boolean` Default value: `true` -Whether or not to treat identical hosts of different schemes/protocols as a single concurrent group. **Example:** when `true`, http://mywebsite.com and https://mywebsite.com may not have outgoing connections at the same time, but http://mywebsite.com and https://yourwebsite.com will. +Whether or not to treat identical hosts of different protocols as a single concurrent group. **Example:** when `true`, http://mydomain.com and https://mydomain.com may not have outgoing connections at the same time, but http://mydomain.com and https://yourdomain.com will. ### `options.ignoreSubdomains` Type: `Boolean` Default value: `true` -Whether or not to treat identical hosts of different subdomains as a single concurrent group. **Example:** when `true`, http://mywebsite.com and http://www.mywebsite.com may not have outgoing connections at the same time, but http://mywebsite.com and http://www.yourwebsite.com will. +Whether or not to treat identical domains of different subdomains as a single concurrent group. **Example:** when `true`, http://mydomain.com and http://www.mydomain.com may not have outgoing connections at the same time, but http://mydomain.com and http://www.yourdomain.com will. This option is not available in the browser version (due to extreme file size). @@ -104,7 +99,7 @@ The maximum number of connections allowed at any given time. A value of `0` will ### `options.maxSocketsPerHost` Type: `Number` -Default value: `1` +Default value: `2` The maximum number of connections per host allowed at any given time. A value of `0` will prevent anything from going out. A value of `Infinity` will provide no per-host concurrency limiting. ### `options.rateLimit` @@ -113,20 +108,21 @@ Default value: `0` The number of milliseconds to wait before each request. For a typical rate limiter, also set `maxSockets` to `1`. -## Handlers +## Events -### `handlers.end` -Called when the last item in the queue has been completed. +### `end` +Called when the last item in the queue has been completed/dequeued. -### `handlers.item` -Called when a queue item's turn has been reached. Arguments are: `input`, `done`. +### `item` +Called when a queue item's turn has been reached. Arguments are: `url`, `data`, `done`. Call the `done` function when your item's operations are complete. [npm-image]: https://img.shields.io/npm/v/limited-request-queue.svg [npm-url]: https://npmjs.org/package/limited-request-queue -[bower-image]: https://img.shields.io/bower/v/limited-request-queue.svg -[bower-url]: https://github.com/stevenvachon/limited-request-queue +[filesize-image]: https://img.shields.io/badge/size-4.1kB%20gzipped-blue.svg [travis-image]: https://img.shields.io/travis/stevenvachon/limited-request-queue.svg [travis-url]: https://travis-ci.org/stevenvachon/limited-request-queue -[david-image]: https://img.shields.io/david/stevenvachon/limited-request-queue.svg -[david-url]: https://david-dm.org/stevenvachon/limited-request-queue +[coveralls-image]: https://img.shields.io/coveralls/stevenvachon/limited-request-queue.svg +[coveralls-url]: https://coveralls.io/github/stevenvachon/limited-request-queue +[greenkeeper-image]: https://badges.greenkeeper.io/stevenvachon/limited-request-queue.svg +[greenkeeper-url]: https://greenkeeper.io/ diff --git a/bower.json b/bower.json deleted file mode 100644 index 73612e3..0000000 --- a/bower.json +++ /dev/null @@ -1,28 +0,0 @@ -{ - "name": "limited-request-queue", - "homepage": "https://github.com/stevenvachon/limited-request-queue", - "license": "MIT", - "authors": [ - "Steven Vachon " - ], - "description": "Interactively manage concurrency for outbound requests.", - "main": "browser/requestqueue.js", - "keywords": [ - "background", - "concurrency", - "ddos", - "http", - "limiting", - "queue", - "rate", - "request", - "throttle" - ], - "ignore": [ - "bower_components", - "lib", - "node_modules", - "package.json", - "test" - ] -} diff --git a/browser/requestqueue.js b/browser/requestqueue.js deleted file mode 100644 index 4190a9d..0000000 --- a/browser/requestqueue.js +++ /dev/null @@ -1 +0,0 @@ -!function(t){if("object"==typeof exports&&"undefined"!=typeof module)module.exports=t();else if("function"==typeof define&&define.amd)define([],t);else{var e;e="undefined"!=typeof window?window:"undefined"!=typeof global?global:"undefined"!=typeof self?self:this,e.RequestQueue=t()}}(function(){return function t(e,o,n){function i(s,u){if(!o[s]){if(!e[s]){var c="function"==typeof require&&require;if(!u&&c)return c(s,!0);if(r)return r(s,!0);var f=new Error("Cannot find module '"+s+"'");throw f.code="MODULE_NOT_FOUND",f}var a=o[s]={exports:{}};e[s][0].call(a.exports,function(t){var o=e[s][1][t];return i(o?o:t)},a,a.exports,t,e,o,n)}return o[s].exports}for(var r="function"==typeof require&&require,s=0;s0?setTimeout(function(){t.apply(null,e)},o):t.apply(null,e))}function r(t,e){var o=e.priorityQueue.indexOf(t.id);return!(o<0)&&(e.priorityQueue.splice(o,1),!0)}function s(t,e){var o=p(t.url,e.options),n=t.id;return o===!1?new TypeError("Invalid URL"):(null==n&&(n=e.counter++),void 0!==e.items[n]?new Error("Non-unique ID"):(e.items[n]={active:!1,hostKey:o,id:n,input:t},e.priorityQueue.push(n),n))}function u(t,e){return function(){e.activeSockets--,c(t,e),f(e)}}function c(t,e){e.activeHosts[t.hostKey]--,e.activeHosts[t.hostKey]<=0&&delete e.activeHosts[t.hostKey],delete e.items[t.id],e.priorityQueue.length<=0&&e.activeSockets<=0&&(e.counter=0,i(e.handlers.end,[]))}function f(t){var e,o,n=t.options.maxSockets-t.activeSockets,s=0;if(t.paused!==!0&&!(n<=0))for(;s0&&(void 0===t.activeHosts[o.hostKey]?(t.activeHosts[o.hostKey]=1,e=!0):t.activeHosts[o.hostKey] { - var key,port,protocol,urlDomain; - - if (url == null) - { - return false; - } - else if (isString(url) === true) - { - try - { - url = new URL(url); - } - catch (error) - { - return false; - } - } - - protocol = url.protocol; - - // TODO :: remove support for node-js url.parse objects - if (isEmptyString(protocol)===true || isEmptyString(url.hostname)===true) - { - return false; - } - - // Remove ":" suffix - if (protocol.indexOf(":") === protocol.length-1) - { - protocol = protocol.substr(0, protocol.length-1); - } - - port = url.port; - - // Get default port - // TODO :: remove support for node-js url.parse objects - if (isEmptyStringOrNumber(port)===true && options.defaultPorts[protocol]!==undefined) - { - port = options.defaultPorts[protocol]; - } - - key = ""; - - if (options.ignoreSchemes === false) + const ignorePorts = defined(optionOverrides.ignorePorts, options.ignorePorts); + const ignoreProtocols = defined(optionOverrides.ignoreProtocols, options.ignoreProtocols); + const ignoreSubdomains = defined(optionOverrides.ignoreSubdomains, options.ignoreSubdomains); + + let key = ""; + + if (ignoreProtocols === false) { - key += protocol + "://"; + key += `${url.protocol}//`; } - - if (options.ignoreSubdomains === false) + + if (ignoreSubdomains === false) { key += url.hostname; } else { - urlDomain = parseDomain(url.hostname); - - // If unknown top-level-domain (.com, etc) - // Or, if running in a browser - if (urlDomain === null) + const hostname = parseDomain(url.hostname); + + // If unknown top-level domain or running in a browser + if (hostname === null) { key += url.hostname; } else { - key += urlDomain.domain + "." + urlDomain.tld; + key += `${hostname.domain}.${hostname.tld}`; } } - - if (options.ignorePorts===false && port!=null) + + if (ignorePorts===false && url.port!=="") { - key += ":" + port; + key += `:${url.port}`; } - - key += "/"; - - return key; -} - - -function isEmptyString(value) -{ - return value==="" || value==null || isString(value)===false; -} - - - -function isEmptyStringOrNumber(value) -{ - return value==="" || value==null || isNaN(value)===true; -} + return key; +}; diff --git a/lib/index.js b/lib/index.js index adaf01c..81d9e4c 100644 --- a/lib/index.js +++ b/lib/index.js @@ -1,172 +1,158 @@ "use strict"; -var defaultOptions = require("./defaultOptions"); -var getHostKey = require("./getHostKey"); +const defined = require("defined"); +const {EventEmitter} = require("events"); +const getHostKey = require("./getHostKey"); +const isURL = require("isurl"); -var isString = require("is-string"); - -function RequestQueue(options, handlers) +const defaultOptions = { - this.activeHosts = {}; // Socket counts stored by host - this.items = {}; // Items stored by ID - this.priorityQueue = []; // List of IDs - - this.activeSockets = 0; - this.counter = 0; - this.handlers = handlers || {}; - this.options = Object.assign({}, defaultOptions, options); - this.paused = false; -} + ignorePorts: true, + ignoreProtocols: true, + ignoreSubdomains: true, + maxSockets: Infinity, + maxSocketsPerHost: 2, + rateLimit: 0 +}; -RequestQueue.prototype.dequeue = function(id) +class RequestQueue extends EventEmitter { - var item = this.items[id]; - var error; - - if (item===undefined || item.active===true) + constructor(options) { - return new Error("ID not found"); + super(); + + this.activeHosts = {}; // Socket counts stored by host + this.items = {}; // Items stored by ID + this.priorityQueue = []; // List of IDs + + this.activeSockets = 0; + this.counter = 0; + this.options = Object.assign({}, defaultOptions, options); + this.paused = false; } - - dequeue(item, this); - remove(item, this); - - return true; -}; -RequestQueue.prototype.enqueue = function(input) -{ - var idOrError; - - // enqueue("url") - if (input==null || isString(input)===true) + dequeue(id) { - input = { url:input }; + const item = this.items[id]; + + if (item===undefined || item.active) + { + return false; + } + + dequeue(this, item); + remove(this, item); + + return true; } - - idOrError = enqueue(input, this); - - if (idOrError instanceof Error === false) + + + + enqueue(url, data, options) { - startNext(this); - } - - return idOrError; -}; + if (!isURL.lenient(url)) + { + throw new TypeError("Invalid URL"); + } + if (options == null) + { + options = this.options; + } + const hostKey = getHostKey(url, this.options, options); + const id = this.counter++; -RequestQueue.prototype.length = function() -{ - return this.priorityQueue.length + this.activeSockets; -}; + this.items[id] = { active:false, data, hostKey, id, options, url }; + this.priorityQueue.push(id); + startNext(this); + return id; + } -RequestQueue.prototype.numActive = function() -{ - return this.activeSockets; -}; + get isPaused() + { + return this.paused; + } -RequestQueue.prototype.numQueued = function() -{ - return this.priorityQueue.length; -}; + get length() + { + return this.priorityQueue.length + this.activeSockets; + } -RequestQueue.prototype.pause = function() -{ - this.paused = true; -}; + get numActive() + { + return this.activeSockets; + } -RequestQueue.prototype.resume = function() -{ - this.paused = false; - - startNext(this); -}; + get numQueued() + { + return this.priorityQueue.length; + } -//::: PRIVATE FUNCTIONS + pause() + { + this.paused = true; + return this; + } -/* - Call a class' event handler if it exists. -*/ -function callHandler(handler, args, timeout) -{ - if (typeof handler === "function") + + + resume() { - if (timeout > 0) - { - setTimeout( function() - { - handler.apply(null, args); - - }, timeout); - } - else - { - handler.apply(null, args); - } + this.paused = false; + + startNext(this); + + return this; } } +//::: PRIVATE FUNCTIONS + + + /* Remove item (id) from queue, but nowhere else. */ -function dequeue(item, instance) +const dequeue = (instance, item) => { - var queueIndex = instance.priorityQueue.indexOf(item.id); - - if (queueIndex < 0) return false; - + const queueIndex = instance.priorityQueue.indexOf(item.id); + instance.priorityQueue.splice(queueIndex, 1); - - return true; -} +}; -/* - Add item to queue and item list. -*/ -function enqueue(input, instance) +const emitEvent = (instance, event, args=[], timeout=0) => { - var hostKey = getHostKey(input.url, instance.options); - var id = input.id; - - if (hostKey === false) + if (timeout > 0) { - return new TypeError("Invalid URL"); + setTimeout(() => instance.emit(event, ...args), timeout); } - - if (id == null) id = instance.counter++; - - if (instance.items[id] !== undefined) + else { - return new Error("Non-unique ID"); + instance.emit(event, ...args); } - - instance.items[id] = { active:false, hostKey:hostKey, id:id, input:input }; - instance.priorityQueue.push(id); - - return id; -} +}; @@ -174,88 +160,90 @@ function enqueue(input, instance) Generate a `done()` function for use in resuming the queue when an item's process has been completed. */ -function getDoneCallback(item, instance) +const getDoneCallback = (instance, item) => { return function() { instance.activeSockets--; - - remove(item, instance); - + + remove(instance, item); startNext(instance); }; -} +}; /* Remove item from item list and activeHosts. */ -function remove(item, instance) +const remove = (instance, item) => { - instance.activeHosts[item.hostKey]--; - - if (instance.activeHosts[item.hostKey] <= 0) + if (--instance.activeHosts[item.hostKey] <= 0) { delete instance.activeHosts[item.hostKey]; } - + delete instance.items[item.id]; - + if (instance.priorityQueue.length<=0 && instance.activeSockets<=0) { instance.counter = 0; // reset - - callHandler(instance.handlers.end, []); + + emitEvent(instance, "end"); } -} +}; /* Possibly start next request(s). */ -function startNext(instance) +const startNext = instance => { - var availableSockets = instance.options.maxSockets - instance.activeSockets; - var i = 0; - var canStart,currItem,numItems; - if (instance.paused === true) return; + + let availableSockets = instance.options.maxSockets - instance.activeSockets; + if (availableSockets <= 0) return; - + + let i = 0; + while (i < instance.priorityQueue.length) { - canStart = false; - currItem = instance.items[ instance.priorityQueue[i] ]; - + let canStart = false; + const item = instance.items[ instance.priorityQueue[i] ]; + + const maxSocketsPerHost = defined(item.options.maxSocketsPerHost, instance.options.maxSocketsPerHost); + // Not important, but feature complete - if (instance.options.maxSocketsPerHost > 0) + if (maxSocketsPerHost > 0) { - if (instance.activeHosts[currItem.hostKey] === undefined) + if (instance.activeHosts[item.hostKey] === undefined) { // Create key with first count - instance.activeHosts[currItem.hostKey] = 1; + instance.activeHosts[item.hostKey] = 1; canStart = true; } - else if (instance.activeHosts[currItem.hostKey] < instance.options.maxSocketsPerHost) + else if (instance.activeHosts[item.hostKey] < maxSocketsPerHost) { - instance.activeHosts[currItem.hostKey]++; + instance.activeHosts[item.hostKey]++; canStart = true; } } - - if (canStart === true) + + if (canStart) { instance.activeSockets++; availableSockets--; - - currItem.active = true; - - dequeue(currItem, instance); - - callHandler(instance.handlers.item, [currItem.input, getDoneCallback(currItem,instance)], instance.options.rateLimit); - + + item.active = true; + + dequeue(instance, item); + + const rateLimit = defined(item.options.rateLimit, instance.options.rateLimit); + + emitEvent(instance, "item", [item.url, item.data, getDoneCallback(instance,item)], rateLimit); + if (availableSockets <= 0) break; } else @@ -264,7 +252,7 @@ function startNext(instance) i++; } } -} +}; diff --git a/lib/parseDomain.js b/lib/parseDomain.js new file mode 100644 index 0000000..062694d --- /dev/null +++ b/lib/parseDomain.js @@ -0,0 +1,7 @@ +"use strict"; + +// Browser shim +module.exports = function() +{ + return null; +}; diff --git a/package.json b/package.json index c92f599..8992831 100644 --- a/package.json +++ b/package.json @@ -1,43 +1,47 @@ { "name": "limited-request-queue", "description": "Interactively manage concurrency for outbound requests.", - "version": "3.0.4", + "version": "4.0.0-alpha", "license": "MIT", "author": "Steven Vachon (https://www.svachon.com/)", "main": "lib", + "browser": { + "parse-domain": "./lib/parseDomain" + }, "repository": "stevenvachon/limited-request-queue", "dependencies": { - "broquire": "^1.0.0", - "is-string": "^1.0.4", - "parse-domain": "~0.2.1", - "whatwg-url": "^3.0.0" + "defined": "^1.0.0", + "isurl": "^2.0.0", + "parse-domain": "^2.1.7" }, "devDependencies": { - "browserify": "^13.1.0", - "chai": "^3.5.0", - "mocha": "^3.0.2", - "mocha-phantomjs": "4.1.0", - "object.assign": "^4.0.4", - "phantomjs-prebuilt": "^2.1.12", - "uglify-js": "^2.7.3" + "@babel/core": "^7.1.6", + "@babel/preset-env": "^7.1.6", + "babelify": "^10.0.0", + "browserify": "^16.2.3", + "chai": "^4.2.0", + "coveralls": "^3.0.2", + "gzip-size-cli": "^3.0.0", + "mocha": "^5.2.0", + "nyc": "^13.1.0", + "terser": "^3.10.12", + "universal-url": "^2.0.0" }, "engines": { - "node": ">= 4" + "node": ">= 6" }, "scripts": { - "browserify": "browserify lib/ --standalone RequestQueue | uglifyjs --compress --mangle -o browser/requestqueue.js", - "test": "npm run test_server && npm run test_browser", - "test_browser": "npm run browserify && mocha-phantomjs test/browser.html", - "test_server": "mocha test/server/ --reporter spec --check-leaks --bail" + "ci": "npm run test && nyc report --reporter=text-lcov | coveralls", + "posttest": "nyc report --reporter=html && browserify lib/ --global-transform [ babelify --presets [ @babel/env ] ] --standalone RequestQueue | terser --compress --mangle | gzip-size", + "test": "nyc --reporter=text-summary mocha test/ --check-leaks --bail" }, "files": [ - "lib", - "license" + "lib" ], "keywords": [ "background", "concurrency", - "ddos", + "DoS", "http", "limiting", "queue", diff --git a/test/0.internal.js b/test/0.internal.js new file mode 100644 index 0000000..66a24f4 --- /dev/null +++ b/test/0.internal.js @@ -0,0 +1,70 @@ +"use strict"; +const {describe, it} = require("mocha"); +const {expect} = require("chai"); +const getHostKey = require("../lib/getHostKey"); +const helpers = require("./helpers"); +const {URL} = require("universal-url"); + + + +describe("Internal API", function() +{ + describe("getHostKey()", function() + { + it("supports URL input", function() + { + const opts = helpers.options(); + + expect( getHostKey( new URL("https://www.google.com/"),opts) ).to.equal("https://www.google.com"); + expect( getHostKey( new URL("https://www.google.com:8080/"),opts) ).to.equal("https://www.google.com:8080"); + expect( getHostKey( new URL("https://127.0.0.1:8080/"),opts) ).to.equal("https://127.0.0.1:8080"); + }); + + + + describe("Options", function() + { + it("ignorePorts = true", function() + { + const opts = helpers.options({ ignorePorts:true }); + + expect( getHostKey( new URL("https://www.google.com/"),opts) ).to.equal("https://www.google.com"); + expect( getHostKey( new URL("https://www.google.com:8080/"),opts) ).to.equal("https://www.google.com"); + expect( getHostKey( new URL("https://127.0.0.1:8080/"),opts) ).to.equal("https://127.0.0.1"); + }); + + + + it("ignoreProtocols = true", function() + { + const opts = helpers.options({ ignoreProtocols:true }); + + expect( getHostKey( new URL("https://www.google.com/"),opts) ).to.equal("www.google.com"); + expect( getHostKey( new URL("https://www.google.com:8080/"),opts) ).to.equal("www.google.com:8080"); + expect( getHostKey( new URL("https://127.0.0.1:8080/"),opts) ).to.equal("127.0.0.1:8080"); + }); + + + + it("ignoreSubdomains = true", function() + { + const opts = helpers.options({ ignoreSubdomains:true }); + + expect( getHostKey( new URL("https://www.google.com/"),opts) ).to.equal("https://google.com"); + expect( getHostKey( new URL("https://www.google.com:8080/"),opts) ).to.equal("https://google.com:8080"); + expect( getHostKey( new URL("https://127.0.0.1:8080/"),opts) ).to.equal("https://127.0.0.1:8080"); + }); + + + + it("all options true", function() + { + const opts = helpers.options({ ignorePorts:true, ignoreProtocols:true, ignoreSubdomains:true }); + + expect( getHostKey( new URL("https://www.google.com/"),opts) ).to.equal("google.com"); + expect( getHostKey( new URL("https://www.google.com:8080/"),opts) ).to.equal("google.com"); + expect( getHostKey( new URL("https://127.0.0.1:8080/"),opts) ).to.equal("127.0.0.1"); + }); + }); + }); +}); diff --git a/test/1.public.js b/test/1.public.js new file mode 100644 index 0000000..b7f4747 --- /dev/null +++ b/test/1.public.js @@ -0,0 +1,1505 @@ +"use strict"; +const {before, describe, it} = require("mocha"); +const {expect} = require("chai"); +const helpers = require("./helpers"); +const RequestQueue = require("../lib"); +const {URL} = require("universal-url"); + + + +describe("Public API", function() +{ + describe("enqueue(), length", function() + { + it("enqueues valid URLs", function() + { + // Pause to prevent first queued item from immediately starting (and thus being auto-dequeued) + const queue = new RequestQueue(helpers.options()).pause(); + const url1 = new URL("http://domain1/"); + const url2 = new URL("http://domain2/"); + + expect( queue.enqueue(url1) ).to.be.a("number"); + expect( queue.enqueue(url2) ).to.be.a("number"); + expect( queue.length ).to.equal(2); + }); + + + + it("rejects non-URLs", function() + { + // Pause to prevent first queued item from immediately starting (and thus being auto-dequeued) + const queue = new RequestQueue(helpers.options()).pause(); + + ["string", 1, true, {}, [], null, undefined].forEach( function(url) + { + expect(() => queue.enqueue(url)).to.throw(TypeError); + expect(queue.length).to.equal(0); + }); + }); + }); + + + + describe("dequeue(), length", function() + { + it("dequeues valid IDs", function() + { + // Pause to prevent first queued item from immediately starting (and thus being auto-dequeued) + const queue = new RequestQueue(helpers.options()).pause(); + + const url = new URL("http://domain/"); + const id = queue.enqueue(url); + + expect( queue.dequeue(id) ).to.be.true; + expect( queue.length ).to.equal(0); + }); + + + + it("rejects invalid IDs", function() + { + // Pause to prevent first queued item from immediately starting (and thus being auto-dequeued) + const queue = new RequestQueue(helpers.options()).pause(); + + const url = new URL("http://domain/"); + const id = queue.enqueue(url); + + expect( queue.dequeue(123456) ).to.be.false; + expect( queue.length ).to.equal(1); + }); + }); + + + + describe("Events", function() + { + describe("item", function() + { + it("works", function(done) + { + let count = 0; + + const queue = new RequestQueue(helpers.options()) + .on("item", function(url, data, itemDone) + { + if (++count >= helpers.urls.length) + { + done(); + } + }); + + helpers.urls.forEach(url => queue.enqueue(new URL(url))); + }); + + + + it("supports custom data", function(done) + { + let count = 0; + + const queue = new RequestQueue(helpers.options()) + .on("item", function(url, data, itemDone) + { + switch (++count) + { + case 1: + { + expect(url.href).to.equal(helpers.urls[0]); + expect(data).to.equal(1); + break; + } + case 2: + { + expect(url.href).to.equal(helpers.urls[1]); + expect(data).to.equal(2); + break; + } + case 3: + { + expect(url.href).to.equal(helpers.urls[2]); + expect(data).to.equal(3); + done(); + break; + } + } + }); + + queue.enqueue(new URL(helpers.urls[0]), 1); + queue.enqueue(new URL(helpers.urls[1]), 2); + queue.enqueue(new URL(helpers.urls[2]), 3); + }); + + + + it("is not called with non-URLs", function(done) + { + const queue = new RequestQueue(helpers.options()) + .on("item", () => done( new Error("this should not have been called") )) + + expect(() => queue.enqueue("url")).to.throw(); + + setTimeout(done, helpers.delay*2); + }); + }); + + + + describe("end", function() + { + it("works", function(done) + { + const queue = new RequestQueue(helpers.options()) + .on("item", function(url, data, itemDone) + { + setTimeout(itemDone, helpers.delay); + }) + .on("end", () => done()); + + helpers.urls.forEach(url => queue.enqueue(new URL(url))); + }); + + + + it("is called when last item is dequeued", function(done) + { + // Pause to prevent first queued item from immediately starting (and thus being auto-dequeued) + const queue = new RequestQueue(helpers.options()).pause() + .on("end", function() + { + // Wait for `dequeued` to receive its value + // since everything here is performed synchronously + setImmediate( function() + { + expect(dequeued).to.be.true; + done(); + }); + }); + + const url = new URL("http://domain/"); + const id = queue.enqueue(new URL(url)); + const dequeued = queue.dequeue(id); + }); + + + + it("is not called simply by calling resume()", function(done) + { + const queue = new RequestQueue(helpers.options()) + .on("end", () => done( new Error("this should not have been called") )); + + queue.resume(); + + setTimeout(done, helpers.delay*2); + }); + + + + it("is not called on erroneous dequeue", function(done) + { + const queue = new RequestQueue(helpers.options()) + .on("end", () => done( new Error("this should not have been called") )); + + queue.dequeue(123); + + setTimeout(done, helpers.delay*2); + }); + }); + }); + + + + describe("pause(), resume(), isPaused()", function() + { + it("works", function() + { + const opts = helpers.options(); + let count = 0; + let resumed = false; + + return helpers.testUrls(helpers.urls, opts, null, function eachItem(url, data, queue) + { + if (++count === 1) + { + queue.pause(); + + expect(queue.isPaused).to.be.true; + + // Wait longer than queue should take if not paused+resumed + setTimeout( function() + { + resumed = true; + queue.resume(); + + expect(queue.isPaused).to.be.false; + + }, helpers.expectedSyncMinDuration()); + } + }) + .then( function(result) + { + expect(resumed).to.be.true; + expect(result.urls).to.deep.equal(helpers.urls); + }); + }); + }); + + + + describe("numActive", function() + { + it("works", function(done) + { + const queue = new RequestQueue(helpers.options()) + .on("item", function(url, data, itemDone) + { + setTimeout(itemDone, helpers.delay); + }) + .on("end", function() + { + expect( queue.numActive ).to.equal(0); + done(); + }); + + helpers.urls.forEach(url => queue.enqueue(new URL(url))); + + expect( queue.numActive ).to.equal(helpers.urls.length); + }); + + + + it("is not affected by dequeue()", function(done) + { + const queue = new RequestQueue(helpers.options()) + .on("item", function(url, data, itemDone) + { + // Wait for `id` to be assigned its value + setImmediate( function() + { + expect( queue.dequeue(id) ).to.be.false; + expect( queue.numActive ).to.equal(1); + }); + + setTimeout(itemDone, helpers.delay); + }) + .on("end", function() + { + expect( queue.numActive ).to.equal(0); + done(); + }); + + const url = new URL("http://domain/"); + const id = queue.enqueue(url); + + expect( queue.numActive ).to.equal(1); + }); + }); + + + + describe("numQueued", function() + { + it("works", function(done) + { + const queue = new RequestQueue(helpers.options()) + .on("item", function(url, data, itemDone) + { + setTimeout(itemDone, helpers.delay); + }) + .on("end", function() + { + expect( queue.numQueued ).to.equal(0); + done(); + }); + + helpers.urls.forEach(url => queue.enqueue(new URL(url))); + + expect( queue.numQueued ).to.equal(0); + }); + }); + + + + describe("Options", function() + { + describe("all disabled", function() + { + it("works", function() + { + const opts = helpers.options(); + + return helpers.testUrls(helpers.urls, opts) + .then(result => expect(result.urls).to.deep.equal(helpers.urls)); + }); + }); + + + + describe("maxSockets", function() + { + before( () => helpers.clearDurations() ); + + + + describe("=0", function() + { + it("can't do anything", function(done) + { + const opts = helpers.options({ maxSockets:0 }); + + helpers.testUrls(helpers.urls, opts) + .then(() => done( new Error("this should not have been called") )); + + setTimeout( () => done(), helpers.expectedSyncMinDuration() ); + }); + }); + + + + [1,2,3,4,Infinity].forEach( function(value) + { + describe(`=${value}`, function() + { + it("works", function() + { + const opts = helpers.options({ maxSockets:value }); + + return helpers.testUrls(helpers.urls, opts) + .then( function(result) + { + expect(result.urls).to.deep.equal(helpers.urls); + + helpers.addDuration(result.duration); + + if (value > 1) + { + expect(result.duration).to.be.below( helpers.previousDuration() ); + } + }) + }); + + + + it("supports ignorePorts=true", function() + { + const opts = helpers.options({ ignorePorts:true, maxSockets:value }); + + return helpers.testUrls(helpers.urls, opts) + .then(result => expect(result.urls).to.deep.equal(helpers.urls)); + }); + + + + it("supports ignoreProtocols=true", function() + { + const opts = helpers.options({ ignoreProtocols:true, maxSockets:value }); + + return helpers.testUrls(helpers.urls, opts) + .then(result => expect(result.urls).to.deep.equal(helpers.urls)); + }); + + + + it("supports ignoreSubdomains=true", function() + { + const opts = helpers.options({ ignoreSubdomains:true, maxSockets:value }); + + return helpers.testUrls(helpers.urls, opts) + .then(result => expect(result.urls).to.deep.equal(helpers.urls)); + }); + + + + it("supports all boolean options true", function() + { + const opts = helpers.options({ ignorePorts:true, ignoreProtocols:true, ignoreSubdomains:true, maxSockets:value }); + + return helpers.testUrls(helpers.urls, opts) + .then(result => expect(result.urls).to.deep.equal(helpers.urls)); + }); + }); + }); + }); + + + + // NOTE :: URLs are visually grouped according to how they are prioritized for concurrency within the queue + describe("maxSocketsPerHost", function() + { + before( () => helpers.clearDurations() ); + + + + describe("=0", function() + { + it("can't do anything", function(done) + { + const opts = helpers.options({ maxSocketsPerHost:0 }); + + helpers.testUrls(helpers.urls, opts) + .then(() => done( new Error("this should not have been called") )); + + setTimeout( () => done(), helpers.expectedSyncMinDuration() ); + }); + }); + + + + describe("=1", function() + { + it("works", function() + { + const opts = helpers.options({ maxSocketsPerHost:1 }); + + return helpers.testUrls(helpers.urls, opts) + .then( function(result) + { + expect(result.urls).to.deep.equal( + [ + "https://www.google.com/", + "http://www.google.com/", + "https://google.com/", + "http://google.com/", + "https://google.com:8080/", + "http://google.com:8080/", + "https://127.0.0.1/", + "http://127.0.0.1/", + + "https://www.google.com/", + "http://www.google.com/", + "https://google.com/", + "http://google.com/", + "https://google.com:8080/", + "http://google.com:8080/", + "https://127.0.0.1/", + "http://127.0.0.1/", + + "https://google.com/something.html", + "http://google.com/something.html", + + "https://google.com/something.html", + "http://google.com/something.html" + ]); + + // For next test + // NOTE :: this is not atomic and technically would fail if running exclusive tests + helpers.addDuration(result.duration); + }); + }); + + + + it("supports ignorePorts=true", function() + { + const opts = helpers.options({ ignorePorts:true, maxSocketsPerHost:1 }); + + return helpers.testUrls(helpers.urls, opts) + .then( function(result) + { + expect(result.urls).to.deep.equal( + [ + "https://www.google.com/", + "http://www.google.com/", + "https://google.com/", + "http://google.com/", + "https://127.0.0.1/", + "http://127.0.0.1/", + + "https://www.google.com/", + "http://www.google.com/", + "https://google.com/", + "http://google.com/", + "https://127.0.0.1/", + "http://127.0.0.1/", + + "https://google.com:8080/", + "http://google.com:8080/", + + "https://google.com:8080/", + "http://google.com:8080/", + + "https://google.com/something.html", + "http://google.com/something.html", + + "https://google.com/something.html", + "http://google.com/something.html" + ]); + }); + }); + + + + it("supports ignoreProtocols=true", function() + { + const opts = helpers.options({ ignoreProtocols:true, maxSocketsPerHost:1 }); + + return helpers.testUrls(helpers.urls, opts) + .then( function(result) + { + expect(result.urls).to.deep.equal( + [ + "https://www.google.com/", + "https://google.com/", + "https://google.com:8080/", + "https://127.0.0.1/", + + "https://www.google.com/", + "https://google.com/", + "https://google.com:8080/", + "https://127.0.0.1/", + + "http://www.google.com/", + "http://google.com/", + "http://google.com:8080/", + "http://127.0.0.1/", + + "http://www.google.com/", + "http://google.com/", + "http://google.com:8080/", + "http://127.0.0.1/", + + "https://google.com/something.html", + + "https://google.com/something.html", + + "http://google.com/something.html", + + "http://google.com/something.html" + ]); + }); + }); + + + + it("supports ignoreSubdomains=true", function() + { + const opts = helpers.options({ ignoreSubdomains:true, maxSocketsPerHost:1 }); + + return helpers.testUrls(helpers.urls, opts) + .then( function(result) + { + expect(result.urls).to.deep.equal( + [ + "https://www.google.com/", + "http://www.google.com/", + "https://google.com:8080/", + "http://google.com:8080/", + "https://127.0.0.1/", + "http://127.0.0.1/", + + "https://www.google.com/", + "http://www.google.com/", + "https://google.com:8080/", + "http://google.com:8080/", + "https://127.0.0.1/", + "http://127.0.0.1/", + + "https://google.com/", + "http://google.com/", + + "https://google.com/", + "http://google.com/", + + "https://google.com/something.html", + "http://google.com/something.html", + + "https://google.com/something.html", + "http://google.com/something.html" + ]); + }); + }); + + + + it("supports all boolean options true", function() + { + const opts = helpers.options({ ignorePorts:true, ignoreProtocols:true, ignoreSubdomains:true, maxSocketsPerHost:1 }); + + return helpers.testUrls(helpers.urls, opts) + .then( function(result) + { + expect(result.urls).to.deep.equal( + [ + "https://www.google.com/", + "https://127.0.0.1/", + + "https://www.google.com/", + "https://127.0.0.1/", + + "http://www.google.com/", + "http://127.0.0.1/", + + "http://www.google.com/", + "http://127.0.0.1/", + + "https://google.com/", + + "https://google.com/", + + "http://google.com/", + + "http://google.com/", + + "https://google.com:8080/", + + "https://google.com:8080/", + + "http://google.com:8080/", + + "http://google.com:8080/", + + "https://google.com/something.html", + + "https://google.com/something.html", + + "http://google.com/something.html", + + "http://google.com/something.html" + ]); + }); + }); + }); + + + + describe("=2", function() + { + it("works", function() + { + const opts = helpers.options({ maxSocketsPerHost:2 }); + + return helpers.testUrls(helpers.urls, opts) + .then( function(result) + { + expect(result.urls).to.deep.equal( + [ + "https://www.google.com/", + "https://www.google.com/", + "http://www.google.com/", + "http://www.google.com/", + "https://google.com/", + "https://google.com/", + "http://google.com/", + "http://google.com/", + "https://google.com:8080/", + "https://google.com:8080/", + "http://google.com:8080/", + "http://google.com:8080/", + + "https://127.0.0.1/", + "https://127.0.0.1/", + "http://127.0.0.1/", + "http://127.0.0.1/", + + "https://google.com/something.html", + "https://google.com/something.html", + "http://google.com/something.html", + "http://google.com/something.html" + ]); + + helpers.addDuration(result.duration); + + expect(result.duration).to.be.below( helpers.previousDuration() ); + }); + }); + + + + it("supports ignorePorts=true", function() + { + const opts = helpers.options({ ignorePorts:true, maxSocketsPerHost:2 }); + + return helpers.testUrls(helpers.urls, opts) + .then( function(result) + { + expect(result.urls).to.deep.equal( + [ + "https://www.google.com/", + "https://www.google.com/", + "http://www.google.com/", + "http://www.google.com/", + "https://google.com/", + "https://google.com/", + "http://google.com/", + "http://google.com/", + "https://127.0.0.1/", + "https://127.0.0.1/", + "http://127.0.0.1/", + "http://127.0.0.1/", + + "https://google.com:8080/", + "https://google.com:8080/", + "http://google.com:8080/", + "http://google.com:8080/", + + "https://google.com/something.html", + "https://google.com/something.html", + "http://google.com/something.html", + "http://google.com/something.html" + ]); + }); + }); + + + + it("supports ignoreProtocols=true", function() + { + const opts = helpers.options({ ignoreProtocols:true, maxSocketsPerHost:2 }); + + return helpers.testUrls(helpers.urls, opts) + .then( function(result) + { + expect(result.urls).to.deep.equal( + [ + "https://www.google.com/", + "https://www.google.com/", + "https://google.com/", + "https://google.com/", + "https://google.com:8080/", + "https://google.com:8080/", + "https://127.0.0.1/", + "https://127.0.0.1/", + + "http://www.google.com/", + "http://www.google.com/", + "http://google.com/", + "http://google.com/", + "http://google.com:8080/", + "http://google.com:8080/", + "http://127.0.0.1/", + "http://127.0.0.1/", + + "https://google.com/something.html", + "https://google.com/something.html", + + "http://google.com/something.html", + "http://google.com/something.html" + ]); + }); + }); + + + + it("supports ignoreSubdomains=true", function() + { + const opts = helpers.options({ ignoreSubdomains:true, maxSocketsPerHost:2 }); + + return helpers.testUrls(helpers.urls, opts) + .then( function(result) + { + expect(result.urls).to.deep.equal( + [ + "https://www.google.com/", + "https://www.google.com/", + "http://www.google.com/", + "http://www.google.com/", + "https://google.com:8080/", + "https://google.com:8080/", + "http://google.com:8080/", + "http://google.com:8080/", + "https://127.0.0.1/", + "https://127.0.0.1/", + "http://127.0.0.1/", + "http://127.0.0.1/", + + "https://google.com/", + "https://google.com/", + "http://google.com/", + "http://google.com/", + + "https://google.com/something.html", + "https://google.com/something.html", + "http://google.com/something.html", + "http://google.com/something.html" + ]); + }); + }); + + + + it("supports all boolean options true", function() + { + const opts = helpers.options({ ignorePorts:true, ignoreProtocols:true, ignoreSubdomains:true, maxSocketsPerHost:2 }); + + return helpers.testUrls(helpers.urls, opts) + .then( function(result) + { + expect(result.urls).to.deep.equal( + [ + "https://www.google.com/", + "https://www.google.com/", + "https://127.0.0.1/", + "https://127.0.0.1/", + + "http://www.google.com/", + "http://www.google.com/", + "http://127.0.0.1/", + "http://127.0.0.1/", + + "https://google.com/", + "https://google.com/", + + "http://google.com/", + "http://google.com/", + + "https://google.com:8080/", + "https://google.com:8080/", + + "http://google.com:8080/", + "http://google.com:8080/", + + "https://google.com/something.html", + "https://google.com/something.html", + + "http://google.com/something.html", + "http://google.com/something.html" + ]); + }); + }); + }); + + + + describe("=3", function() + { + it("works", function() + { + const opts = helpers.options({ maxSocketsPerHost:3 }); + + return helpers.testUrls(helpers.urls, opts) + .then( function(result) + { + expect(result.urls).to.deep.equal( + [ + "https://www.google.com/", + "https://www.google.com/", + "http://www.google.com/", + "http://www.google.com/", + "https://google.com/", + "https://google.com/", + "http://google.com/", + "http://google.com/", + "https://google.com:8080/", + "https://google.com:8080/", + "http://google.com:8080/", + "http://google.com:8080/", + "https://google.com/something.html", + "http://google.com/something.html", + "https://127.0.0.1/", + "https://127.0.0.1/", + "http://127.0.0.1/", + "http://127.0.0.1/", + + "https://google.com/something.html", + "http://google.com/something.html" + ]); + + helpers.addDuration(result.duration); + + expect(result.duration).to.be.at.most( helpers.previousDuration() + 10 ); + }); + }); + + + + it("supports ignorePorts=true", function() + { + const opts = helpers.options({ ignorePorts:true, maxSocketsPerHost:3 }); + + return helpers.testUrls(helpers.urls, opts) + .then( function(result) + { + expect(result.urls).to.deep.equal( + [ + "https://www.google.com/", + "https://www.google.com/", + "http://www.google.com/", + "http://www.google.com/", + "https://google.com/", + "https://google.com/", + "http://google.com/", + "http://google.com/", + "https://google.com:8080/", + "http://google.com:8080/", + "https://127.0.0.1/", + "https://127.0.0.1/", + "http://127.0.0.1/", + "http://127.0.0.1/", + + "https://google.com:8080/", + "https://google.com/something.html", + "http://google.com:8080/", // TODO :: why is this one here and not after its https cousin? + + "http://google.com/something.html", + "https://google.com/something.html", + "http://google.com/something.html" + ]); + }); + }); + + + + it("supports ignoreProtocols=true", function() + { + const opts = helpers.options({ ignoreProtocols:true, maxSocketsPerHost:3 }); + + return helpers.testUrls(helpers.urls, opts) + .then( function(result) + { + expect(result.urls).to.deep.equal( + [ + "https://www.google.com/", + "https://www.google.com/", + "http://www.google.com/", + "https://google.com/", + "https://google.com/", + "http://google.com/", + "https://google.com:8080/", + "https://google.com:8080/", + "http://google.com:8080/", + "https://127.0.0.1/", + "https://127.0.0.1/", + "http://127.0.0.1/", + + "http://www.google.com/", + "http://google.com/", + "https://google.com/something.html", + "https://google.com/something.html", + "http://google.com:8080/", + "http://127.0.0.1/", + + "http://google.com/something.html", + "http://google.com/something.html" + ]); + }); + }); + + + + it("supports ignoreSubdomains=true", function() + { + const opts = helpers.options({ ignoreSubdomains:true, maxSocketsPerHost:3 }); + + return helpers.testUrls(helpers.urls, opts) + .then( function(result) + { + expect(result.urls).to.deep.equal( + [ + "https://www.google.com/", + "https://www.google.com/", + "http://www.google.com/", + "http://www.google.com/", + "https://google.com/", + "http://google.com/", + "https://google.com:8080/", + "https://google.com:8080/", + "http://google.com:8080/", + "http://google.com:8080/", + "https://127.0.0.1/", + "https://127.0.0.1/", + "http://127.0.0.1/", + "http://127.0.0.1/", + + "https://google.com/", + "https://google.com/something.html", + "http://google.com/", // TODO :: why is this one here and not after its https cousin? + "http://google.com/something.html", + "https://google.com/something.html", + "http://google.com/something.html" + ]); + }); + }); + + + + it("supports all boolean options true", function() + { + const opts = helpers.options({ ignorePorts:true, ignoreProtocols:true, ignoreSubdomains:true, maxSocketsPerHost:3 }); + + return helpers.testUrls(helpers.urls, opts) + .then( function(result) + { + expect(result.urls).to.deep.equal( + [ + "https://www.google.com/", + "https://www.google.com/", + "http://www.google.com/", + "https://127.0.0.1/", + "https://127.0.0.1/", + "http://127.0.0.1/", + + "http://www.google.com/", + "https://google.com/", + "https://google.com/", + "http://127.0.0.1/", + + "http://google.com/", + "http://google.com/", + "https://google.com:8080/", + + "https://google.com:8080/", + "http://google.com:8080/", + "http://google.com:8080/", + + "https://google.com/something.html", + "https://google.com/something.html", + "http://google.com/something.html", + + "http://google.com/something.html" + ]); + }); + }); + }); + + + + describe("=4", function() + { + it("works", function() + { + const opts = helpers.options({ maxSocketsPerHost:4 }); + + return helpers.testUrls(helpers.urls, opts) + .then( function(result) + { + expect(result.urls).to.deep.equal(helpers.urls); + + helpers.addDuration(result.duration); + + expect(result.duration).to.be.below( helpers.previousDuration() ); + }); + }); + + + + it("supports ignorePorts=true", function() + { + const opts = helpers.options({ ignorePorts:true, maxSocketsPerHost:4 }); + + return helpers.testUrls(helpers.urls, opts) + .then( function(result) + { + expect(result.urls).to.deep.equal( + [ + "https://www.google.com/", + "https://www.google.com/", + "http://www.google.com/", + "http://www.google.com/", + "https://google.com/", + "https://google.com/", + "http://google.com/", + "http://google.com/", + "https://google.com:8080/", + "https://google.com:8080/", + "http://google.com:8080/", + "http://google.com:8080/", + "https://127.0.0.1/", + "https://127.0.0.1/", + "http://127.0.0.1/", + "http://127.0.0.1/", + + "https://google.com/something.html", + "https://google.com/something.html", + "http://google.com/something.html", + "http://google.com/something.html" + ]); + }); + }); + + + + it("supports ignoreProtocols=true", function() + { + const opts = helpers.options({ ignoreProtocols:true, maxSocketsPerHost:4 }); + + return helpers.testUrls(helpers.urls, opts) + .then( function(result) + { + expect(result.urls).to.deep.equal( + [ + "https://www.google.com/", + "https://www.google.com/", + "http://www.google.com/", + "http://www.google.com/", + "https://google.com/", + "https://google.com/", + "http://google.com/", + "http://google.com/", + "https://google.com:8080/", + "https://google.com:8080/", + "http://google.com:8080/", + "http://google.com:8080/", + "https://127.0.0.1/", + "https://127.0.0.1/", + "http://127.0.0.1/", + "http://127.0.0.1/", + + "https://google.com/something.html", + "https://google.com/something.html", + "http://google.com/something.html", + "http://google.com/something.html" + ]); + }); + }); + + + + it("supports ignoreSubdomains=true", function() + { + const opts = helpers.options({ ignoreSubdomains:true, maxSocketsPerHost:4 }); + + return helpers.testUrls(helpers.urls, opts) + .then( function(result) + { + expect(result.urls).to.deep.equal( + [ + "https://www.google.com/", + "https://www.google.com/", + "http://www.google.com/", + "http://www.google.com/", + "https://google.com/", + "https://google.com/", + "http://google.com/", + "http://google.com/", + "https://google.com:8080/", + "https://google.com:8080/", + "http://google.com:8080/", + "http://google.com:8080/", + "https://127.0.0.1/", + "https://127.0.0.1/", + "http://127.0.0.1/", + "http://127.0.0.1/", + + "https://google.com/something.html", + "https://google.com/something.html", + "http://google.com/something.html", + "http://google.com/something.html" + ]); + }); + }); + + + + it("supports all boolean options true", function() + { + const opts = helpers.options({ ignorePorts:true, ignoreProtocols:true, ignoreSubdomains:true, maxSocketsPerHost:4 }); + + return helpers.testUrls(helpers.urls, opts) + .then( function(result) + { + expect(result.urls).to.deep.equal( + [ + "https://www.google.com/", + "https://www.google.com/", + "http://www.google.com/", + "http://www.google.com/", + "https://127.0.0.1/", + "https://127.0.0.1/", + "http://127.0.0.1/", + "http://127.0.0.1/", + + "https://google.com/", + "https://google.com/", + "http://google.com/", + "http://google.com/", + + "https://google.com:8080/", + "https://google.com:8080/", + "http://google.com:8080/", + "http://google.com:8080/", + + "https://google.com/something.html", + "https://google.com/something.html", + "http://google.com/something.html", + "http://google.com/something.html" + ]); + }); + }); + }); + + + + describe("=Infinity", function() + { + it("works", function() + { + const opts = helpers.options({ maxSocketsPerHost:Infinity }); + + return helpers.testUrls(helpers.urls, opts) + .then( function(result) + { + expect(result.urls).to.deep.equal(helpers.urls); + + helpers.addDuration(result.duration); + + expect(result.duration).to.be.at.most( helpers.previousDuration() + 10 ); + }); + }); + + + + it("supports ignorePorts=true", function() + { + const opts = helpers.options({ ignorePorts:true, maxSocketsPerHost:Infinity }); + + return helpers.testUrls(helpers.urls, opts) + .then(result => expect(result.urls).to.deep.equal(helpers.urls)); + }); + + + + it("supports ignoreProtocols=true", function() + { + const opts = helpers.options({ ignoreProtocols:true, maxSocketsPerHost:Infinity }); + + return helpers.testUrls(helpers.urls, opts) + .then(result => expect(result.urls).to.deep.equal(helpers.urls)); + }); + + + + it("supports ignoreSubdomains=true", function() + { + const opts = helpers.options({ ignoreSubdomains:true, maxSocketsPerHost:Infinity }); + + return helpers.testUrls(helpers.urls, opts) + .then(result => expect(result.urls).to.deep.equal(helpers.urls)); + }); + + + + it("supports all boolean options true", function() + { + const opts = helpers.options({ ignorePorts:true, ignoreProtocols:true, ignoreSubdomains:true, maxSocketsPerHost:Infinity }); + + return helpers.testUrls(helpers.urls, opts) + .then(result => expect(result.urls).to.deep.equal(helpers.urls)); + }); + }); + }); + + + + describe("rateLimit=50", function() + { + it("works", function() + { + const opts = helpers.options({ rateLimit:50 }); + + return helpers.testUrls(helpers.urls, opts) + .then( function(result) + { + expect(result.duration).to.be.at.least(50); + expect(result.urls).to.deep.equal(helpers.urls); + }); + }); + + + + it("supports ignorePorts=true", function() + { + const opts = helpers.options({ ignorePorts:true, rateLimit:50 }); + + return helpers.testUrls(helpers.urls, opts) + .then( function(result) + { + expect(result.duration).to.be.at.least(50); + expect(result.urls).to.deep.equal(helpers.urls); + }); + }); + + + + it("supports ignoreProtocols=true", function() + { + const opts = helpers.options({ ignoreProtocols:true, rateLimit:50 }); + + return helpers.testUrls(helpers.urls, opts) + .then( function(result) + { + expect(result.duration).to.be.at.least(50); + expect(result.urls).to.deep.equal(helpers.urls); + }); + }); + + + + it("supports ignoreSubdomains=true", function() + { + const opts = helpers.options({ ignoreSubdomains:true, rateLimit:50 }); + + return helpers.testUrls(helpers.urls, opts) + .then( function(result) + { + expect(result.duration).to.be.at.least(50); + expect(result.urls).to.deep.equal(helpers.urls); + }); + }); + + + + it("supports all boolean options true", function() + { + const opts = helpers.options({ ignorePorts:true, ignoreProtocols:true, ignoreSubdomains:true, rateLimit:50 }); + + return helpers.testUrls(helpers.urls, opts) + .then( function(result) + { + expect(result.duration).to.be.at.least(50); + expect(result.urls).to.deep.equal(helpers.urls); + }); + }); + }); + + + + describe("all boolean options true, maxSockets=2, maxSocketsPerHost=1, rateLimit=50", function() + { + it("works", function() + { + const opts = helpers.options({ ignorePorts:true, ignoreProtocols:true, ignoreSubdomains:true, maxSockets:2, maxSocketsPerHost:1, rateLimit:50 }); + + return helpers.testUrls(helpers.urls, opts) + .then( function(result) + { + expect(result.duration).to.be.at.least( (helpers.urls.length - 4) * (50 + helpers.delay) ); + expect(result.urls).to.deep.equal( + [ + "https://www.google.com/", + "https://127.0.0.1/", + + "https://www.google.com/", + "https://127.0.0.1/", + + "http://www.google.com/", + "http://127.0.0.1/", + + "http://www.google.com/", + "http://127.0.0.1/", + + "https://google.com/", + + "https://google.com/", + + "http://google.com/", + + "http://google.com/", + + "https://google.com:8080/", + + "https://google.com:8080/", + + "http://google.com:8080/", + + "http://google.com:8080/", + + "https://google.com/something.html", + + "https://google.com/something.html", + + "http://google.com/something.html", + + "http://google.com/something.html" + ]); + }); + }); + }); + + + + describe("all boolean options true, maxSockets=2, maxSocketsPerHost=1, rateLimit=50 -- via enqueue()", function() + { + it("works", function() + { + const opts = helpers.options({ maxSockets:2 }); + const optOverrides = helpers.options({ ignorePorts:true, ignoreProtocols:true, ignoreSubdomains:true, maxSocketsPerHost:1, rateLimit:50 }); + + return helpers.testUrls(helpers.urls, opts, optOverrides) + .then( function(result) + { + expect(result.duration).to.be.at.least( (helpers.urls.length - 4) * (50 + helpers.delay) ); + expect(result.urls).to.deep.equal( + [ + "https://www.google.com/", + "https://127.0.0.1/", + + "https://www.google.com/", + "https://127.0.0.1/", + + "http://www.google.com/", + "http://127.0.0.1/", + + "http://www.google.com/", + "http://127.0.0.1/", + + "https://google.com/", + + "https://google.com/", + + "http://google.com/", + + "http://google.com/", + + "https://google.com:8080/", + + "https://google.com:8080/", + + "http://google.com:8080/", + + "http://google.com:8080/", + + "https://google.com/something.html", + + "https://google.com/something.html", + + "http://google.com/something.html", + + "http://google.com/something.html" + ]); + }); + }); + }); + + + + describe("default options", function() + { + it("works", function() + { + return helpers.testUrls(helpers.urls) + .then( function(result) + { + expect(result.urls).to.deep.equal( + [ + "https://www.google.com/", + "https://www.google.com/", + "https://127.0.0.1/", + "https://127.0.0.1/", + + "http://www.google.com/", + "http://www.google.com/", + "http://127.0.0.1/", + "http://127.0.0.1/", + + "https://google.com/", + "https://google.com/", + + "http://google.com/", + "http://google.com/", + + "https://google.com:8080/", + "https://google.com:8080/", + + "http://google.com:8080/", + "http://google.com:8080/", + + "https://google.com/something.html", + "https://google.com/something.html", + + "http://google.com/something.html", + "http://google.com/something.html" + ]); + }); + }); + }); + }); +}); diff --git a/test/2.helpers.js b/test/2.helpers.js new file mode 100644 index 0000000..4e39e3c --- /dev/null +++ b/test/2.helpers.js @@ -0,0 +1,24 @@ +"use strict"; +const {describe, it} = require("mocha"); +const {expect} = require("chai"); +const helpers = require("./helpers"); + + + +// NOTE :: this makes sure that other tests were tested correctly +describe("Test Helpers", function() +{ + it("helpers.testUrls() rejects non-URLs", function(done) + { + const opts = helpers.options(); + const urls = [ + "https://www.google.com/", + "path/to/resource.html", + "http://www.google.com/" + ]; + + helpers.testUrls(urls, opts) + .then(() => done( new Error("this should not have been called") )) + .catch(error => done()); + }); +}); diff --git a/test/browser.html b/test/browser.html deleted file mode 100644 index ce98535..0000000 --- a/test/browser.html +++ /dev/null @@ -1,90 +0,0 @@ - - - - -limited-request-queue - - - - - - - - - - -
- - - diff --git a/test/helpers/durations.js b/test/helpers/durations.js new file mode 100644 index 0000000..4c5d81f --- /dev/null +++ b/test/helpers/durations.js @@ -0,0 +1,33 @@ +"use strict"; +const durations = []; + + + +const addDuration = duration => durations.push(duration); + + + +const clearDurations = () => durations.length = 0; + + + +const previousDuration = () => +{ + if (durations.length > 1) + { + return durations[durations.length - 2]; + } + else + { + return -1; + } +}; + + + +module.exports = +{ + add: addDuration, + clear: clearDurations, + previous: previousDuration +}; diff --git a/test/server/helpers/index.js b/test/helpers/index.js similarity index 65% rename from test/server/helpers/index.js rename to test/helpers/index.js index 1a68bce..e75a5ea 100644 --- a/test/server/helpers/index.js +++ b/test/helpers/index.js @@ -1,14 +1,14 @@ "use strict"; -var durations = require("./durations"); -var testUrls = require("./testUrls"); +const durations = require("./durations"); +const testUrls = require("./testUrls"); module.exports = { - addDurationGroup: durations.addGroup, + addDuration: durations.add, clearDurations: durations.clear, - compareDurations: durations.compare, + previousDuration: durations.previous, options: require("./options"), diff --git a/test/helpers/options.js b/test/helpers/options.js new file mode 100644 index 0000000..57a6d89 --- /dev/null +++ b/test/helpers/options.js @@ -0,0 +1,20 @@ +"use strict"; + + + +const options = overrides => Object.assign +( + { + ignorePorts: false, + ignoreProtocols: false, + ignoreSubdomains: false, + maxSockets: Infinity, + maxSocketsPerHost: Infinity, + rateLimit: 0 + }, + overrides +); + + + +module.exports = options; diff --git a/test/helpers/testUrls.js b/test/helpers/testUrls.js new file mode 100644 index 0000000..fc0dfb8 --- /dev/null +++ b/test/helpers/testUrls.js @@ -0,0 +1,75 @@ +"use strict"; +const RequestQueue = require("../../lib"); +const {URL} = require("universal-url"); + +const delay = 18; // long enough without trying everyone's patience +const _urls = +[ + "https://www.google.com/", + "https://www.google.com/", + + "http://www.google.com/", + "http://www.google.com/", + + "https://google.com/", + "https://google.com/", + + "http://google.com/", + "http://google.com/", + + "https://google.com:8080/", + "https://google.com:8080/", + + "http://google.com:8080/", + "http://google.com:8080/", + + "https://google.com/something.html", + "https://google.com/something.html", + + "http://google.com/something.html", + "http://google.com/something.html", + + "https://127.0.0.1/", + "https://127.0.0.1/", + + "http://127.0.0.1/", + "http://127.0.0.1/" +]; + + + +const expectedSyncMinDuration = () => _urls.length * delay + 50; + + + +const testUrls = (urls, options, optionOverrides, eachCallback) => new Promise((resolve, reject) => +{ + const results = []; + const startTime = Date.now(); + + const queue = new RequestQueue(options) + .on("item", function(url, data, done) + { + results.push(url.href); + + if (typeof eachCallback === "function") + { + eachCallback(url, data, queue); + } + + // Simulate a remote connection + setTimeout(() => done(), delay); + }) + .on("end", function() + { + const duration = Date.now() - startTime; + + resolve({ duration, urls:results }); + }); + + urls.forEach(url => queue.enqueue(new URL(url), null, optionOverrides)); +}); + + + +module.exports = { delay, expectedSyncMinDuration, testUrls, urls:_urls }; diff --git a/test/server/0.internal.js b/test/server/0.internal.js deleted file mode 100644 index 2327f90..0000000 --- a/test/server/0.internal.js +++ /dev/null @@ -1,103 +0,0 @@ -"use strict"; -var getHostKey = require("../../lib/getHostKey"); -var helpers = require("./helpers"); - -var expect = require("chai").expect; -var parseUrl = require("url").parse; -var URL = require("whatwg-url").URL; - - - -describe("Internal API", function() -{ - describe("getHostKey", function() - { - it("supports String input", function() - { - var options = helpers.options(); - - expect( getHostKey("https://www.google.com:8080/",options) ).to.equal("https://www.google.com:8080/"); - expect( getHostKey("https://127.0.0.1:8080/",options) ).to.equal("https://127.0.0.1:8080/"); - }); - - - - it("supports URL input", function() - { - var options = helpers.options(); - - expect( getHostKey( new URL("https://www.google.com:8080/"),options) ).to.equal("https://www.google.com:8080/"); - expect( getHostKey( new URL("https://127.0.0.1:8080/"),options) ).to.equal("https://127.0.0.1:8080/"); - }); - - - - it("supports Node URL input", function() - { - var options = helpers.options(); - - expect( getHostKey( parseUrl("https://www.google.com:8080/"),options) ).to.equal("https://www.google.com:8080/"); - expect( getHostKey( parseUrl("https://127.0.0.1:8080/"),options) ).to.equal("https://127.0.0.1:8080/"); - }); - - - - it("rejects invalid input", function() - { - var options = helpers.options(); - - expect( getHostKey("/path/",options) ).to.be.false; - expect( getHostKey("resource.html",options) ).to.be.false; - expect( getHostKey("",options) ).to.be.false; - expect( getHostKey({},options) ).to.be.false; - expect( getHostKey([],options) ).to.be.false; - expect( getHostKey(true,options) ).to.be.false; - expect( getHostKey(0,options) ).to.be.false; - expect( getHostKey(null,options) ).to.be.false; - expect( getHostKey(undefined,options) ).to.be.false; - }); - - - - describe("Options", function() - { - it("ignorePorts = true", function() - { - var options = helpers.options({ ignorePorts:true }); - - expect( getHostKey("https://www.google.com:8080/",options) ).to.equal("https://www.google.com/"); - expect( getHostKey("https://127.0.0.1/",options) ).to.equal("https://127.0.0.1/"); - }); - - - - it("ignoreSchemes = true", function() - { - var options = helpers.options({ ignoreSchemes:true }); - - expect( getHostKey("https://www.google.com:8080/",options) ).to.equal("www.google.com:8080/"); - expect( getHostKey("https://127.0.0.1:8080/",options) ).to.equal("127.0.0.1:8080/"); - }); - - - - it("ignoreSubdomains = true", function() - { - var options = helpers.options({ ignoreSubdomains:true }); - - expect( getHostKey("https://www.google.com:8080/",options) ).to.equal("https://google.com:8080/"); - expect( getHostKey("https://127.0.0.1:8080/",options) ).to.equal("https://127.0.0.1:8080/"); - }); - - - - it("all options true", function() - { - var options = helpers.options({ ignorePorts:true, ignoreSchemes:true, ignoreSubdomains:true }); - - expect( getHostKey("https://www.google.com:8080/",options) ).to.equal("google.com/"); - expect( getHostKey("https://127.0.0.1:8080/",options) ).to.equal("127.0.0.1/"); - }); - }); - }); -}); diff --git a/test/server/1.public.js b/test/server/1.public.js deleted file mode 100644 index f247679..0000000 --- a/test/server/1.public.js +++ /dev/null @@ -1,1481 +0,0 @@ -"use strict"; -var helpers = require("./helpers"); -var RequestQueue = require("../../lib"); - -var expect = require("chai").expect; -var parseUrl = require("url").parse; -var URL = require("whatwg-url").URL; - - - -describe("Public API", function() -{ - describe("enqueue() / dequeue() / length()", function() - { - it("enqueues valid URLs", function() - { - var queue = new RequestQueue(helpers.options()); - - // Prevent first queued item from immediately starting (and thus being auto-dequeued) - queue.pause(); - - [ - helpers.urls[0], - { url:helpers.urls[0] }, - { url:parseUrl(helpers.urls[0]) }, - { url:new URL(helpers.urls[0]) } - - ].forEach(url => - { - expect( queue.enqueue(url) ).to.be.a("number"); - }); - - expect( queue.length() ).to.equal(4); - }); - - - - it("dequeues valid IDs", function() - { - var queue = new RequestQueue(helpers.options()); - - // Prevent first queued item from immediately starting (and thus being auto-dequeued) - queue.pause(); - - var id = queue.enqueue(helpers.urls[0]); - - expect( queue.dequeue(id) ).to.be.true; - expect( queue.length() ).to.equal(0); - }); - - - - it("rejects and does not enqueue erroneous URLs", function() - { - var queue = new RequestQueue(helpers.options()); - - // Prevent first queued item from immediately starting (and thus being auto-dequeued) - queue.pause(); - - [ - "/path/", - "resource.html", - "", - 1, - null, - { url:parseUrl("/path/") }, - { url:{} }, - { url:[] }, - { url:"/path/" }, - { url:1 }, - { url:null } - - ].forEach(url => - { - expect( queue.enqueue(url) ).to.be.instanceOf(Error); - expect( queue.length() ).to.equal(0); - }); - }); - - - - it("rejects and neither enqueues nor dequeues invalid IDs", function() - { - var queue = new RequestQueue(helpers.options()); - - // Prevent first queued item from immediately starting (and thus being auto-dequeued) - queue.pause(); - - expect( queue.enqueue({id:123, url:helpers.urls[0]}) ).to.not.be.instanceOf(Error); - expect( queue.enqueue({id:123, url:helpers.urls[0]}) ).to.be.instanceOf(Error); - - expect( queue.dequeue(123456) ).to.be.instanceOf(Error); - - expect( queue.length() ).to.equal(1); - }); - }); - - - - describe("Handlers", function() - { - describe("item", function() - { - it("works", function(done) - { - var count = 0; - var queue = new RequestQueue(helpers.options(), - { - item: function(input, itemDone) - { - if (++count >= helpers.urls.length) - { - done(); - } - } - }); - - helpers.urls.forEach(queue.enqueue, queue); - }); - - - - it("supports custom IDs and custom data", function(done) - { - var count = 0; - var queue = new RequestQueue(helpers.options(), - { - item: function(input, itemDone) - { - switch (++count) - { - case 1: - { - expect(input.id).to.equal(0); - expect(input.url).to.equal(helpers.urls[0]); - expect(input.data).to.equal(1); - break; - } - case 2: - { - expect(input.id).to.equal(1); - expect(input.url).to.equal(helpers.urls[1]); - expect(input.data).to.equal(2); - break; - } - case 3: - { - expect(input.id).to.equal(2); - expect(input.url).to.equal(helpers.urls[2]); - expect(input.data).to.equal(3); - done(); - break; - } - } - } - }); - - queue.enqueue({ id:0, url:helpers.urls[0], data:1 }); - queue.enqueue({ id:1, url:helpers.urls[1], data:2 }); - queue.enqueue({ id:2, url:helpers.urls[2], data:3 }); - }); - - - - it("is not called with erroneous URLs", function(done) - { - var queue = new RequestQueue(helpers.options(), - { - item: function(input, itemDone) - { - done( new Error("this should not have been called") ); - } - }); - - ["/path/","resource.html",""].forEach(queue.enqueue, queue); - - done(); - }); - - - - it("is not called with erroneous custom IDs", function(done) - { - var count = 0; - - var queue = new RequestQueue(helpers.options(), - { - item: function(input, itemDone) - { - if (++count == 1) - { - // Simulate a remote connection - setTimeout(done, helpers.delay); - } - else - { - done( new Error("this should not have been called") ); - } - } - }); - - queue.enqueue({id:123, url:helpers.urls[0]}); - queue.enqueue({id:123, url:helpers.urls[0]}); - queue.dequeue(123456); - }); - }); - - - - describe("end", function() - { - it("works", function(done) - { - var queue = new RequestQueue(helpers.options(), - { - item: function(input, itemDone) - { - setTimeout(itemDone, helpers.delay); - }, - end: function() - { - done(); - } - }); - - helpers.urls.forEach(queue.enqueue, queue); - }); - - - - it("is called when last item is dequeued", function(done) - { - var queue = new RequestQueue(helpers.options(), - { - end: function() - { - // Wait for `dequeued` to receive its value - // since everything here is performed synchronously - setImmediate( () => - { - expect(dequeued).to.be.true; - done(); - }); - } - }); - - // Prevent first queued item from immediately starting (and thus being auto-dequeued) - queue.pause(); - - var id = queue.enqueue(helpers.urls[0]); - var dequeued = queue.dequeue(id); - }); - - - - it("is not called simply by calling resume()", function(done) - { - var queue = new RequestQueue(helpers.options(), - { - end: function() - { - done( new Error("this should not have been called") ); - } - }); - - queue.resume(); - - setTimeout(done, helpers.delay*2); - }); - - - - it("is not called on erroneous dequeue", function(done) - { - var queue = new RequestQueue(helpers.options(), - { - end: function() - { - done( new Error("this should not have been called") ); - } - }); - - queue.dequeue("fakeid"); - - setTimeout(done, helpers.delay*2); - }); - }); - }); - - - - describe("pause() / resume()", function() - { - it("works", function(done) - { - var count = 0; - var resumed = false; - - helpers.testUrls(helpers.urls, helpers.options(), function(results) - { - expect(resumed).to.be.true; - expect(results).to.deep.equal(helpers.urls); - done(); - }, - function(input, queue) - { - if (++count === 1) - { - queue.pause(); - - // Wait longer than queue should take if not paused+resumed - setTimeout( () => - { - resumed = true; - queue.resume(); - - }, helpers.expectedSyncMinDuration()); - } - }); - }); - }); - - - - describe("numActive()", function() - { - it("works", function(done) - { - var queue = new RequestQueue(helpers.options(), - { - item: function(input, itemDone) - { - setTimeout(itemDone, helpers.delay); - }, - end: function() - { - expect( queue.numActive() ).to.equal(0); - done(); - } - }); - - helpers.urls.forEach(queue.enqueue, queue); - - expect( queue.numActive() ).to.equal(helpers.urls.length); - }); - }); - - - - describe("numQueued()", function() - { - it("works", function(done) - { - var queue = new RequestQueue(helpers.options(), - { - item: function(input, itemDone) - { - setTimeout(itemDone, helpers.delay); - }, - end: function() - { - expect( queue.numQueued() ).to.equal(0); - done(); - } - }); - - helpers.urls.forEach(queue.enqueue, queue); - - expect( queue.numQueued() ).to.equal(0); - }); - }); - - - - describe("Options", function() - { - describe("all disabled", function() - { - it("works", function(done) - { - helpers.testUrls(helpers.urls, helpers.options(), function(results) - { - expect(results).to.deep.equal(helpers.urls); - done(); - }); - }); - }); - - - - describe("maxSockets", function() - { - before( () => helpers.clearDurations() ); - - - - describe("=0", function() - { - it("does nothing", function(done) - { - helpers.testUrls(helpers.urls, helpers.options({ maxSockets:0 }), function(results) - { - done( new Error("this should not have been called") ); - }); - - setTimeout( () => done(), helpers.expectedSyncMinDuration() ); - }); - }); - - - - [1,2,3,4,Infinity].forEach(value => - { - describe("="+value, function() - { - before( () => helpers.addDurationGroup() ); - - - - it("works", function(done) - { - helpers.testUrls(helpers.urls, helpers.options({ maxSockets:value }), function(results, duration) - { - helpers.compareDurations(duration, function(prevGroupDuration) - { - expect(duration).to.be.below(prevGroupDuration); - }); - - expect(results).to.deep.equal(helpers.urls); - done(); - }); - }); - - - - it("supports ignorePorts=true", function(done) - { - helpers.testUrls(helpers.urls, helpers.options({ ignorePorts:true, maxSockets:value }), function(results) - { - expect(results).to.deep.equal(helpers.urls); - done(); - }); - }); - - - - it("supports ignoreSchemes=true", function(done) - { - helpers.testUrls(helpers.urls, helpers.options({ ignoreSchemes:true, maxSockets:value }), function(results) - { - expect(results).to.deep.equal(helpers.urls); - done(); - }); - }); - - - - it("supports ignoreSubdomains=true", function(done) - { - helpers.testUrls(helpers.urls, helpers.options({ ignoreSubdomains:true, maxSockets:value }), function(results) - { - expect(results).to.deep.equal(helpers.urls); - done(); - }); - }); - - - - it("supports all boolean options true", function(done) - { - helpers.testUrls(helpers.urls, helpers.options({ ignorePorts:true, ignoreSchemes:true, ignoreSubdomains:true, maxSockets:value }), function(results) - { - expect(results).to.deep.equal(helpers.urls); - done(); - }); - }); - }); - }); - }); - - - - describe("maxSocketsPerHost", function() - { - before( () => helpers.clearDurations() ); - - - - describe("=0", function() - { - it("does nothing", function(done) - { - helpers.testUrls(helpers.urls, helpers.options({ maxSocketsPerHost:0 }), function(results) - { - done( new Error("this should not have been called") ); - }); - - setTimeout( () => done(), helpers.expectedSyncMinDuration() ); - }); - }); - - - - describe("=1", function() - { - before( () => helpers.addDurationGroup() ); - - - - it("works", function(done) - { - helpers.testUrls(helpers.urls, helpers.options({ maxSocketsPerHost:1 }), function(results, duration) - { - helpers.compareDurations(duration, function(prevGroupDuration) - { - expect(duration).to.be.below(prevGroupDuration); - }); - - expect(results).to.deep.equal( - [ - "https://www.google.com/", - "http://www.google.com/", - "https://google.com/", - "http://google.com/", - "https://google.com:8080/", - "http://google.com:8080/", - "https://127.0.0.1/", - "http://127.0.0.1/", - - "https://www.google.com/", - "http://www.google.com/", - "https://google.com/", - "http://google.com/", - "https://google.com:8080/", - "http://google.com:8080/", - "https://127.0.0.1/", - "http://127.0.0.1/", - - "https://google.com/something.html", - "http://google.com/something.html", - "https://google.com/something.html", - "http://google.com/something.html" - ]); - done(); - }); - }); - - - - it("supports ignorePorts=true", function(done) - { - helpers.testUrls(helpers.urls, helpers.options({ ignorePorts:true, maxSocketsPerHost:1 }), function(results) - { - expect(results).to.deep.equal( - [ - "https://www.google.com/", - "http://www.google.com/", - "https://google.com/", - "http://google.com/", - - "https://127.0.0.1/", - "http://127.0.0.1/", - - "https://www.google.com/", - "http://www.google.com/", - "https://google.com/", - "http://google.com/", - - "https://127.0.0.1/", - "http://127.0.0.1/", - - "https://google.com:8080/", - "http://google.com:8080/", - "https://google.com:8080/", - "http://google.com:8080/", - - "https://google.com/something.html", - "http://google.com/something.html", - "https://google.com/something.html", - "http://google.com/something.html" - ]); - done(); - }); - }); - - - - it("supports ignoreSchemes=true", function(done) - { - helpers.testUrls(helpers.urls, helpers.options({ ignoreSchemes:true, maxSocketsPerHost:1 }), function(results) - { - expect(results).to.deep.equal( - [ - "https://www.google.com/", - "http://www.google.com/", // this LOOKS wrong, but isn't; it's consecutive because port is different from above (80 vs 443) - "https://google.com/", - "http://google.com/", - - "https://google.com:8080/", - - "https://127.0.0.1/", - "http://127.0.0.1/", - - "https://www.google.com/", - "http://www.google.com/", - "https://google.com/", - "http://google.com/", - - "https://google.com:8080/", - - "https://127.0.0.1/", - "http://127.0.0.1/", - - "https://google.com/something.html", - "http://google.com/something.html", - - "http://google.com:8080/", - - "https://google.com/something.html", - "http://google.com/something.html", - - "http://google.com:8080/" - ]); - done(); - }); - }); - - - - it("supports ignoreSubdomains=true", function(done) - { - helpers.testUrls(helpers.urls, helpers.options({ ignoreSubdomains:true, maxSocketsPerHost:1 }), function(results) - { - expect(results).to.deep.equal( - [ - "https://www.google.com/", - "http://www.google.com/", - - "https://google.com:8080/", - "http://google.com:8080/", - - "https://127.0.0.1/", - "http://127.0.0.1/", - - "https://www.google.com/", - "http://www.google.com/", - "https://google.com:8080/", - "http://google.com:8080/", - - "https://127.0.0.1/", - "http://127.0.0.1/", - - "https://google.com/", - "http://google.com/", - "https://google.com/", - "http://google.com/", - - "https://google.com/something.html", - "http://google.com/something.html", - "https://google.com/something.html", - "http://google.com/something.html" - ]); - done(); - }); - }); - - - - it("supports all boolean options true", function(done) - { - helpers.testUrls(helpers.urls, helpers.options({ ignorePorts:true, ignoreSchemes:true, ignoreSubdomains:true, maxSocketsPerHost:1 }), function(results) - { - expect(results).to.deep.equal( - [ - "https://www.google.com/", - "https://127.0.0.1/", - "https://www.google.com/", - "https://127.0.0.1/", - "http://www.google.com/", - "http://127.0.0.1/", - "http://www.google.com/", - "http://127.0.0.1/", - - "https://google.com/", - "https://google.com/", - "http://google.com/", - "http://google.com/", - - "https://google.com:8080/", - "https://google.com:8080/", - "http://google.com:8080/", - "http://google.com:8080/", - - "https://google.com/something.html", - "https://google.com/something.html", - "http://google.com/something.html", - "http://google.com/something.html" - ]); - done(); - }); - }); - }); - - - - describe("=2", function() - { - before( () => helpers.addDurationGroup() ); - - - - it("works", function(done) - { - helpers.testUrls(helpers.urls, helpers.options({ maxSocketsPerHost:2 }), function(results, duration) - { - helpers.compareDurations(duration, function(prevGroupDuration) - { - expect(duration).to.be.below(prevGroupDuration); - }); - - expect(results).to.deep.equal( - [ - "https://www.google.com/", - "https://www.google.com/", - "http://www.google.com/", - "http://www.google.com/", - "https://google.com/", - "https://google.com/", - "http://google.com/", - "http://google.com/", - "https://google.com:8080/", - "https://google.com:8080/", - "http://google.com:8080/", - "http://google.com:8080/", - - "https://127.0.0.1/", - "https://127.0.0.1/", - "http://127.0.0.1/", - "http://127.0.0.1/", - - "https://google.com/something.html", - "https://google.com/something.html", - "http://google.com/something.html", - "http://google.com/something.html" - ]); - done(); - }); - }); - - - - it("supports ignorePorts=true", function(done) - { - helpers.testUrls(helpers.urls, helpers.options({ ignorePorts:true, maxSocketsPerHost:2 }), function(results) - { - expect(results).to.deep.equal( - [ - "https://www.google.com/", - "https://www.google.com/", - "http://www.google.com/", - "http://www.google.com/", - "https://google.com/", - "https://google.com/", - "http://google.com/", - "http://google.com/", - - "https://127.0.0.1/", - "https://127.0.0.1/", - "http://127.0.0.1/", - "http://127.0.0.1/", - - "https://google.com:8080/", - "https://google.com:8080/", - "http://google.com:8080/", - "http://google.com:8080/", - - "https://google.com/something.html", - "https://google.com/something.html", - "http://google.com/something.html", - "http://google.com/something.html" - ]); - done(); - }); - }); - - - - it("supports ignoreSchemes=true", function(done) - { - helpers.testUrls(helpers.urls, helpers.options({ ignoreSchemes:true, maxSocketsPerHost:2 }), function(results) - { - expect(results).to.deep.equal( - [ - "https://www.google.com/", - "https://www.google.com/", - "http://www.google.com/", - "http://www.google.com/", - "https://google.com/", - "https://google.com/", - "http://google.com/", - "http://google.com/", - "https://google.com:8080/", - "https://google.com:8080/", - - "https://127.0.0.1/", - "https://127.0.0.1/", - "http://127.0.0.1/", - "http://127.0.0.1/", - - "https://google.com/something.html", - "https://google.com/something.html", - "http://google.com/something.html", - "http://google.com/something.html", - - "http://google.com:8080/", - "http://google.com:8080/" - ]); - done(); - }); - }); - - - - it("supports ignoreSubdomains=true", function(done) - { - helpers.testUrls(helpers.urls, helpers.options({ ignoreSubdomains:true, maxSocketsPerHost:2 }), function(results) - { - expect(results).to.deep.equal( - [ - "https://www.google.com/", - "https://www.google.com/", - "http://www.google.com/", - "http://www.google.com/", - - "https://google.com:8080/", - "https://google.com:8080/", - "http://google.com:8080/", - "http://google.com:8080/", - - "https://127.0.0.1/", - "https://127.0.0.1/", - "http://127.0.0.1/", - "http://127.0.0.1/", - - "https://google.com/", - "https://google.com/", - "http://google.com/", - "http://google.com/", - - "https://google.com/something.html", - "https://google.com/something.html", - "http://google.com/something.html", - "http://google.com/something.html" - ]); - done(); - }); - }); - - - - it("supports all boolean options true", function(done) - { - helpers.testUrls(helpers.urls, helpers.options({ ignorePorts:true, ignoreSchemes:true, ignoreSubdomains:true, maxSocketsPerHost:2 }), function(results) - { - expect(results).to.deep.equal( - [ - "https://www.google.com/", - "https://www.google.com/", - - "https://127.0.0.1/", - "https://127.0.0.1/", - - "http://www.google.com/", - "http://www.google.com/", - - "http://127.0.0.1/", - "http://127.0.0.1/", - - "https://google.com/", - "https://google.com/", - "http://google.com/", - "http://google.com/", - - "https://google.com:8080/", - "https://google.com:8080/", - "http://google.com:8080/", - "http://google.com:8080/", - - "https://google.com/something.html", - "https://google.com/something.html", - "http://google.com/something.html", - "http://google.com/something.html" - ]); - done(); - }); - }); - }); - - - - describe("=3", function() - { - before( () => helpers.addDurationGroup() ); - - - - it("works", function(done) - { - helpers.testUrls(helpers.urls, helpers.options({ maxSocketsPerHost:3 }), function(results, duration) - { - helpers.compareDurations(duration, function(prevGroupDuration) - { - expect(duration).to.be.within(prevGroupDuration-20, prevGroupDuration+20); - }); - - expect(results).to.deep.equal( - [ - "https://www.google.com/", - "https://www.google.com/", - "http://www.google.com/", - "http://www.google.com/", - "https://google.com/", - "https://google.com/", - "http://google.com/", - "http://google.com/", - "https://google.com:8080/", - "https://google.com:8080/", - "http://google.com:8080/", - "http://google.com:8080/", - "https://google.com/something.html", - - "http://google.com/something.html", - - "https://127.0.0.1/", - "https://127.0.0.1/", - "http://127.0.0.1/", - "http://127.0.0.1/", - - "https://google.com/something.html", - "http://google.com/something.html" - ]); - done(); - }); - }); - - - - it("supports ignorePorts=true", function(done) - { - helpers.testUrls(helpers.urls, helpers.options({ ignorePorts:true, maxSocketsPerHost:3 }), function(results) - { - expect(results).to.deep.equal( - [ - "https://www.google.com/", - "https://www.google.com/", - "http://www.google.com/", - "http://www.google.com/", - "https://google.com/", - "https://google.com/", - "http://google.com/", - "http://google.com/", - "https://google.com:8080/", - "http://google.com:8080/", - - "https://127.0.0.1/", - "https://127.0.0.1/", - "http://127.0.0.1/", - "http://127.0.0.1/", - - "https://google.com:8080/", - - "https://google.com/something.html", - - "http://google.com:8080/", - - "http://google.com/something.html", - "https://google.com/something.html", - "http://google.com/something.html" - ]); - done(); - }); - }); - - - - it("supports ignoreSchemes=true", function(done) - { - helpers.testUrls(helpers.urls, helpers.options({ ignoreSchemes:true, maxSocketsPerHost:3 }), function(results) - { - expect(results).to.deep.equal( - [ - "https://www.google.com/", - "https://www.google.com/", - "http://www.google.com/", - "http://www.google.com/", - "https://google.com/", - "https://google.com/", - "http://google.com/", - "http://google.com/", - "https://google.com:8080/", - "https://google.com:8080/", - "http://google.com:8080/", - - "https://google.com/something.html", - - "http://google.com/something.html", - - "https://127.0.0.1/", - "https://127.0.0.1/", - "http://127.0.0.1/", - "http://127.0.0.1/", - - "https://google.com/something.html", - - "http://google.com/something.html", - - "http://google.com:8080/" - ]); - done(); - }); - }); - - - - it("supports ignoreSubdomains=true", function(done) - { - helpers.testUrls(helpers.urls, helpers.options({ ignoreSubdomains:true, maxSocketsPerHost:3 }), function(results) - { - expect(results).to.deep.equal( - [ - "https://www.google.com/", - "https://www.google.com/", - "http://www.google.com/", - "http://www.google.com/", - "https://google.com/", - "http://google.com/", - - "https://google.com:8080/", - "https://google.com:8080/", - "http://google.com:8080/", - "http://google.com:8080/", - - "https://127.0.0.1/", - "https://127.0.0.1/", - "http://127.0.0.1/", - "http://127.0.0.1/", - - "https://google.com/", - - "https://google.com/something.html", - - "http://google.com/", - - "http://google.com/something.html", - "https://google.com/something.html", - "http://google.com/something.html" - ]); - done(); - }); - }); - - - - it("supports all boolean options true", function(done) - { - helpers.testUrls(helpers.urls, helpers.options({ ignorePorts:true, ignoreSchemes:true, ignoreSubdomains:true, maxSocketsPerHost:3 }), function(results) - { - expect(results).to.deep.equal( - [ - "https://www.google.com/", - "https://www.google.com/", - "http://www.google.com/", - - "https://127.0.0.1/", - "https://127.0.0.1/", - "http://127.0.0.1/", - - "http://www.google.com/", - - "https://google.com/", - "https://google.com/", - - "http://127.0.0.1/", - - "http://google.com/", - "http://google.com/", - - "https://google.com:8080/", - "https://google.com:8080/", - "http://google.com:8080/", - "http://google.com:8080/", - - "https://google.com/something.html", - "https://google.com/something.html", - "http://google.com/something.html", - "http://google.com/something.html" - ]); - done(); - }); - }); - }); - - - - describe("=4", function() - { - before( () => helpers.addDurationGroup() ); - - - - it("works", function(done) - { - helpers.testUrls(helpers.urls, helpers.options({ maxSocketsPerHost:4 }), function(results, duration) - { - helpers.compareDurations(duration, function(prevGroupDuration) - { - expect(duration).to.be.below(prevGroupDuration); - }); - - expect(results).to.deep.equal(helpers.urls); - done(); - }); - }); - - - - it("supports ignorePorts=true", function(done) - { - helpers.testUrls(helpers.urls, helpers.options({ ignorePorts:true, maxSocketsPerHost:4 }), function(results) - { - expect(results).to.deep.equal( - [ - "https://www.google.com/", - "https://www.google.com/", - "http://www.google.com/", - "http://www.google.com/", - "https://google.com/", - "https://google.com/", - "http://google.com/", - "http://google.com/", - "https://google.com:8080/", - "https://google.com:8080/", - "http://google.com:8080/", - "http://google.com:8080/", - - "https://127.0.0.1/", - "https://127.0.0.1/", - "http://127.0.0.1/", - "http://127.0.0.1/", - - "https://google.com/something.html", - "https://google.com/something.html", - "http://google.com/something.html", - "http://google.com/something.html" - ]); - done(); - }); - }); - - - - it("supports ignoreSchemes=true", function(done) - { - helpers.testUrls(helpers.urls, helpers.options({ ignoreSchemes:true, maxSocketsPerHost:4 }), function(results) - { - expect(results).to.deep.equal(helpers.urls); - done(); - }); - }); - - - - it("supports ignoreSubdomains=true", function(done) - { - helpers.testUrls(helpers.urls, helpers.options({ ignoreSubdomains:true, maxSocketsPerHost:4 }), function(results) - { - expect(results).to.deep.equal( - [ - "https://www.google.com/", - "https://www.google.com/", - "http://www.google.com/", - "http://www.google.com/", - "https://google.com/", - "https://google.com/", - "http://google.com/", - "http://google.com/", - "https://google.com:8080/", - "https://google.com:8080/", - "http://google.com:8080/", - "http://google.com:8080/", - - "https://127.0.0.1/", - "https://127.0.0.1/", - "http://127.0.0.1/", - "http://127.0.0.1/", - - "https://google.com/something.html", - "https://google.com/something.html", - "http://google.com/something.html", - "http://google.com/something.html" - ]); - done(); - }); - }); - - - - it("supports all boolean options true", function(done) - { - helpers.testUrls(helpers.urls, helpers.options({ ignorePorts:true, ignoreSchemes:true, ignoreSubdomains:true, maxSocketsPerHost:4 }), function(results) - { - expect(results).to.deep.equal( - [ - "https://www.google.com/", - "https://www.google.com/", - "http://www.google.com/", - "http://www.google.com/", - - "https://127.0.0.1/", - "https://127.0.0.1/", - "http://127.0.0.1/", - "http://127.0.0.1/", - - "https://google.com/", - "https://google.com/", - "http://google.com/", - "http://google.com/", - "https://google.com:8080/", - "https://google.com:8080/", - "http://google.com:8080/", - "http://google.com:8080/", - "https://google.com/something.html", - "https://google.com/something.html", - "http://google.com/something.html", - "http://google.com/something.html" - ]); - done(); - }); - }); - }); - - - - describe("=Infinity", function() - { - before( () => helpers.addDurationGroup() ); - - - - it("works", function(done) - { - helpers.testUrls(helpers.urls, helpers.options({ maxSocketsPerHost:Infinity }), function(results, duration) - { - helpers.compareDurations(duration, function(prevGroupDuration) - { - expect(duration).to.be.within(prevGroupDuration-20, prevGroupDuration+20); - }); - - expect(results).to.deep.equal(helpers.urls); - done(); - }); - }); - - - - it("supports ignorePorts=true", function(done) - { - helpers.testUrls(helpers.urls, helpers.options({ ignorePorts:true, maxSocketsPerHost:Infinity }), function(results) - { - expect(results).to.deep.equal(helpers.urls); - done(); - }); - }); - - - - it("supports ignoreSchemes=true", function(done) - { - helpers.testUrls(helpers.urls, helpers.options({ ignoreSchemes:true, maxSocketsPerHost:Infinity }), function(results) - { - expect(results).to.deep.equal(helpers.urls); - done(); - }); - }); - - - - it("supports ignoreSubdomains=true", function(done) - { - helpers.testUrls(helpers.urls, helpers.options({ ignoreSubdomains:true, maxSocketsPerHost:Infinity }), function(results) - { - expect(results).to.deep.equal(helpers.urls); - done(); - }); - }); - - - - it("supports all boolean options true", function(done) - { - helpers.testUrls(helpers.urls, helpers.options({ ignorePorts:true, ignoreSchemes:true, ignoreSubdomains:true, maxSocketsPerHost:Infinity }), function(results) - { - expect(results).to.deep.equal(helpers.urls); - done(); - }); - }); - }); - }); - - - - describe("rateLimit=50", function() - { - it("works", function(done) - { - helpers.testUrls(helpers.urls, helpers.options({ rateLimit:50 }), function(results, duration) - { - expect(duration).to.be.at.least(50); - expect(results).to.deep.equal(helpers.urls); - done(); - }); - }); - - - - it("supports ignorePorts=true", function(done) - { - helpers.testUrls(helpers.urls, helpers.options({ ignorePorts:true, rateLimit:50 }), function(results, duration) - { - expect(duration).to.be.at.least(50); - expect(results).to.deep.equal(helpers.urls); - done(); - }); - }); - - - - it("supports ignoreSchemes=true", function(done) - { - helpers.testUrls(helpers.urls, helpers.options({ ignoreSchemes:true, rateLimit:50 }), function(results, duration) - { - expect(duration).to.be.at.least(50); - expect(results).to.deep.equal(helpers.urls); - done(); - }); - }); - - - - it("supports ignoreSubdomains=true", function(done) - { - helpers.testUrls(helpers.urls, helpers.options({ ignoreSubdomains:true, rateLimit:50 }), function(results, duration) - { - expect(duration).to.be.at.least(50); - expect(results).to.deep.equal(helpers.urls); - done(); - }); - }); - - - - it("supports all boolean options true", function(done) - { - helpers.testUrls(helpers.urls, helpers.options({ ignorePorts:true, ignoreSchemes:true, ignoreSubdomains:true, rateLimit:50 }), function(results, duration) - { - expect(duration).to.be.at.least(50); - expect(results).to.deep.equal(helpers.urls); - done(); - }); - }); - }); - - - - describe("all boolean options true, maxSockets=2, maxSocketsPerHost=1, rateLimit=50", function() - { - it("works", function(done) - { - this.timeout(0); - - helpers.testUrls(helpers.urls, helpers.options({ ignorePorts:true, ignoreSchemes:true, ignoreSubdomains:true, maxSockets:2, maxSocketsPerHost:1, rateLimit:50 }), function(results, duration) - { - expect(duration).to.be.at.least( (helpers.urls.length-4) * (50 + helpers.delay) ); - expect(results).to.deep.equal( - [ - "https://www.google.com/", - - "https://127.0.0.1/", - - "https://www.google.com/", - - "https://127.0.0.1/", - - "http://www.google.com/", - - "http://127.0.0.1/", - - "http://www.google.com/", - - "http://127.0.0.1/", - - "https://google.com/", - "https://google.com/", - "http://google.com/", - "http://google.com/", - "https://google.com:8080/", - "https://google.com:8080/", - "http://google.com:8080/", - "http://google.com:8080/", - "https://google.com/something.html", - "https://google.com/something.html", - "http://google.com/something.html", - "http://google.com/something.html" - ]); - done(); - }); - }); - }); - - - - describe("default options", function() - { - it("works", function(done) - { - helpers.testUrls(helpers.urls, undefined, function(results) - { - expect(results).to.deep.equal( - [ - "https://www.google.com/", - "https://127.0.0.1/", - "https://www.google.com/", - "https://127.0.0.1/", - "http://www.google.com/", - "http://127.0.0.1/", - "http://www.google.com/", - "http://127.0.0.1/", - - "https://google.com/", - "https://google.com/", - "http://google.com/", - "http://google.com/", - - "https://google.com:8080/", - "https://google.com:8080/", - "http://google.com:8080/", - "http://google.com:8080/", - - "https://google.com/something.html", - "https://google.com/something.html", - "http://google.com/something.html", - "http://google.com/something.html" - ]); - done(); - }); - }); - }); - }); - - - - describe("Test suite functions", function() - { - it("helpers.testUrls reports erroneous URLs", function(done) - { - var urls = [ - "https://www.google.com/", - "path/to/resource.html", - "http://www.google.com/" - ]; - - helpers.testUrls(urls, helpers.options(), function(results) - { - expect(results[0]).to.be.instanceOf(Error); - expect(results[1]).to.equal("https://www.google.com/"); - expect(results[2]).to.equal("http://www.google.com/"); - done(); - }); - }); - }); -}); diff --git a/test/server/helpers/durations.js b/test/server/helpers/durations.js deleted file mode 100644 index b93e012..0000000 --- a/test/server/helpers/durations.js +++ /dev/null @@ -1,41 +0,0 @@ -"use strict"; -var durations = []; - - - -function addDurationGroup() -{ - durations.push([]); -} - - - -function clearDurations() -{ - durations.length = 0; -} - - - -function compareDurations(duration, callback) -{ - var curGroup = durations[ durations.length-1 ]; - var prevGroupDuration; - - curGroup.push(duration); - - if (durations.length > 1) - { - prevGroupDuration = durations[durations.length-2][ curGroup.length-1 ]; - callback(prevGroupDuration); - } -} - - - -module.exports = -{ - addGroup: addDurationGroup, - clear: clearDurations, - compare: compareDurations -}; diff --git a/test/server/helpers/options.js b/test/server/helpers/options.js deleted file mode 100644 index a1b630f..0000000 --- a/test/server/helpers/options.js +++ /dev/null @@ -1,29 +0,0 @@ -"use strict"; -var appDefaultOptions = require("../../../lib/defaultOptions"); - -var testDefaultOptions = -{ - // All other options will use default values - // as this will ensure that when they change, tests WILL break - ignorePorts: false, - ignoreSchemes: false, - ignoreSubdomains: false, - maxSocketsPerHost: Infinity -}; - - - -function options(overrides) -{ - return Object.assign - ( - {}, - appDefaultOptions, - testDefaultOptions, - overrides - ); -} - - - -module.exports = options; diff --git a/test/server/helpers/testUrls.js b/test/server/helpers/testUrls.js deleted file mode 100644 index c1ebc50..0000000 --- a/test/server/helpers/testUrls.js +++ /dev/null @@ -1,114 +0,0 @@ -"use strict"; -var RequestQueue = require("../../../lib"); - -var delay = 18; // long enough without trying everyone's patience -var _urls = -[ - "https://www.google.com/", - "https://www.google.com/", - - "http://www.google.com/", - "http://www.google.com/", - - "https://google.com/", - "https://google.com/", - - "http://google.com/", - "http://google.com/", - - "https://google.com:8080/", - "https://google.com:8080/", - - "http://google.com:8080/", - "http://google.com:8080/", - - "https://google.com/something.html", - "https://google.com/something.html", - - "http://google.com/something.html", - "http://google.com/something.html", - - "https://127.0.0.1/", - "https://127.0.0.1/", - - "http://127.0.0.1/", - "http://127.0.0.1/" -]; - - - -function doneCheck(result, results, urls, startTime, callback) -{ - var duration; - - if (results.push(result) >= urls.length) - { - duration = Date.now() - startTime; - - callback(results, duration); - } -} - - - -function expectedSyncMinDuration() -{ - return _urls.length * delay + 50; -} - - - -function testUrls(urls, libOptions, completeCallback, eachCallback) -{ - var queued; - var results = []; - var startTime = Date.now(); - - var queue = new RequestQueue(libOptions, - { - item: function(input, done) - { - if (typeof eachCallback === "function") - { - eachCallback(input, queue); - } - - // Simulate a remote connection - setTimeout( () => - { - done(); - doneCheck(input.url, results, urls, startTime, completeCallback); - - }, delay); - } - }); - - urls.forEach(url => - { - var error = queue.enqueue(url); - var input; - - if (error instanceof Error) - { - if (typeof eachCallback === "function") - { - // Simulate `item` handler argument - input = { url:url }; - - eachCallback(input, queue); - } - - doneCheck(error, results, urls, startTime, completeCallback); - } - }); -} - - - -module.exports = -{ - delay: delay, - expectedSyncMinDuration: expectedSyncMinDuration, - testUrls: testUrls, - urls: _urls -};