diff --git a/lib/file-handlers/css.js b/lib/file-handlers/css.js index 7a17135c..4a7cfcd9 100644 --- a/lib/file-handlers/css.js +++ b/lib/file-handlers/css.js @@ -1,5 +1,5 @@ -var Promise = require('bluebird'); var _ = require('underscore'); +var Promise = require('bluebird'); var getCssUrls = require('css-url-parser'); var Resource = require('../resource'); var utils = require('../utils'); @@ -22,10 +22,10 @@ function loadCss (context, resource) { }); }); - return Promise.settle(promises).then(function () { + return utils.waitAllFulfilled(promises).then(function () { resource.setText(text); return resource; }); } -module.exports = loadCss; \ No newline at end of file +module.exports = loadCss; diff --git a/lib/file-handlers/html.js b/lib/file-handlers/html.js index 95344f98..95d28058 100644 --- a/lib/file-handlers/html.js +++ b/lib/file-handlers/html.js @@ -1,5 +1,5 @@ -var Promise = require('bluebird'); var cheerio = require('cheerio'); +var Promise = require('bluebird'); var utils = require('../utils'); var Resource = require('../resource'); @@ -63,11 +63,11 @@ function loadResources (context, resource, source) { return Promise.reject(); }); - return Promise.settle(promises).then(function () { + return utils.waitAllFulfilled(promises).then(function () { text = $.html(); resource.setText(text); return resource; }); } -module.exports = loadHtml; \ No newline at end of file +module.exports = loadHtml; diff --git a/lib/request.js b/lib/request.js index b6c6ce62..266fe27a 100644 --- a/lib/request.js +++ b/lib/request.js @@ -1,6 +1,7 @@ var _ = require('underscore'); var Promise = require('bluebird'); -var request = Promise.promisifyAll(require('request')); +var request = require('request'); +var get = Promise.promisify(request.get); var defaultOptions = { method: 'GET', @@ -21,10 +22,10 @@ function makeRequest(options, url) { var requestOptions = getCustomOptions(options); requestOptions.url = url; - return request.getAsync(requestOptions).then(function (data) { + return get(requestOptions).then(function handleResponse(data) { return { - url: data[0].request.href, - body: data[0].body + url: data.request.href, + body: data.body }; }); } diff --git a/lib/scraper.js b/lib/scraper.js index dd4a46d4..4c12472f 100644 --- a/lib/scraper.js +++ b/lib/scraper.js @@ -165,7 +165,7 @@ Scraper.prototype.scrape = function scrape(callback) { .then(self.prepare) .then(self.load) .catch(self.errorCleanup) - .nodeify(callback); + .asCallback(callback); }; module.exports = Scraper; diff --git a/lib/utils.js b/lib/utils.js index 79b066bf..8729f8e9 100644 --- a/lib/utils.js +++ b/lib/utils.js @@ -1,5 +1,6 @@ var url = require('url'); var path = require('path'); +var Promise = require('bluebird'); function isUrl(path) { var urlRegexp = /^((http[s]?:)?\/\/)/; @@ -29,10 +30,17 @@ function getFilenameFromUrl (u) { return path.basename(url.parse(u).pathname); } +function waitAllFulfilled(promises) { + return Promise.all(promises.map(function(promise) { + return promise.reflect(); + })); +} + module.exports = { isUrl: isUrl, getUrl: getUrl, getUnixPath: getUnixPath, getRelativePath: getRelativePath, - getFilenameFromUrl: getFilenameFromUrl + getFilenameFromUrl: getFilenameFromUrl, + waitAllFulfilled: waitAllFulfilled }; diff --git a/package.json b/package.json index d4ceb599..274b0678 100644 --- a/package.json +++ b/package.json @@ -31,17 +31,17 @@ }, "homepage": "https://github.com/s0ph1e/node-website-scraper", "dependencies": { - "bluebird": "^2.3.2", + "bluebird": "^3.0.1", "cheerio": "0.11.0", "compare-urls": "^1.0.0", "css-url-parser": "^0.1.0", - "fs-extra": "^0.22.1", + "fs-extra": "^0.26.0", "request": "^2.42.0", "underscore": "^1.7.0" }, "devDependencies": { "codeclimate-test-reporter": "^0.1.0", - "istanbul": "^0.3.17", + "istanbul": "^0.4.0", "mocha": "^2.2.5", "nock": "^2.9.1", "should": "^7.0.2",