From 17f6f125c8714d901bc7fbee363bcb6bb25125a6 Mon Sep 17 00:00:00 2001 From: s0ph1e Date: Tue, 26 Apr 2016 13:45:13 +0300 Subject: [PATCH 1/2] Cleanup tests for html, css handlers and byType generator --- test/unit/file-handlers/css-test.js | 1 - test/unit/file-handlers/html-test.js | 59 ++--- .../unit/file-name-generators/by-type-test.js | 205 ++++++------------ 3 files changed, 87 insertions(+), 178 deletions(-) diff --git a/test/unit/file-handlers/css-test.js b/test/unit/file-handlers/css-test.js index 823bda15..350eed7a 100644 --- a/test/unit/file-handlers/css-test.js +++ b/test/unit/file-handlers/css-test.js @@ -16,7 +16,6 @@ describe('Css handler', function () { beforeEach(function() { scraper = new Scraper(defaultScraperOpts); - scraper.prepare(); }); describe('#loadCss(context, resource)', function() { diff --git a/test/unit/file-handlers/html-test.js b/test/unit/file-handlers/html-test.js index f28b0c43..f7f0ef7b 100644 --- a/test/unit/file-handlers/html-test.js +++ b/test/unit/file-handlers/html-test.js @@ -3,16 +3,13 @@ var _ = require('lodash'); var Promise = require('bluebird'); var sinon = require('sinon'); require('sinon-as-promised')(Promise); -var nock = require('nock'); -var fs = require('fs-extra'); var Scraper = require('../../../lib/scraper'); var Resource = require('../../../lib/resource'); var loadHtml = require('../../../lib/file-handlers/html'); -var testDirname = __dirname + '/.html-test'; var defaultScraperOpts = { urls: [ 'http://example.com' ], - directory: testDirname, + directory: __dirname + '/.html-test', sources: [ { selector: 'img', attr: 'src' }, { selector: 'img', attr: 'srcset' }, @@ -26,16 +23,7 @@ var scraper; describe('Html handler', function () { beforeEach(function() { - nock.cleanAll(); - nock.disableNetConnect(); scraper = new Scraper(defaultScraperOpts); - scraper.prepare(); - }); - - afterEach(function() { - nock.cleanAll(); - nock.enableNetConnect(); - fs.removeSync(testDirname); }); describe('#loadHtml(context, resource)', function() { @@ -96,21 +84,19 @@ describe('Html handler', function () { }); it('should not call loadResource if no sources in html', function(done) { - var loadResourceSpy = sinon.spy(scraper, 'loadResource'); + var loadResourceStub = sinon.stub(scraper, 'loadResource').resolves(); var po = new Resource('http://example.com', 'index.html'); po.setText(''); loadHtml(scraper, po).then(function() { - loadResourceSpy.called.should.be.eql(false); + loadResourceStub.called.should.be.eql(false); done(); }).catch(done); }); it('should not call loadResource if source attr is empty', function(done) { - nock('http://example.com').get('/test.png').reply(200, 'OK'); - - var loadResourceSpy = sinon.spy(scraper, 'loadResource'); + var loadResourceStub = sinon.stub(scraper, 'loadResource').resolves(); var html = ' \ \ @@ -123,15 +109,13 @@ describe('Html handler', function () { po.setText(html); loadHtml(scraper, po).then(function() { - loadResourceSpy.called.should.be.eql(false); + loadResourceStub.called.should.be.eql(false); done(); }).catch(done); }); it('should call loadResource once with correct params', function(done) { - nock('http://example.com').get('/test.png').reply(200, 'OK'); - - var loadResourceSpy = sinon.spy(scraper, 'loadResource'); + var loadResourceStub = sinon.stub(scraper, 'loadResource').resolves(); var html = ' \ \ @@ -144,18 +128,18 @@ describe('Html handler', function () { po.setText(html); loadHtml(scraper, po).then(function() { - loadResourceSpy.calledOnce.should.be.eql(true); - loadResourceSpy.args[0][0].url.should.be.eql('http://example.com/test.png'); + loadResourceStub.calledOnce.should.be.eql(true); + loadResourceStub.args[0][0].url.should.be.eql('http://example.com/test.png'); done(); }).catch(done); }); it('should call loadResource for each found source with correct params', function(done) { - nock('http://example.com').get('/a.jpg').reply(200, 'OK'); - nock('http://example.com').get('/b.css').reply(200, 'OK'); - nock('http://example.com').get('/c.js').reply(200, 'OK'); + var loadResourceStub = sinon.stub(scraper, 'loadResource'); + loadResourceStub.onFirstCall().resolves(new Resource('http://example.com/a.jpg', 'a.jpg')); + loadResourceStub.onSecondCall().resolves(new Resource('http://example.com/b.css', 'b.css')); + loadResourceStub.onThirdCall().resolves(new Resource('http://example.com/c.js', 'c.js')); - var loadResourceSpy = sinon.spy(scraper, 'loadResource'); var html = '\ \ \ @@ -174,10 +158,10 @@ describe('Html handler', function () { // order of loading is determined by order of sources in scraper options loadHtml(scraper, parentResource).then(function() { - loadResourceSpy.calledThrice.should.be.eql(true); - loadResourceSpy.args[0][0].url.should.be.eql('http://example.com/a.jpg'); - loadResourceSpy.args[1][0].url.should.be.eql('http://example.com/b.css'); - loadResourceSpy.args[2][0].url.should.be.eql('http://example.com/c.js'); + loadResourceStub.calledThrice.should.be.eql(true); + loadResourceStub.args[0][0].url.should.be.eql('http://example.com/a.jpg'); + loadResourceStub.args[1][0].url.should.be.eql('http://example.com/b.css'); + loadResourceStub.args[2][0].url.should.be.eql('http://example.com/c.js'); updateChildSpy.calledThrice.should.be.eql(true); done(); @@ -185,8 +169,6 @@ describe('Html handler', function () { }); it('should not replace the sources in text, for which loadResource returned null', function(done) { - nock('http://example.com').get('/scripts/c.js').once().reply(200, 'OK'); - var loadStub = sinon.stub(scraper, 'loadResource'); loadStub.onFirstCall().returns(Promise.resolve(null)); loadStub.onSecondCall().returns(Promise.resolve(null)); @@ -225,10 +207,6 @@ describe('Html handler', function () { }); it('should replace all sources in text with local files', function(done) { - nock('http://example.com').get('/public/img/a.jpg').reply(200, 'OK'); - nock('http://example.com').get('/b.css').reply(200, 'OK'); - nock('http://example.com').get('/scripts/c.js').once().reply(200, 'OK'); - var loadStub = sinon.stub(scraper, 'loadResource'); loadStub.onFirstCall().returns(Promise.resolve(new Resource('http://example.com/public/img/a.jpg', 'local/a.jpg'))); loadStub.onSecondCall().returns(Promise.resolve(new Resource('http://example.com/b.css', 'local/b.css'))); @@ -262,8 +240,6 @@ describe('Html handler', function () { }); it('should keep hash in url for html resources', function (done) { - nock('http://example.com').get('/page1.html').reply(200, 'OK'); - var resourceStub = new Resource('http://example.com/page1.html', 'local/page1.html'); sinon.stub(resourceStub, 'getType').returns('html'); sinon.stub(scraper, 'loadResource').returns(Promise.resolve(resourceStub)); @@ -287,8 +263,6 @@ describe('Html handler', function () { }); it('should remove hash from url for not-html resources', function (done) { - nock('http://example.com').get('/page1.html').reply(200, 'OK'); - var resourceStub = new Resource('http://example.com/page1.html', 'local/page1.html'); sinon.stub(resourceStub, 'getType').returns('other'); sinon.stub(scraper, 'loadResource').returns(Promise.resolve(resourceStub)); @@ -431,7 +405,6 @@ describe('Html handler', function () { defaultFilename: 'index.html', prettifyUrls: true }, defaultScraperOpts)); - scraper.prepare(); var loadStub = sinon.stub(scraper, 'loadResource'); loadStub.onFirstCall().returns(Promise.resolve(new Resource('http://example.com/other-page/index.html', 'other-page/index.html'))); diff --git a/test/unit/file-name-generators/by-type-test.js b/test/unit/file-name-generators/by-type-test.js index 73df7348..23c3abbf 100644 --- a/test/unit/file-name-generators/by-type-test.js +++ b/test/unit/file-name-generators/by-type-test.js @@ -4,162 +4,99 @@ var Scraper = require('../../../lib/scraper'); var Resource = require('../../../lib/resource'); var byTypeFilenameGenerator = require('../../../lib/filename-generators/by-type'); -var testDirname = __dirname + '/.scraper-test'; - describe('byTypeFilenameGenerator', function() { - it('should return resource filename', function(done) { - var s = new Scraper({ - urls: 'http://example.com', - directory: testDirname, - subdirectories: null - }); - s.prepare().then(function() { - var r = new Resource('http://example.com/a.png', 'b.png'); - var filename = byTypeFilenameGenerator(r, s.options, s.loadedResources); - filename.should.be.eql('b.png'); - done(); - }).catch(done); - }); + var s; - it('should return url-based filename if resource has no filename', function(done) { - var s = new Scraper({ + beforeEach(function() { + s = new Scraper({ urls: 'http://example.com', - directory: testDirname, + directory: __dirname + '/.scraper-test', subdirectories: null }); - - s.prepare().then(function() { - var r = new Resource('http://example.com/a.png', ''); - var filename = byTypeFilenameGenerator(r, s.options, s.loadedResources); - filename.should.be.eql('a.png'); - done(); - }).catch(done); }); - it('should add missed extensions for html resources', function (done) { - var s = new Scraper({ - urls: 'http://example.com', - directory: testDirname, - subdirectories: null - }); - - s.prepare().then(function() { - var r = new Resource('http://example.com/about', ''); - r.getType = sinon.stub().returns('html'); - var filename = byTypeFilenameGenerator(r, s.options, s.loadedResources); - filename.should.be.eql('about.html'); - done(); - }).catch(done); + it('should return resource filename', function() { + var r = new Resource('http://example.com/a.png', 'b.png'); + var filename = byTypeFilenameGenerator(r, s.options, s.loadedResources); + filename.should.be.eql('b.png'); }); - it('should add missed extensions for css resources', function (done) { - var s = new Scraper({ - urls: 'http://example.com', - directory: testDirname, - subdirectories: null - }); - - s.prepare().then(function() { - var r = new Resource('http://example.com/css', ''); - r.getType = sinon.stub().returns('css'); - var filename = byTypeFilenameGenerator(r, s.options, s.loadedResources); - filename.should.be.eql('css.css'); - done(); - }).catch(done); + it('should return url-based filename if resource has no filename', function() { + var r = new Resource('http://example.com/a.png', ''); + var filename = byTypeFilenameGenerator(r, s.options, s.loadedResources); + filename.should.be.eql('a.png'); }); - it('should not add missed extensions for other resources', function (done) { - var s = new Scraper({ - urls: 'http://example.com', - directory: testDirname, - subdirectories: null - }); - - s.prepare().then(function() { - var r = new Resource('http://1.gravatar.com/avatar/4d63e4a045c7ff22accc33dc08442f86?s=140&d=%2Fwp-content%2Fuploads%2F2015%2F05%2FGood-JOb-150x150.jpg&r=g', ''); - r.getType = sinon.stub().returns('home'); - var filename = byTypeFilenameGenerator(r, s.options, s.loadedResources); - filename.should.be.eql('4d63e4a045c7ff22accc33dc08442f86'); - done(); - }).catch(done); + it('should add missed extensions for html resources', function () { + var r = new Resource('http://example.com/about', ''); + r.getType = sinon.stub().returns('html'); + var filename = byTypeFilenameGenerator(r, s.options, s.loadedResources); + filename.should.be.eql('about.html'); }); - it('should return filename with correct subdirectory', function(done) { - var s = new Scraper({ - urls: 'http://example.com', - directory: testDirname, - subdirectories: [ - { directory: 'img', extensions: ['.jpg', '.png', '.svg'] } - ] - }); + it('should add missed extensions for css resources', function () { + var r = new Resource('http://example.com/css', ''); + r.getType = sinon.stub().returns('css'); + var filename = byTypeFilenameGenerator(r, s.options, s.loadedResources); + filename.should.be.eql('css.css'); + }); - s.prepare().then(function() { - var r = new Resource('http://example.com/a.png'); - var filename = byTypeFilenameGenerator(r, s.options, s.loadedResources); - filename.should.be.eql('img/a.png'); - done(); - }).catch(done); + it('should not add missed extensions for other resources', function () { + var r = new Resource('http://1.gravatar.com/avatar/4d63e4a045c7ff22accc33dc08442f86?s=140&d=%2Fwp-content%2Fuploads%2F2015%2F05%2FGood-JOb-150x150.jpg&r=g', ''); + r.getType = sinon.stub().returns('home'); + var filename = byTypeFilenameGenerator(r, s.options, s.loadedResources); + filename.should.be.eql('4d63e4a045c7ff22accc33dc08442f86'); }); - it('should return different filename if desired filename is occupied', function(done) { - var s = new Scraper({ - urls: 'http://example.com', - directory: testDirname, - subdirectories: null - }); + it('should return filename with correct subdirectory', function() { + s.options.subdirectories = [ + { directory: 'img', extensions: ['.jpg', '.png', '.svg'] } + ]; - s.prepare().then(function() { - var r1 = new Resource('http://first-example.com/a.png'); - var r2 = new Resource('http://second-example.com/a.png'); + var r = new Resource('http://example.com/a.png'); + var filename = byTypeFilenameGenerator(r, s.options, s.loadedResources); + filename.should.be.eql('img/a.png'); + }); - var f1 = byTypeFilenameGenerator(r1, s.options, s.loadedResources); - f1.should.be.eql('a.png'); - r1.setFilename(f1); - s.addLoadedResource(r1); + it('should return different filename if desired filename is occupied', function() { + var r1 = new Resource('http://first-example.com/a.png'); + var r2 = new Resource('http://second-example.com/a.png'); - var f2 = byTypeFilenameGenerator(r2, s.options, s.loadedResources); - f2.should.be.not.eql('a.png'); + var f1 = byTypeFilenameGenerator(r1, s.options, s.loadedResources); + f1.should.be.eql('a.png'); + r1.setFilename(f1); + s.addLoadedResource(r1); - done(); - }).catch(done); + var f2 = byTypeFilenameGenerator(r2, s.options, s.loadedResources); + f2.should.be.not.eql('a.png'); }); - it('should return different filename if desired filename is occupied N times', function(done) { - var s = new Scraper({ - urls: 'http://example.com', - directory: testDirname, - subdirectories: null - }); - - s.prepare().then(function() { - var r1 = new Resource('http://first-example.com/a.png'); - var r2 = new Resource('http://second-example.com/a.png'); - var r3 = new Resource('http://third-example.com/a.png'); - var r4 = new Resource('http://fourth-example.com/a.png'); - - var f1 = byTypeFilenameGenerator(r1, s.options, s.loadedResources); - f1.should.be.eql('a.png'); - r1.setFilename(f1); - s.addLoadedResource(r1); - - var f2 = byTypeFilenameGenerator(r2, s.options, s.loadedResources); - f2.should.be.not.eql(r1.getFilename()); - r2.setFilename(f2); - s.addLoadedResource(r2); - - var f3 = byTypeFilenameGenerator(r3, s.options, s.loadedResources); - f3.should.be.not.eql(r1.getFilename()); - f3.should.be.not.eql(r2.getFilename()); - r3.setFilename(f3); - s.addLoadedResource(r3); - - var f4 = byTypeFilenameGenerator(r4, s.options, s.loadedResources); - f4.should.be.not.eql(r1.getFilename()); - f4.should.be.not.eql(r2.getFilename()); - f4.should.be.not.eql(r3.getFilename()); - - done(); - }).catch(done); + it('should return different filename if desired filename is occupied N times', function() { + var r1 = new Resource('http://first-example.com/a.png'); + var r2 = new Resource('http://second-example.com/a.png'); + var r3 = new Resource('http://third-example.com/a.png'); + var r4 = new Resource('http://fourth-example.com/a.png'); + + var f1 = byTypeFilenameGenerator(r1, s.options, s.loadedResources); + f1.should.be.eql('a.png'); + r1.setFilename(f1); + s.addLoadedResource(r1); + + var f2 = byTypeFilenameGenerator(r2, s.options, s.loadedResources); + f2.should.be.not.eql(r1.getFilename()); + r2.setFilename(f2); + s.addLoadedResource(r2); + + var f3 = byTypeFilenameGenerator(r3, s.options, s.loadedResources); + f3.should.be.not.eql(r1.getFilename()); + f3.should.be.not.eql(r2.getFilename()); + r3.setFilename(f3); + s.addLoadedResource(r3); + + var f4 = byTypeFilenameGenerator(r4, s.options, s.loadedResources); + f4.should.be.not.eql(r1.getFilename()); + f4.should.be.not.eql(r2.getFilename()); + f4.should.be.not.eql(r3.getFilename()); }); }); \ No newline at end of file From 2d68f6f0e76cbe13b7abe8c24512732461f7b6cb Mon Sep 17 00:00:00 2001 From: s0ph1e Date: Tue, 26 Apr 2016 14:20:26 +0300 Subject: [PATCH 2/2] Fix typo --- test/unit/file-name-generators/by-site-structure-test.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/unit/file-name-generators/by-site-structure-test.js b/test/unit/file-name-generators/by-site-structure-test.js index 2d5aadbe..f86c8597 100644 --- a/test/unit/file-name-generators/by-site-structure-test.js +++ b/test/unit/file-name-generators/by-site-structure-test.js @@ -4,7 +4,7 @@ var bySiteStructureFilenameGenerator = require('../../../lib/filename-generators var options = { defaultFilename: 'index.html' }; -describe('byTypeFilenameGenerator', function() { +describe('byStructureFilenameGenerator', function() { it('should return the normalized absolute path of the resource url', function(){ var r1 = new Resource('http://example.com/some/path/a.png'); bySiteStructureFilenameGenerator(r1, options).should.equal('/some/path/a.png'); @@ -41,5 +41,5 @@ describe('byTypeFilenameGenerator', function() { it('should normalize to safe absolute paths, without ..', function(){ var r = new Resource('http://example.com/some/path/../../../../images/a.png'); bySiteStructureFilenameGenerator(r, options).should.equal('/images/a.png'); - }) + }); });