From fae00c0d08f3c6ffc637c242ac342191e44f4849 Mon Sep 17 00:00:00 2001 From: Ilya Antipenko Date: Sat, 11 Mar 2017 12:29:48 +0200 Subject: [PATCH 1/3] Add failing tests for different string cases in extensions --- test/unit/filename-generator/by-type-test.js | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/test/unit/filename-generator/by-type-test.js b/test/unit/filename-generator/by-type-test.js index 3a2f55cb..90d8c1b4 100644 --- a/test/unit/filename-generator/by-type-test.js +++ b/test/unit/filename-generator/by-type-test.js @@ -57,6 +57,22 @@ describe('FilenameGenerator: byType', function() { filename.should.equalFileSystemPath('img/a.png'); }); + it('should return filename with correct subdirectory when string cases are different', function() { + var options = { + subdirectories: [ + { directory: 'img', extensions: ['.JPG', '.png'] } + ] + }; + + var r1 = new Resource('http://example.com/a.jpg'); + var f1 = byTypeFilenameGenerator(r1, options, []); + f1.should.equalFileSystemPath('img/a.jpg'); + + var r2 = new Resource('http://example.com/b.PNG'); + var f2 = byTypeFilenameGenerator(r2, options, []); + f2.should.equalFileSystemPath('img/b.PNG'); + }); + it('should return different filename if desired filename is occupied', function() { var r = new Resource('http://second-example.com/a.png'); var filename = byTypeFilenameGenerator(r, {}, [ 'a.png' ]); From 36287b08e2884034213f5a8435e8854e8b9863e2 Mon Sep 17 00:00:00 2001 From: Ilya Antipenko Date: Sat, 11 Mar 2017 13:03:16 +0200 Subject: [PATCH 2/3] Fix ignoring different string cases in extensions --- lib/scraper.js | 6 ++++++ lib/utils/index.js | 2 +- test/unit/filename-generator/by-type-test.js | 12 ++++-------- test/unit/scraper-init-test.js | 13 +++++++++++++ 4 files changed, 24 insertions(+), 9 deletions(-) diff --git a/lib/scraper.js b/lib/scraper.js index 86b49b95..b416e4c2 100644 --- a/lib/scraper.js +++ b/lib/scraper.js @@ -22,6 +22,12 @@ function Scraper (options) { self.options.request = _.extend({}, defaults.request, options.request); self.options.urls = _.isArray(self.options.urls) ? self.options.urls : [self.options.urls]; + if (self.options.subdirectories) { + self.options.subdirectories.forEach((element) => { + element.extensions = element.extensions.map((ext) => ext.toLowerCase()); + }); + } + if (self.options.recursive) { self.options.sources = _.union(self.options.sources, recursiveSources); } diff --git a/lib/utils/index.js b/lib/utils/index.js index bdfacbe3..ff7f277b 100644 --- a/lib/utils/index.js +++ b/lib/utils/index.js @@ -78,7 +78,7 @@ function getHashFromUrl (u) { * @returns {string|null} - extension */ function getFilenameExtension (filepath) { - return (typeof filepath === 'string') ? path.extname(filepath) : null; + return (typeof filepath === 'string') ? path.extname(filepath).toLowerCase() : null; } function shortenFilename (filename) { diff --git a/test/unit/filename-generator/by-type-test.js b/test/unit/filename-generator/by-type-test.js index 90d8c1b4..7ed60d14 100644 --- a/test/unit/filename-generator/by-type-test.js +++ b/test/unit/filename-generator/by-type-test.js @@ -60,17 +60,13 @@ describe('FilenameGenerator: byType', function() { it('should return filename with correct subdirectory when string cases are different', function() { var options = { subdirectories: [ - { directory: 'img', extensions: ['.JPG', '.png'] } + { directory: 'img', extensions: ['.png'] } ] }; - var r1 = new Resource('http://example.com/a.jpg'); - var f1 = byTypeFilenameGenerator(r1, options, []); - f1.should.equalFileSystemPath('img/a.jpg'); - - var r2 = new Resource('http://example.com/b.PNG'); - var f2 = byTypeFilenameGenerator(r2, options, []); - f2.should.equalFileSystemPath('img/b.PNG'); + var r = new Resource('http://example.com/a.PNG'); + var f = byTypeFilenameGenerator(r, options, []); + f.should.equalFileSystemPath('img/a.PNG'); }); it('should return different filename if desired filename is occupied', function() { diff --git a/test/unit/scraper-init-test.js b/test/unit/scraper-init-test.js index 0fa7c0f3..d47a5776 100644 --- a/test/unit/scraper-init-test.js +++ b/test/unit/scraper-init-test.js @@ -103,6 +103,19 @@ describe('Scraper initialization', function () { s.options.subdirectories.should.eql(defaultsMock.subdirectories); }); + it('should convert extensions to lower case', function () { + + var s = new Scraper({ + urls: urls, + directory: testDirname, + subdirectories: [ + { directory: 'dir', extensions: ['.TXT'] } + ] + }); + + s.options.subdirectories[0].extensions.should.eql(['.txt']); + }); + it('should use passed subdirectories if subdirectories were passed', function () { var s = new Scraper({ urls: urls, From efc57fd8a1145b391aace723fc41735f355afa92 Mon Sep 17 00:00:00 2001 From: Ilya Antipenko Date: Sat, 11 Mar 2017 13:42:32 +0200 Subject: [PATCH 3/3] Refactor test to use plain values in asserts --- test/unit/scraper-init-test.js | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/test/unit/scraper-init-test.js b/test/unit/scraper-init-test.js index d47a5776..5d84701d 100644 --- a/test/unit/scraper-init-test.js +++ b/test/unit/scraper-init-test.js @@ -1,5 +1,5 @@ var should = require('should'); -var proxyquire = require('proxyquire'); +var proxyquire = require('proxyquire').noCallThru(); var sinon = require('sinon'); var path = require('path'); var Scraper = require('../../lib/scraper'); @@ -10,10 +10,10 @@ var urls = [ 'http://example.com' ]; describe('Scraper initialization', function () { describe('defaultFilename', function() { - var defaultsMock, Scraper; + var Scraper; before(function() { - defaultsMock = { defaultFilename: 'dummyFilename.txt' }; + var defaultsMock = { defaultFilename: 'dummyFilename.txt' }; Scraper = proxyquire('../../lib/scraper', { './config/defaults': defaultsMock }); @@ -25,7 +25,7 @@ describe('Scraper initialization', function () { directory: testDirname }); - s.options.defaultFilename.should.equalFileSystemPath(defaultsMock.defaultFilename); + s.options.defaultFilename.should.equalFileSystemPath('dummyFilename.txt'); }); it('should use defaultFilename sources if defaultFilename were passed', function () { @@ -40,10 +40,10 @@ describe('Scraper initialization', function () { }); describe('sources', function() { - var defaultsMock, Scraper; + var Scraper; before(function() { - defaultsMock = { sources: ['1', '2', '3'] }; + var defaultsMock = { sources: ['1', '2', '3'] }; Scraper = proxyquire('../../lib/scraper', { './config/defaults': defaultsMock }); @@ -55,7 +55,7 @@ describe('Scraper initialization', function () { directory: testDirname }); - s.options.sources.should.eql(defaultsMock.sources); + s.options.sources.should.eql(['1', '2', '3']); }); it('should use passed sources if sources were passed', function () { @@ -85,10 +85,10 @@ describe('Scraper initialization', function () { }); describe('subdirectories', function () { - var defaultsMock, Scraper; + var Scraper; before(function() { - defaultsMock = { directories: { directory: 'dir', extensions: ['.txt'] }, }; + var defaultsMock = { subdirectories: [{ directory: 'dir', extensions: ['.txt'] }] }; Scraper = proxyquire('../../lib/scraper', { './config/defaults': defaultsMock }); @@ -100,7 +100,7 @@ describe('Scraper initialization', function () { directory: testDirname }); - s.options.subdirectories.should.eql(defaultsMock.subdirectories); + s.options.subdirectories.should.eql([{ directory: 'dir', extensions: ['.txt'] }]); }); it('should convert extensions to lower case', function () { @@ -138,10 +138,10 @@ describe('Scraper initialization', function () { }); describe('request', function () { - var defaultsMock, Scraper; + var Scraper; before(function() { - defaultsMock = { request: { a: 1, b: 2 } }; + var defaultsMock = { request: { a: 1, b: 2 } }; Scraper = proxyquire('../../lib/scraper', { './config/defaults': defaultsMock }); @@ -153,7 +153,7 @@ describe('Scraper initialization', function () { directory: testDirname }); - s.options.request.should.eql(defaultsMock.request); + s.options.request.should.eql({ a: 1, b: 2 }); }); it('should merge default and passed objects if request were passed', function () {