diff --git a/lib/scraper.js b/lib/scraper.js index 86b49b95..b416e4c2 100644 --- a/lib/scraper.js +++ b/lib/scraper.js @@ -22,6 +22,12 @@ function Scraper (options) { self.options.request = _.extend({}, defaults.request, options.request); self.options.urls = _.isArray(self.options.urls) ? self.options.urls : [self.options.urls]; + if (self.options.subdirectories) { + self.options.subdirectories.forEach((element) => { + element.extensions = element.extensions.map((ext) => ext.toLowerCase()); + }); + } + if (self.options.recursive) { self.options.sources = _.union(self.options.sources, recursiveSources); } diff --git a/lib/utils/index.js b/lib/utils/index.js index bdfacbe3..ff7f277b 100644 --- a/lib/utils/index.js +++ b/lib/utils/index.js @@ -78,7 +78,7 @@ function getHashFromUrl (u) { * @returns {string|null} - extension */ function getFilenameExtension (filepath) { - return (typeof filepath === 'string') ? path.extname(filepath) : null; + return (typeof filepath === 'string') ? path.extname(filepath).toLowerCase() : null; } function shortenFilename (filename) { diff --git a/test/unit/filename-generator/by-type-test.js b/test/unit/filename-generator/by-type-test.js index 3a2f55cb..7ed60d14 100644 --- a/test/unit/filename-generator/by-type-test.js +++ b/test/unit/filename-generator/by-type-test.js @@ -57,6 +57,18 @@ describe('FilenameGenerator: byType', function() { filename.should.equalFileSystemPath('img/a.png'); }); + it('should return filename with correct subdirectory when string cases are different', function() { + var options = { + subdirectories: [ + { directory: 'img', extensions: ['.png'] } + ] + }; + + var r = new Resource('http://example.com/a.PNG'); + var f = byTypeFilenameGenerator(r, options, []); + f.should.equalFileSystemPath('img/a.PNG'); + }); + it('should return different filename if desired filename is occupied', function() { var r = new Resource('http://second-example.com/a.png'); var filename = byTypeFilenameGenerator(r, {}, [ 'a.png' ]); diff --git a/test/unit/scraper-init-test.js b/test/unit/scraper-init-test.js index 0fa7c0f3..5d84701d 100644 --- a/test/unit/scraper-init-test.js +++ b/test/unit/scraper-init-test.js @@ -1,5 +1,5 @@ var should = require('should'); -var proxyquire = require('proxyquire'); +var proxyquire = require('proxyquire').noCallThru(); var sinon = require('sinon'); var path = require('path'); var Scraper = require('../../lib/scraper'); @@ -10,10 +10,10 @@ var urls = [ 'http://example.com' ]; describe('Scraper initialization', function () { describe('defaultFilename', function() { - var defaultsMock, Scraper; + var Scraper; before(function() { - defaultsMock = { defaultFilename: 'dummyFilename.txt' }; + var defaultsMock = { defaultFilename: 'dummyFilename.txt' }; Scraper = proxyquire('../../lib/scraper', { './config/defaults': defaultsMock }); @@ -25,7 +25,7 @@ describe('Scraper initialization', function () { directory: testDirname }); - s.options.defaultFilename.should.equalFileSystemPath(defaultsMock.defaultFilename); + s.options.defaultFilename.should.equalFileSystemPath('dummyFilename.txt'); }); it('should use defaultFilename sources if defaultFilename were passed', function () { @@ -40,10 +40,10 @@ describe('Scraper initialization', function () { }); describe('sources', function() { - var defaultsMock, Scraper; + var Scraper; before(function() { - defaultsMock = { sources: ['1', '2', '3'] }; + var defaultsMock = { sources: ['1', '2', '3'] }; Scraper = proxyquire('../../lib/scraper', { './config/defaults': defaultsMock }); @@ -55,7 +55,7 @@ describe('Scraper initialization', function () { directory: testDirname }); - s.options.sources.should.eql(defaultsMock.sources); + s.options.sources.should.eql(['1', '2', '3']); }); it('should use passed sources if sources were passed', function () { @@ -85,10 +85,10 @@ describe('Scraper initialization', function () { }); describe('subdirectories', function () { - var defaultsMock, Scraper; + var Scraper; before(function() { - defaultsMock = { directories: { directory: 'dir', extensions: ['.txt'] }, }; + var defaultsMock = { subdirectories: [{ directory: 'dir', extensions: ['.txt'] }] }; Scraper = proxyquire('../../lib/scraper', { './config/defaults': defaultsMock }); @@ -100,7 +100,20 @@ describe('Scraper initialization', function () { directory: testDirname }); - s.options.subdirectories.should.eql(defaultsMock.subdirectories); + s.options.subdirectories.should.eql([{ directory: 'dir', extensions: ['.txt'] }]); + }); + + it('should convert extensions to lower case', function () { + + var s = new Scraper({ + urls: urls, + directory: testDirname, + subdirectories: [ + { directory: 'dir', extensions: ['.TXT'] } + ] + }); + + s.options.subdirectories[0].extensions.should.eql(['.txt']); }); it('should use passed subdirectories if subdirectories were passed', function () { @@ -125,10 +138,10 @@ describe('Scraper initialization', function () { }); describe('request', function () { - var defaultsMock, Scraper; + var Scraper; before(function() { - defaultsMock = { request: { a: 1, b: 2 } }; + var defaultsMock = { request: { a: 1, b: 2 } }; Scraper = proxyquire('../../lib/scraper', { './config/defaults': defaultsMock }); @@ -140,7 +153,7 @@ describe('Scraper initialization', function () { directory: testDirname }); - s.options.request.should.eql(defaultsMock.request); + s.options.request.should.eql({ a: 1, b: 2 }); }); it('should merge default and passed objects if request were passed', function () {