From 28bd040d76d8ff09e2021553f8e526f92cb8c145 Mon Sep 17 00:00:00 2001 From: cslee Date: Wed, 31 May 2017 18:19:12 +0800 Subject: [PATCH 1/3] Add domain directory in bySiteStructure filenameGenerator (#199) --- lib/filename-generator/by-site-structure.js | 3 +++ lib/utils/index.js | 11 +++++++++++ 2 files changed, 14 insertions(+) diff --git a/lib/filename-generator/by-site-structure.js b/lib/filename-generator/by-site-structure.js index 16964007..7bd6db74 100644 --- a/lib/filename-generator/by-site-structure.js +++ b/lib/filename-generator/by-site-structure.js @@ -6,9 +6,12 @@ var resourceTypeExtensions = require('../config/resource-ext-by-type'); module.exports = function generateFilename (resource, options) { var resourceUrl = resource.getUrl(); + var host = utils.getHostFromUrl(resourceUrl); var filePath = utils.getFilepathFromUrl(resourceUrl); var extension = utils.getFilenameExtension(filePath); + filePath = path.join(host.replace(':', '_'), filePath); + // If we have HTML from 'http://example.com/path' => set 'path/index.html' as filepath if (resource.isHtml()) { var htmlExtensions = resourceTypeExtensions[resourceTypes.html]; diff --git a/lib/utils/index.js b/lib/utils/index.js index 90f29d4d..09a3d474 100644 --- a/lib/utils/index.js +++ b/lib/utils/index.js @@ -71,6 +71,16 @@ function getHashFromUrl (u) { return url.parse(u).hash || ''; } +/** + * Returns host with port from given url + * Example: http://example.com:8080/some/path/file.js => example.com:8080 + * @param {string} u - url + * @returns {string} host with port + */ +function getHostFromUrl (u) { + return url.parse(u).host; +} + /** * Returns extension for given filepath * Example: some/path/file.js => .js @@ -135,6 +145,7 @@ module.exports = { getFilepathFromUrl, getFilenameExtension, getHashFromUrl, + getHostFromUrl, shortenFilename, waitAllFulfilled, normalizeUrl, From 99e78bb99bb48cf7cd2591e5098947007d78baae Mon Sep 17 00:00:00 2001 From: cslee Date: Wed, 31 May 2017 18:52:54 +0800 Subject: [PATCH 2/3] Add domain directory in bySiteStructure filenameGenerator - fix tests (#199) --- .../by-site-structure-test.js | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/test/unit/filename-generator/by-site-structure-test.js b/test/unit/filename-generator/by-site-structure-test.js index 5e0077d3..530ec563 100644 --- a/test/unit/filename-generator/by-site-structure-test.js +++ b/test/unit/filename-generator/by-site-structure-test.js @@ -10,13 +10,13 @@ var options = { defaultFilename: 'index.html' }; describe('FilenameGenerator: bySiteStructure', function() { it('should return the normalized relative path of the resource url', function(){ var r1 = new Resource('http://example.com/some/path/a.png'); - bySiteStructureFilenameGenerator(r1, options).should.equalFileSystemPath('some/path/a.png'); + bySiteStructureFilenameGenerator(r1, options).should.equalFileSystemPath('example.com/some/path/a.png'); var r2 = new Resource('http://example.com/a.png'); - bySiteStructureFilenameGenerator(r2, options).should.equalFileSystemPath('a.png'); + bySiteStructureFilenameGenerator(r2, options).should.equalFileSystemPath('example.com/a.png'); var r3 = new Resource('http://example.com/some/path/../images/a.png'); - bySiteStructureFilenameGenerator(r3, options).should.equalFileSystemPath('some/images/a.png'); + bySiteStructureFilenameGenerator(r3, options).should.equalFileSystemPath('example.com/some/images/a.png'); }); it('should add the defaultFilename to the path, for html resources without extension', function(){ @@ -24,15 +24,15 @@ describe('FilenameGenerator: bySiteStructure', function() { var r1 = new Resource('http://example.com/some/path/'); r1.isHtml = isHtmlMock; - bySiteStructureFilenameGenerator(r1, options).should.equalFileSystemPath('some/path/index.html'); + bySiteStructureFilenameGenerator(r1, options).should.equalFileSystemPath('example.com/some/path/index.html'); var r2 = new Resource('http://example.com/some/path'); r2.isHtml = isHtmlMock; - bySiteStructureFilenameGenerator(r2, options).should.equalFileSystemPath('some/path/index.html'); + bySiteStructureFilenameGenerator(r2, options).should.equalFileSystemPath('example.com/some/path/index.html'); var r3 = new Resource('http://example.com'); r3.isHtml = isHtmlMock; - bySiteStructureFilenameGenerator(r3, options).should.equalFileSystemPath('index.html'); + bySiteStructureFilenameGenerator(r3, options).should.equalFileSystemPath('example.com/index.html'); }); it('should add the defaultFilename to the path, for html resources with wrong extension', function(){ @@ -40,19 +40,19 @@ describe('FilenameGenerator: bySiteStructure', function() { var r1 = new Resource('http://example.com/some/path/test.com'); r1.isHtml = isHtmlMock; - bySiteStructureFilenameGenerator(r1, options).should.equalFileSystemPath('some/path/test.com/index.html'); + bySiteStructureFilenameGenerator(r1, options).should.equalFileSystemPath('example.com/some/path/test.com/index.html'); }); it('should normalize to safe relative paths, without ..', function(){ var r = new Resource('http://example.com/some/path/../../../../images/a.png'); - bySiteStructureFilenameGenerator(r, options).should.equalFileSystemPath('images/a.png'); + bySiteStructureFilenameGenerator(r, options).should.equalFileSystemPath('example.com/images/a.png'); }); it('should not replace thrice dot in filenames', function() { // if it replaces them we receive 'some/path/../../../../etc/passwd' // path.resolve('some/path/../../../../etc/passwd'); = '/etc/passwd' => which is not safe var r = new Resource('http://example.com/some/path/.../.../.../.../etc/passwd'); - bySiteStructureFilenameGenerator(r, options).should.equalFileSystemPath('some/path/.../.../.../.../etc/passwd'); + bySiteStructureFilenameGenerator(r, options).should.equalFileSystemPath('example.com/some/path/.../.../.../.../etc/passwd'); }); it('should shorten filename', function() { @@ -74,10 +74,10 @@ describe('FilenameGenerator: bySiteStructure', function() { it('should return decoded filepath', function() { var r = new Resource('https://developer.mozilla.org/ru/docs/JavaScript_%D1%88%D0%B5%D0%BB%D0%BB%D1%8B'); var filename = bySiteStructureFilenameGenerator(r, options); - filename.should.equalFileSystemPath('ru/docs/JavaScript_шеллы'); + filename.should.equalFileSystemPath('developer.mozilla.org/ru/docs/JavaScript_шеллы'); var r2 = new Resource('https://developer.mozilla.org/Hello%20G%C3%BCnter.png'); var filename2 = bySiteStructureFilenameGenerator(r2, options); - filename2.should.equalFileSystemPath('Hello Günter.png'); + filename2.should.equalFileSystemPath('developer.mozilla.org/Hello Günter.png'); }); }); From cf2cd967fc187375919c0c78ae3774c9d7fdd946 Mon Sep 17 00:00:00 2001 From: cslee Date: Thu, 1 Jun 2017 10:11:00 +0800 Subject: [PATCH 3/3] Add domain directory in bySiteStructure filenameGenerator - add test for url with port number (#199) --- test/unit/filename-generator/by-site-structure-test.js | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/test/unit/filename-generator/by-site-structure-test.js b/test/unit/filename-generator/by-site-structure-test.js index 530ec563..89d34706 100644 --- a/test/unit/filename-generator/by-site-structure-test.js +++ b/test/unit/filename-generator/by-site-structure-test.js @@ -19,6 +19,11 @@ describe('FilenameGenerator: bySiteStructure', function() { bySiteStructureFilenameGenerator(r3, options).should.equalFileSystemPath('example.com/some/images/a.png'); }); + it('should replace the colon, for url with port number', function(){ + var r1 = new Resource('http://example.com:8080/some/path/a.png'); + bySiteStructureFilenameGenerator(r1, options).should.equalFileSystemPath('example.com_8080/some/path/a.png'); + }); + it('should add the defaultFilename to the path, for html resources without extension', function(){ var isHtmlMock = sinon.stub().returns(true);