Skip to content

Use sanitizeFilaname in filename generators #425

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
Sep 23, 2020
Merged

Use sanitizeFilaname in filename generators #425

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
a1883a5
Use sanitizeFilaname in filename generators
s0ph1e Sep 23, 2020
6ef295a
Sanitize path
s0ph1e Sep 23, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions lib/filename-generator/by-site-structure.js
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
const _ = require('lodash');
const path = require('path');
const url = require('url');
const sanitizeFilename = require('sanitize-filename');
const utils = require('../utils');
const resourceTypes = require('../config/resource-types');
const resourceTypeExtensions = require('../config/resource-ext-by-type');
Expand All @@ -12,7 +13,7 @@ module.exports = function generateFilename (resource, {defaultFilename}) {
let filePath = utils.getFilepathFromUrl(resourceUrl);
const extension = utils.getFilenameExtension(filePath);

filePath = path.join(host.replace(':', '_'), filePath);
filePath = path.join(host, filePath);

// If have query string
if (urlParsed.query) {
Expand Down Expand Up @@ -49,8 +50,7 @@ module.exports = function generateFilename (resource, {defaultFilename}) {

function sanitizeFilepath (filePath) {
filePath = path.normalize(filePath);
let pathParts = filePath.split(path.sep);
pathParts = _.pull(pathParts, '..');
const pathParts = filePath.split(path.sep).map(pathPart => sanitizeFilename(pathPart, {replacement: '_'}));
pathParts[pathParts.length - 1] = utils.shortenFilename(_.last(pathParts));
return pathParts.join(path.sep);
}
3 changes: 2 additions & 1 deletion lib/filename-generator/by-type.js
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,13 +1,14 @@
const _ = require('lodash');
const path = require('path');
const sanitizeFilename = require('sanitize-filename');
const utils = require('../utils');
const typeExtensions = require('../config/resource-ext-by-type');

module.exports = function generateFilename (resource, {subdirectories, defaultFilename}, occupiedFileNames) {
const occupiedNames = getSubDirectoryNames({subdirectories}).concat(occupiedFileNames);

let filename = getFilenameForResource(resource, {subdirectories, defaultFilename});
filename = utils.shortenFilename(filename);
filename = utils.shortenFilename(sanitizeFilename(filename, {replacement: '_'}));

const extension = utils.getFilenameExtension(filename);
const directory = getDirectoryByExtension(extension, {subdirectories, defaultFilename});
Expand Down
21 changes: 21 additions & 0 deletions package-lock.json
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 1 addition & 0 deletions package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,7 @@
"normalize-url": "^4.0.0",
"p-queue": "^6.6.1",
"request": "^2.85.0",
"sanitize-filename": "^1.6.3",
"srcset": "^2.0.0"
},
"devDependencies": {
Expand Down
113 changes: 62 additions & 51 deletions test/unit/filename-generator/by-site-structure-test.js
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,103 +1,114 @@
var _ = require('lodash');
var should = require('should');
const _ = require('lodash');
const should = require('should');
require('../../utils/assertions');
var sinon = require('sinon');
var Resource = require('../../../lib/resource');
var bySiteStructureFilenameGenerator = require('../../../lib/filename-generator/by-site-structure');
const sinon = require('sinon');
const Resource = require('../../../lib/resource');
const bySiteStructureFilenameGenerator = require('../../../lib/filename-generator/by-site-structure');

var options = { defaultFilename: 'index.html' };
const options = { defaultFilename: 'index.html' };

describe('FilenameGenerator: bySiteStructure', function() {
it('should return the normalized relative path of the resource url', function(){
var r1 = new Resource('http://example.com/some/path/a.png');
describe('FilenameGenerator: bySiteStructure', () => {
it('should return the normalized relative path of the resource url', () =>{
const r1 = new Resource('http://example.com/some/path/a.png');
bySiteStructureFilenameGenerator(r1, options).should.equalFileSystemPath('example.com/some/path/a.png');

var r2 = new Resource('http://example.com/a.png');
const r2 = new Resource('http://example.com/a.png');
bySiteStructureFilenameGenerator(r2, options).should.equalFileSystemPath('example.com/a.png');
});

it('should remove . and .. from path', () => {
const r1 = new Resource('http://example.com/some/path/../images/a.png');
bySiteStructureFilenameGenerator(r1, options).should.equalFileSystemPath('example.com/some/images/a.png');

const r2 = new Resource('http://example.com/some/path/../../../images/b.png');
bySiteStructureFilenameGenerator(r2, options).should.equalFileSystemPath('example.com/images/b.png');

var r3 = new Resource('http://example.com/some/path/../images/a.png');
bySiteStructureFilenameGenerator(r3, options).should.equalFileSystemPath('example.com/some/images/a.png');
const r3 = new Resource('http://example.com/some/path/./images/c.png');
bySiteStructureFilenameGenerator(r3, options).should.equalFileSystemPath('example.com/some/path/images/c.png');
});

it('should replace the colon, for url with port number', function(){
var r1 = new Resource('http://example.com:8080/some/path/a.png');
it('should replace the colon, for url with port number', () =>{
const r1 = new Resource('http://example.com:8080/some/path/a.png');
bySiteStructureFilenameGenerator(r1, options).should.equalFileSystemPath('example.com_8080/some/path/a.png');
});

it('should add the defaultFilename to the path, for html resources without extension', function(){
var isHtmlMock = sinon.stub().returns(true);
it('should replace not allowed characters from filename', () => {
const r1 = new Resource('http://example.com/some/path/<*a*>.png');
bySiteStructureFilenameGenerator(r1, options).should.equalFileSystemPath('example.com/some/path/__a__.png');
});

it('should replace not allowed characters from path', () => {
const r1 = new Resource('http://example.com/some:path/a.png');
bySiteStructureFilenameGenerator(r1, options).should.equalFileSystemPath('example.com/some_path/a.png');
});

it('should add the defaultFilename to the path, for html resources without extension', () =>{
const isHtmlMock = sinon.stub().returns(true);

var r1 = new Resource('http://example.com/some/path/');
const r1 = new Resource('http://example.com/some/path/');
r1.isHtml = isHtmlMock;
bySiteStructureFilenameGenerator(r1, options).should.equalFileSystemPath('example.com/some/path/index.html');

var r2 = new Resource('http://example.com/some/path');
const r2 = new Resource('http://example.com/some/path');
r2.isHtml = isHtmlMock;
bySiteStructureFilenameGenerator(r2, options).should.equalFileSystemPath('example.com/some/path/index.html');

var r3 = new Resource('http://example.com');
const r3 = new Resource('http://example.com');
r3.isHtml = isHtmlMock;
bySiteStructureFilenameGenerator(r3, options).should.equalFileSystemPath('example.com/index.html');
});

it('should add the defaultFilename to the path, for html resources with wrong extension', function(){
var isHtmlMock = sinon.stub().returns(true);
it('should add the defaultFilename to the path, for html resources with wrong extension', () =>{
const isHtmlMock = sinon.stub().returns(true);

var r1 = new Resource('http://example.com/some/path/test.com');
const r1 = new Resource('http://example.com/some/path/test.com');
r1.isHtml = isHtmlMock;
bySiteStructureFilenameGenerator(r1, options).should.equalFileSystemPath('example.com/some/path/test.com/index.html');
});

it('should normalize to safe relative paths, without ..', function(){
var r = new Resource('http://example.com/some/path/../../../../images/a.png');
it('should normalize to safe relative paths, without ..', () =>{
const r = new Resource('http://example.com/some/path/../../../../images/a.png');
bySiteStructureFilenameGenerator(r, options).should.equalFileSystemPath('example.com/images/a.png');
});

it('should not replace thrice dot in filenames', function() {
// if it replaces them we receive 'some/path/../../../../etc/passwd'
// path.resolve('some/path/../../../../etc/passwd'); = '/etc/passwd' => which is not safe
var r = new Resource('http://example.com/some/path/.../.../.../.../etc/passwd');
bySiteStructureFilenameGenerator(r, options).should.equalFileSystemPath('example.com/some/path/.../.../.../.../etc/passwd');
});

it('should shorten filename', function() {
var resourceFilename = _.repeat('1', 1000) + '.png';
var r = new Resource('http://example.com/' + resourceFilename);
var filename = bySiteStructureFilenameGenerator(r, options);
it('should shorten filename', () => {
const resourceFilename = _.repeat('1', 1000) + '.png';
const r = new Resource('http://example.com/' + resourceFilename);
const filename = bySiteStructureFilenameGenerator(r, options);
should(filename.length).be.lessThan(255);
});

it('should shorten filename if resource is html without ext and default name is too long', function() {
var defaultFilename = _.repeat('1', 1000) + '.html';
var r = new Resource('http://example.com/path');
it('should shorten filename if resource is html without ext and default name is too long', () => {
const defaultFilename = _.repeat('1', 1000) + '.html';
const r = new Resource('http://example.com/path');
r.isHtml = sinon.stub().returns(true);
var filepath = bySiteStructureFilenameGenerator(r, { defaultFilename: defaultFilename });
var filename = _.last(filepath.split('/'));
const filepath = bySiteStructureFilenameGenerator(r, { defaultFilename: defaultFilename });
const filename = _.last(filepath.split('/'));
should(filename.length).be.lessThan(255);
});

it('should return decoded filepath', function() {
var r = new Resource('https://developer.mozilla.org/ru/docs/JavaScript_%D1%88%D0%B5%D0%BB%D0%BB%D1%8B');
var filename = bySiteStructureFilenameGenerator(r, options);
it('should return decoded filepath', () => {
const r = new Resource('https://developer.mozilla.org/ru/docs/JavaScript_%D1%88%D0%B5%D0%BB%D0%BB%D1%8B');
const filename = bySiteStructureFilenameGenerator(r, options);
filename.should.equalFileSystemPath('developer.mozilla.org/ru/docs/JavaScript_шеллы');

var r2 = new Resource('https://developer.mozilla.org/Hello%20G%C3%BCnter.png');
var filename2 = bySiteStructureFilenameGenerator(r2, options);
const r2 = new Resource('https://developer.mozilla.org/Hello%20G%C3%BCnter.png');
const filename2 = bySiteStructureFilenameGenerator(r2, options);
filename2.should.equalFileSystemPath('developer.mozilla.org/Hello Günter.png');
});

it('should keep query strings', function () {
var isHtmlMock = sinon.stub().returns(true);
it('should keep query strings', () => {
const isHtmlMock = sinon.stub().returns(true);

var r1 = new Resource('http://example.com/path?q=test');
const r1 = new Resource('http://example.com/path?q=test');
r1.isHtml = isHtmlMock;
bySiteStructureFilenameGenerator(r1, options).should.equalFileSystemPath('example.com/path/q=test.html');

var r2 = new Resource('http://example.com/path?q1=test1&q2=test2');
const r2 = new Resource('http://example.com/path?q1=test1&q2=test2');
r2.isHtml = isHtmlMock;
bySiteStructureFilenameGenerator(r2, options).should.equalFileSystemPath('example.com/path/q1=test1&q2=test2.html');

var r3 = new Resource('http://example.com/path/picture.png?q1=test1&q2=test2');
const r3 = new Resource('http://example.com/path/picture.png?q1=test1&q2=test2');
bySiteStructureFilenameGenerator(r3, options).should.equalFileSystemPath('example.com/path/picture_q1=test1&q2=test2.png');
})
});
});
Loading