Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We鈥檒l occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

馃棞 Set db soft limits #9225

Merged
merged 3 commits into from
Nov 9, 2017
Merged

馃棞 Set db soft limits #9225

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
31cce3e
馃棞 Set db soft limits
aileen Nov 8, 2017
1b4cbe9
Specific error message for isLength validator
aileen Nov 9, 2017
559f422
simplify logic for translating `isLength` for soft limits
kirrg001 Nov 9, 2017
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 10 additions & 10 deletions core/server/data/schema/schema.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ module.exports = {
posts: {
id: {type: 'string', maxlength: 24, nullable: false, primary: true},
uuid: {type: 'string', maxlength: 36, nullable: false, validations: {isUUID: true}},
title: {type: 'string', maxlength: 2000, nullable: false},
title: {type: 'string', maxlength: 2000, nullable: false, validations: {isLength: {max: 255}}},
slug: {type: 'string', maxlength: 191, nullable: false, unique: true},
mobiledoc: {type: 'text', maxlength: 1000000000, fieldtype: 'long', nullable: true},
html: {type: 'text', maxlength: 1000000000, fieldtype: 'long', nullable: true},
Expand All @@ -14,8 +14,8 @@ module.exports = {
status: {type: 'string', maxlength: 50, nullable: false, defaultTo: 'draft'},
locale: {type: 'string', maxlength: 6, nullable: true},
visibility: {type: 'string', maxlength: 50, nullable: false, defaultTo: 'public', validations: {isIn: [['public']]}},
meta_title: {type: 'string', maxlength: 2000, nullable: true},
meta_description: {type: 'string', maxlength: 2000, nullable: true},
meta_title: {type: 'string', maxlength: 2000, nullable: true, validations: {isLength: {max: 300}}},
meta_description: {type: 'string', maxlength: 2000, nullable: true, validations: {isLength: {max: 500}}},
author_id: {type: 'string', maxlength: 24, nullable: false},
created_at: {type: 'dateTime', nullable: false},
created_by: {type: 'string', maxlength: 24, nullable: false},
Expand Down Expand Up @@ -44,17 +44,17 @@ module.exports = {
email: {type: 'string', maxlength: 191, nullable: false, unique: true, validations: {isEmail: true}},
profile_image: {type: 'string', maxlength: 2000, nullable: true},
cover_image: {type: 'string', maxlength: 2000, nullable: true},
bio: {type: 'text', maxlength: 65535, nullable: true},
bio: {type: 'text', maxlength: 65535, nullable: true, validations: {isLength: {max: 200}}},
website: {type: 'string', maxlength: 2000, nullable: true, validations: {isEmptyOrURL: true}},
location: {type: 'text', maxlength: 65535, nullable: true},
location: {type: 'text', maxlength: 65535, nullable: true, validations: {isLength: {max: 150}}},
facebook: {type: 'string', maxlength: 2000, nullable: true},
twitter: {type: 'string', maxlength: 2000, nullable: true},
accessibility: {type: 'text', maxlength: 65535, nullable: true},
status: {type: 'string', maxlength: 50, nullable: false, defaultTo: 'active'},
locale: {type: 'string', maxlength: 6, nullable: true},
visibility: {type: 'string', maxlength: 50, nullable: false, defaultTo: 'public', validations: {isIn: [['public']]}},
meta_title: {type: 'string', maxlength: 2000, nullable: true},
meta_description: {type: 'string', maxlength: 2000, nullable: true},
meta_title: {type: 'string', maxlength: 2000, nullable: true, validations: {isLength: {max: 300}}},
meta_description: {type: 'string', maxlength: 2000, nullable: true, validations: {isLength: {max: 500}}},
tour: {type: 'text', maxlength: 65535, nullable: true},
last_seen: {type: 'dateTime', nullable: true},
created_at: {type: 'dateTime', nullable: false},
Expand Down Expand Up @@ -116,12 +116,12 @@ module.exports = {
id: {type: 'string', maxlength: 24, nullable: false, primary: true},
name: {type: 'string', maxlength: 191, nullable: false, validations: {matches: /^([^,]|$)/}},
slug: {type: 'string', maxlength: 191, nullable: false, unique: true},
description: {type: 'text', maxlength: 65535, nullable: true},
description: {type: 'text', maxlength: 65535, nullable: true, validations: {isLength: {max: 500}}},
feature_image: {type: 'string', maxlength: 2000, nullable: true},
parent_id: {type: 'string', nullable: true},
visibility: {type: 'string', maxlength: 50, nullable: false, defaultTo: 'public', validations: {isIn: [['public', 'internal']]}},
meta_title: {type: 'string', maxlength: 2000, nullable: true},
meta_description: {type: 'string', maxlength: 2000, nullable: true},
meta_title: {type: 'string', maxlength: 2000, nullable: true, validations: {isLength: {max: 300}}},
meta_description: {type: 'string', maxlength: 2000, nullable: true, validations: {isLength: {max: 500}}},
created_at: {type: 'dateTime', nullable: false},
created_by: {type: 'string', maxlength: 24, nullable: false},
updated_at: {type: 'dateTime', nullable: true},
Expand Down
68 changes: 45 additions & 23 deletions core/server/data/validation/index.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -190,7 +190,12 @@ validateSchema = function validateSchema(tableName, model) {

// TODO: check if mandatory values should be enforced
if (model[columnKey] !== null && model[columnKey] !== undefined) {
// check length
// check soft limits first, if present (validations: {isLength: {min, max}})
if (_.has(schema[tableName][columnKey], 'validations.isLength')) {

This comment was marked as abuse.

Sign in to view

This comment was marked as abuse.

Sign in to view

validationErrors = validationErrors.concat(validate(strVal, columnKey, schema[tableName][columnKey].validations, tableName));
}

// check hard limits (maxlength)
if (schema[tableName][columnKey].hasOwnProperty('maxlength')) {
if (!validator.isLength(strVal, 0, schema[tableName][columnKey].maxlength)) {
message = i18n.t('notices.data.validation.index.valueExceedsMaxLength',
Expand All @@ -206,7 +211,7 @@ validateSchema = function validateSchema(tableName, model) {
}
}

// check validations objects
// check any other validations objects
if (schema[tableName][columnKey].hasOwnProperty('validations')) {
validationErrors = validationErrors.concat(validate(strVal, columnKey, schema[tableName][columnKey].validations));
}
Expand Down Expand Up @@ -243,7 +248,7 @@ validateSettings = function validateSettings(defaultSettings, model) {
matchingDefault = defaultSettings[values.key];

if (matchingDefault && matchingDefault.validations) {
validationErrors = validationErrors.concat(validate(values.value, values.key, matchingDefault.validations));
validationErrors = validationErrors.concat(validate(values.value, values.key, matchingDefault.validations, 'settings'));
}

if (validationErrors.length !== 0) {
Expand All @@ -253,24 +258,28 @@ validateSettings = function validateSettings(defaultSettings, model) {
return Promise.resolve();
};

// Validate default settings using the validator module.
// Each validation's key is a method name and its value is an array of options
//
// eg:
// validations: { isURL: true, isLength: [20, 40] }
//
// will validate that a setting's length is a URL between 20 and 40 chars.
//
// If you pass a boolean as the value, it will specify the "good" result. By default
// the "good" result is assumed to be true.
//
// eg:
// validations: { isNull: false } // means the "good" result would
// // fail the `isNull` check, so
// // not null.
//
// available validators: https://github.com/chriso/validator.js#validators
validate = function validate(value, key, validations) {
/**
* Validate keys using the validator module.
* Each validation's key is a method name and its value is an array of options
* eg:
* validations: { isURL: true, isLength: [20, 40] }
* will validate that a values's length is a URL between 20 and 40 chars.
*
* If you pass a boolean as the value, it will specify the "good" result. By default
* the "good" result is assumed to be true.
* eg:
* validations: { isNull: false } // means the "good" result would
* // fail the `isNull` check, so
* // not null.
*
* available validators: https://github.com/chriso/validator.js#validators
* @param {String} value the value to validate.
* @param {String} key the db column key of the value to validate.
* @param {Object} validations the validations object as described above.
* @param {String} tableName (optional) the db table of the value to validate, used for error message.
* @return {Array} returns an Array including the found validation errors (empty if none found);
*/
validate = function validate(value, key, validations, tableName) {
var validationErrors = [];
value = _.toString(value);

Expand All @@ -286,8 +295,21 @@ validate = function validate(value, key, validations) {

validationOptions.unshift(value);

// equivalent of validator.isSomething(option1, option2)
if (validator[validationName].apply(validator, validationOptions) !== goodResult) {
// send a more userfriendly error message for the `isLength` validation, when soft limit is exceeded
if (validationName === 'isLength' && validator[validationName].apply(validator, validationOptions) !== goodResult) {
var message = i18n.t('notices.data.validation.index.valueExceedsMaxLength',
{
tableName: tableName || '',
columnKey: key,
maxlength: validations.isLength.max
});

validationErrors.push(new errors.ValidationError({
message: message,
context: tableName ? tableName + '.' + key : key
}));
// equivalent of validator.isSomething(option1, option2)
} else if (validator[validationName].apply(validator, validationOptions) !== goodResult) {
validationErrors.push(new errors.ValidationError({
message: i18n.t('notices.data.validation.index.validationFailed', {validationName: validationName, key: key})
}));
Expand Down
84 changes: 81 additions & 3 deletions core/test/integration/data/importer/importers/data_spec.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -572,14 +572,92 @@ describe('Import', function () {
testUtils.fixtures.loadExportFixture('export-001', {lts:true}).then(function (exported) {
exportData = exported;

// change title to 1001 characters
exportData.data.posts[0].title = new Array(2002).join('a');
// change title to 300 characters (soft limit is 255)
exportData.data.posts[0].title = new Array(300).join('a');
exportData.data.posts[0].tags = 'Tag';
return dataImporter.doImport(exportData);
}).then(function () {
(1).should.eql(0, 'Data import should not resolve promise.');
}).catch(function (error) {
error[0].message.should.eql('Value in [posts.title] exceeds maximum length of 2000 characters.');
error[0].message.should.eql('Value in [posts.title] exceeds maximum length of 255 characters.');
error[0].errorType.should.eql('ValidationError');

Promise.all([
knex('users').select(),
knex('posts').select(),
knex('tags').select()
]).then(function (importedData) {
should.exist(importedData);

importedData.length.should.equal(3, 'Did not get data successfully');

var users = importedData[0],
posts = importedData[1],
tags = importedData[2];

// we always have 1 user, the default user we added
users.length.should.equal(1, 'There should only be one user');

// Nothing should have been imported
posts.length.should.equal(0, 'Wrong number of posts');
tags.length.should.equal(0, 'no new tags');

done();
});
}).catch(done);
});
it('doesn\'t import a tag when meta title too long', function (done) {
var exportData;

testUtils.fixtures.loadExportFixture('export-001', {lts:true}).then(function (exported) {
exportData = exported;

// change meta_title to 305 characters (soft limit is 300)
exportData.data.tags[0].meta_title = new Array(305).join('a');
return dataImporter.doImport(exportData);
}).then(function () {
(1).should.eql(0, 'Data import should not resolve promise.');
}).catch(function (error) {
error[0].message.should.eql('Value in [tags.meta_title] exceeds maximum length of 300 characters.');
error[0].errorType.should.eql('ValidationError');

Promise.all([
knex('users').select(),
knex('posts').select(),
knex('tags').select()
]).then(function (importedData) {
should.exist(importedData);

importedData.length.should.equal(3, 'Did not get data successfully');

var users = importedData[0],
posts = importedData[1],
tags = importedData[2];

// we always have 1 user, the default user we added
users.length.should.equal(1, 'There should only be one user');

// Nothing should have been imported
posts.length.should.equal(0, 'Wrong number of posts');
tags.length.should.equal(0, 'no new tags');

done();
});
}).catch(done);
});
it('doesn\'t import a user user when bio too long', function (done) {
var exportData;

testUtils.fixtures.loadExportFixture('export-001', {lts:true}).then(function (exported) {
exportData = exported;

// change bio to 300 characters (soft limit is 200)
exportData.data.users[0].bio = new Array(300).join('a');
return dataImporter.doImport(exportData);
}).then(function () {
(1).should.eql(0, 'Data import should not resolve promise.');
}).catch(function (error) {
error[0].message.should.eql('Value in [users.bio] exceeds maximum length of 200 characters.');
error[0].errorType.should.eql('ValidationError');

Promise.all([
Expand Down