Permalink
Browse files

Merge pull request #88 from neotoma/features/improve-data-storage

Add contentType dataTemplate support
  • Loading branch information...
markmhx committed Nov 13, 2017
2 parents d9b1f15 + ecb9c0e commit fc5a2a2412ad405f5e1c670f1f6963c4300fe527
View
@@ -49,6 +49,8 @@ queue.process('storeItemData', function(queueJob, done) {
};
var getJob = (item, done) => {
if (!queueJob.data.jobId) { return done(undefined, item, undefined); }
Job.findById(queueJob.data.jobId, (error, job) => {
if (error) {
done(error);
@@ -61,7 +63,7 @@ queue.process('storeItemData', function(queueJob, done) {
};
var storeItemData = (item, job, done) => {
debug.start('queueJob storeItemData', item.id, job.id);
debug.start('queueJob storeItemData', item.id, job ? job.id : null);
module.exports.storeItemData(item, queueJob.data.data, job, done);
};
@@ -338,6 +340,26 @@ module.exports.itemsPageNextPagination = function(page, pagination, contentType)
return nextPagination;
};
/**
* Callbacks ID used to store item on storage.
* @param {Item} item - Item.
* @param {Object} data - Raw item data from source.
* @param {function} done - Error-first callback function expecting ID as second parameter.
*/
module.exports.storageId = function(item, data, done) {
var validate = function(done) {
validateParams([{
name: 'item', variable: item, required: true, requiredProperties: ['id', 'contentType']
}], done);
};
var storageId = function(done) {
done(undefined, `${item.source.lowercaseName()}-${data.id}`);
};
async.waterfall([validate, storageId], done);
};
/**
* Callbacks file system path used to store item on storage.
* @param {Item} item - Item.
@@ -351,13 +373,15 @@ module.exports.storagePath = function(item, data, done) {
}], done);
};
var storagePath = function(done) {
var path = '/' + item.source.kebabName() + '/' + item.contentType.pluralKebabName() + '/' + item.slug(data) + '.json';
debug.success('storagePath: %s', path);
done(undefined, path);
var storageId = function(done) {
module.exports.storageId(item, data, done);
};
var storagePath = function(id, done) {
done(undefined, `/${item.contentType.pluralLowercaseName()}/${id}.json`);
};
async.waterfall([validate, storagePath], done);
async.waterfall([validate, storageId, storagePath], done);
};
/**
@@ -794,6 +818,71 @@ module.exports.storeItemData = function(item, data, job, done) {
});
};
var storageId = function(done) {
module.exports.storageId(item, data, done);
};
var formatData = function(id, done) {
var formattedData = {
'id': id,
'type': item.contentType.pluralLowercaseName(),
'attributes': {}
};
if (!item.contentType.dataTemplate) {
formattedData.attributes = data;
data = formattedData;
return done();
}
var extractValue = function(data, path) {
try {
var value = path;
var variables = path.match(/\$\{.+?\}/g);
variables.forEach((variable) => {
var matches = variable.match(/\$\{(.+?)\}/);
var variableValue = _.get(data, matches[1]);
if (!variableValue) {
throw new Error('Variable value null or undefined');
}
debug.trace('replace %s with %s', variable, variableValue);
value = value.replace(variable, _.get(data, matches[1]));
});
debug.trace('variables for %s: %o -> %s', path, variables, value);
return value;
} catch (error) {
return _.get(data, path);
}
};
Object.keys(item.contentType.dataTemplate).forEach((key) => {
switch (typeof item.contentType.dataTemplate[key]) {
case 'string':
formattedData.attributes[key] = extractValue(data, item.contentType.dataTemplate[key]);
break;
case 'object':
formattedData.attributes[key] = extractValue(data, item.contentType.dataTemplate[key]['path']);
if (item.contentType.dataTemplate[key]['type'] === 'epoch') {
debug.trace('converting epoch time: %s', formattedData.attributes[key]);
var date = new Date(formattedData.attributes[key] * 1000);
formattedData.attributes[key] = date.toISOString();
}
break;
}
});
data = formattedData;
done();
};
var storeFile = function(done) {
module.exports.storeFile(item.user, item.storage, item.storagePath, data, (error, storeFileResult) => {
if (error) {
@@ -846,6 +935,8 @@ module.exports.storeItemData = function(item, data, job, done) {
validate,
setupLog,
updateStorageAttemptedAt,
storageId,
formatData,
storeFile,
updateStorageProperties,
updateJob,
@@ -915,7 +1006,7 @@ module.exports.storeFile = function(user, storage, path, data, done) {
var prepareData = function(done) {
debug.start('storeFile (path: %s)', path);
if (!(data instanceof Buffer)) {
data = JSON.stringify(data);
data = JSON.stringify(data, null, 2);
}
done();
View
@@ -15,6 +15,8 @@ var validateParams = require('./validateParams');
var jsonapi = {};
jsonapi.adminFlag = (process.env.SYNC_SERVER_PUBLIC_ADMIN == 'true') ? 'public' : 'admin';
/**
* Returns JSON API resource object representing provided Mongoose document
* Document properties filtered out per model settings
View
@@ -23,7 +23,7 @@ module.exports = {
jsonUrl: 'http://example.com/foo.json',
jsonData: function() {
return { foo1: 'bar1', foo2: 'bar2' };
return { id: 'barId', foo1: 'bar1', foo2: 'bar2' };
},
pagination: () => {
@@ -6,6 +6,7 @@
var async = require('async');
var ContactVerificationRequest = require('./contactVerificationRequest');
var debug = require('debug')('syncServer:contactVerification');
var jsonapi = require('app/lib/jsonapi');
var logger = require('app/lib/logger');
var modelFactory = require('app/factories/model');
var NotificationRequest = require('app/models/notificationRequest');
@@ -26,14 +27,14 @@ module.exports = modelFactory.new('ContactVerification', {
user: { ref: 'User' }
}, {
jsonapi: {
delete: 'admin',
delete: jsonapi.adminFlag,
get: {
allowed: 'public',
queryConditions: function(req, done) {
done(undefined, { session: req.session.id });
}
},
patch: 'admin',
patch: jsonapi.adminFlag,
post: {
allowed: 'public',
View
@@ -3,19 +3,23 @@
* @module
*/
var jsonapi = require('app/lib/jsonapi');
var modelFactory = require('app/factories/model');
var nameMethods = require('./methods/name');
var mongoose = require('app/lib/mongoose');
var nameMethods = require('app/models/methods/name');
/**
* Represents type of content available from source for storage
* @class ContentType
* @property {string} name - Name of contentType (e.g. "Photo")
*/
module.exports = modelFactory.new('ContentType', {
name: { type: String, required: true }
name: { type: String, required: true },
dataTemplate: { type: mongoose.Schema.Types.Mixed }
}, {
jsonapi: {
get: 'public',
post: 'admin'
patch: jsonapi.adminFlag,
post: jsonapi.adminFlag
}
}, nameMethods);
}, nameMethods);
View
@@ -3,15 +3,8 @@
* @module
*/
var _ = require('lodash');
var emojiStrip = require('emoji-strip');
var modelFactory = require('app/factories/model');
var queryConditions = require('./queryConditions');
var sanitizeFilename = require('sanitize-filename');
var convertToFilename = function(content) {
return _.toLower(emojiStrip(sanitizeFilename(content).replace(/[^\x00-\x7F]/g, '').replace('.','').replace('-', ' ').replace(/ {2}/g, ' ').replace(/ +/g, '-').replace(/|+/g, '-')));
};
/**
* Represents atomic unit of content available from source for storage
@@ -49,36 +42,4 @@ module.exports = modelFactory.new('Item', {
queryConditions: queryConditions.userMatchesRequester
}
}
}, {
slug: function(data) {
var parts = [];
if (data) {
if (data.createdAt) {
var date = new Date(data.createdAt * 1000);
var dateString = date.toISOString();
parts.push(dateString.substring(0, dateString.indexOf('T')));
}
if (data.venue && data.venue.name) {
parts.push(convertToFilename(data.venue.name));
} else if (data.firstName || data.lastName) {
if (data.firstName) {
parts.push(data.firstName);
}
if (data.lastName) {
parts.push(data.lastName);
}
} else if (data.text) {
parts.push(data.text);
}
}
if (!parts.length) {
parts.push(this.id);
}
return parts.map((part) => convertToFilename(part)).join('-');
}
});
View
@@ -3,8 +3,9 @@
* @module
*/
var jsonapi = require('app/lib/jsonapi');
var modelFactory = require('app/factories/model');
var nameMethods = require('./methods/name');
var nameMethods = require('app/models/methods/name');
var templateCompiler = require('es6-template-strings');
var methods = Object.assign({
@@ -66,10 +67,10 @@ module.exports = modelFactory.new('Source', {
totalItemsAvailableFromPagePathTemplate: String
}, {
jsonapi: {
delete: 'admin',
delete: jsonapi.adminFlag,
filteredProperties: ['clientId', 'clientSecret'],
get: 'public',
patch: 'admin',
post: 'admin'
patch: jsonapi.adminFlag,
post: jsonapi.adminFlag
}
}, methods);
}, methods);
View
@@ -3,6 +3,7 @@
* @module
*/
var jsonapi = require('app/lib/jsonapi');
var modelFactory = require('app/factories/model');
var templateCompiler = require('es6-template-strings');
var validateParams = require('app/lib/validateParams');
@@ -35,11 +36,11 @@ module.exports = modelFactory.new('Storage', {
}
}, {
jsonapi: {
delete: 'admin',
delete: jsonapi.adminFlag,
filteredProperties: ['clientId', 'clientSecret'],
get: 'public',
patch: 'admin',
post: 'admin'
patch: jsonapi.adminFlag,
post: jsonapi.adminFlag
}
}, {
/**
@@ -54,8 +55,9 @@ module.exports = modelFactory.new('Storage', {
'Authorization': 'Bearer ' + userStorageAuth.storageToken,
'Content-Type': 'application/octet-stream',
'Dropbox-API-Arg': JSON.stringify({
autorename: true,
autorename: false,
mode: 'add',
mute: true,
path: path
})
};
@@ -76,9 +78,10 @@ module.exports = modelFactory.new('Storage', {
}]);
return templateCompiler(this.itemPutUrlTemplate, {
accessToken: userStorageAuth.storageToken,
apiVersion: this.apiVersion,
host: this.host,
path: path,
accessToken: userStorageAuth.storageToken
path: path
});
}
});
View
@@ -9,7 +9,6 @@
"cookie-parser": "^1.4.3",
"cors": "^2.8.1",
"debug": "^3.1.0",
"emoji-strip": "^1.0.0",
"es6-template-strings": "^2.0.1",
"eslint": "^4.10.0",
"express": "~4.16.2",
@@ -58,7 +57,6 @@
"request": "^2.75.0",
"require-dir": "^0.3.1",
"rsync": "^0.6.1",
"sanitize-filename": "^1.6.1",
"sinon": "~4.1.0",
"socket.io": "~2.0.4",
"supertest": "^3.0.0",
@@ -49,8 +49,8 @@ describe('itemController.storagePath method', function() {
});
},
result: function(result, done) {
assert.equal(result, `/${this.params[0].source.kebabName()}/${this.params[0].contentType.pluralKebabName()}/${this.params[0].id}.json`);
assert.equal(result, `/${this.params[0].contentType.pluralLowercaseName()}/${this.params[0].source.lowercaseName()}-${this.params[1].id}.json`);
done();
}
}]);
});
});
Oops, something went wrong.

0 comments on commit fc5a2a2

Please sign in to comment.